diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala b/common/src/main/scala/org/apache/comet/CometConf.scala index b02f526e3..b8b4799da 100644 --- a/common/src/main/scala/org/apache/comet/CometConf.scala +++ b/common/src/main/scala/org/apache/comet/CometConf.scala @@ -197,11 +197,11 @@ object CometConf extends ShimCometConf { "'native' is for native shuffle which has best performance in general. " + "'jvm' is for jvm-based columnar shuffle which has higher coverage than native shuffle. " + "'auto' is for Comet to choose the best shuffle mode based on the query plan. " + - "By default, this config is 'jvm'.") + "By default, this config is 'auto'.") .stringConf .transform(_.toLowerCase(Locale.ROOT)) .checkValues(Set("native", "jvm", "auto")) - .createWithDefault("jvm") + .createWithDefault("auto") val COMET_EXEC_BROADCAST_FORCE_ENABLED: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.broadcast.enabled") diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 0679bacbb..b45d2588b 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -51,7 +51,7 @@ Comet provides the following configuration settings. | spark.comet.exec.project.enabled | Whether to enable project by default. | true | | spark.comet.exec.shuffle.codec | The codec of Comet native shuffle used to compress shuffle data. Only zstd is supported. | zstd | | spark.comet.exec.shuffle.enabled | Whether to enable Comet native shuffle. By default, this config is false. Note that this requires setting 'spark.shuffle.manager' to 'org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager'. 'spark.shuffle.manager' must be set before starting the Spark application and cannot be changed during the application. | false | -| spark.comet.exec.shuffle.mode | The mode of Comet shuffle. This config is only effective if Comet shuffle is enabled. Available modes are 'native', 'jvm', and 'auto'. 'native' is for native shuffle which has best performance in general. 'jvm' is for jvm-based columnar shuffle which has higher coverage than native shuffle. 'auto' is for Comet to choose the best shuffle mode based on the query plan. By default, this config is 'jvm'. | jvm | +| spark.comet.exec.shuffle.mode | The mode of Comet shuffle. This config is only effective if Comet shuffle is enabled. Available modes are 'native', 'jvm', and 'auto'. 'native' is for native shuffle which has best performance in general. 'jvm' is for jvm-based columnar shuffle which has higher coverage than native shuffle. 'auto' is for Comet to choose the best shuffle mode based on the query plan. By default, this config is 'auto'. | auto | | spark.comet.exec.sort.enabled | Whether to enable sort by default. | true | | spark.comet.exec.sortMergeJoin.enabled | Whether to enable sortMergeJoin by default. | true | | spark.comet.exec.stddev.enabled | Whether to enable stddev by default. stddev is slower than Spark's implementation. | true | diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt index 3752ee154..707707a74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/explain.txt @@ -9,7 +9,7 @@ : : +- CometBroadcastHashJoin (26) : : :- CometFilter (12) : : : +- CometHashAggregate (11) - : : : +- CometColumnarExchange (10) + : : : +- CometExchange (10) : : : +- CometHashAggregate (9) : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) @@ -22,10 +22,10 @@ : : +- CometBroadcastExchange (25) : : +- CometFilter (24) : : +- CometHashAggregate (23) - : : +- CometColumnarExchange (22) + : : +- CometExchange (22) : : +- CometHashAggregate (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -86,9 +86,9 @@ Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -(10) CometColumnarExchange +(10) CometExchange Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] @@ -128,9 +128,9 @@ Input [3]: [sr_customer_sk#12, sr_store_sk#13, sr_return_amt#14] Keys [2]: [sr_customer_sk#12, sr_store_sk#13] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#14))] -(19) CometColumnarExchange +(19) CometExchange Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#18] -Arguments: hashpartitioning(sr_customer_sk#12, sr_store_sk#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_customer_sk#12, sr_store_sk#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometHashAggregate Input [3]: [sr_customer_sk#12, sr_store_sk#13, sum#18] @@ -142,9 +142,9 @@ Input [2]: [ctr_store_sk#19, ctr_total_return#20] Keys [1]: [ctr_store_sk#19] Functions [1]: [partial_avg(ctr_total_return#20)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ctr_store_sk#19, sum#21, count#22] -Arguments: hashpartitioning(ctr_store_sk#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ctr_store_sk#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (23) CometHashAggregate Input [3]: [ctr_store_sk#19, sum#21, count#22] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/simplified.txt index 28b22a2e8..5aa8750a7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q1/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ctr_customer_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_store_sk] CometFilter [ctr_customer_sk,ctr_store_sk,ctr_total_return] CometHashAggregate [ctr_customer_sk,ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum,sum(UnscaledValue(sr_return_amt))] - CometColumnarExchange [sr_customer_sk,sr_store_sk] #1 + CometExchange [sr_customer_sk,sr_store_sk] #1 CometHashAggregate [sr_customer_sk,sr_store_sk,sum,sr_return_amt] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] @@ -31,10 +31,10 @@ WholeStageCodegen (1) CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #4 CometFilter [(avg(ctr_total_return) * 1.2),ctr_store_sk] CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_store_sk,sum,count,avg(ctr_total_return)] - CometColumnarExchange [ctr_store_sk] #5 + CometExchange [ctr_store_sk] #5 CometHashAggregate [ctr_store_sk,sum,count,ctr_total_return] CometHashAggregate [ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum,sum(UnscaledValue(sr_return_amt))] - CometColumnarExchange [sr_customer_sk,sr_store_sk] #6 + CometExchange [sr_customer_sk,sr_store_sk] #6 CometHashAggregate [sr_customer_sk,sr_store_sk,sum,sr_return_amt] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/explain.txt index 6f8c539fb..4a29b7260 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/explain.txt @@ -1,51 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- * ColumnarToRow (45) - +- CometColumnarExchange (44) - +- RowToColumnar (43) - +- * HashAggregate (42) - +- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (33) - : +- * ColumnarToRow (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (39) - +- * ColumnarToRow (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.customer_demographics (36) +TakeOrderedAndProject (45) ++- * HashAggregate (44) + +- Exchange (43) + +- * HashAggregate (42) + +- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * ColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (39) + +- * ColumnarToRow (38) + +- CometFilter (37) + +- CometScan parquet spark_catalog.default.customer_demographics (36) (1) Scan parquet spark_catalog.default.customer @@ -245,56 +243,50 @@ Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#31] Results [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] -(43) RowToColumnar +(43) Exchange Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(44) CometColumnarExchange -Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] -Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(45) ColumnarToRow [codegen id : 6] -Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] - -(46) HashAggregate [codegen id : 6] +(44) HashAggregate [codegen id : 6] Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] Keys [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#33] Results [14]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, count(1)#33 AS cnt1#34, cd_purchase_estimate#26, count(1)#33 AS cnt2#35, cd_credit_rating#27, count(1)#33 AS cnt3#36, cd_dep_count#28, count(1)#33 AS cnt4#37, cd_dep_employed_count#29, count(1)#33 AS cnt5#38, cd_dep_college_count#30, count(1)#33 AS cnt6#39] -(47) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [14]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#34, cd_purchase_estimate#26, cnt2#35, cd_credit_rating#27, cnt3#36, cd_dep_count#28, cnt4#37, cd_dep_employed_count#29, cnt5#38, cd_dep_college_count#30, cnt6#39] Arguments: 100, [cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_education_status#25 ASC NULLS FIRST, cd_purchase_estimate#26 ASC NULLS FIRST, cd_credit_rating#27 ASC NULLS FIRST, cd_dep_count#28 ASC NULLS FIRST, cd_dep_employed_count#29 ASC NULLS FIRST, cd_dep_college_count#30 ASC NULLS FIRST], [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#34, cd_purchase_estimate#26, cnt2#35, cd_credit_rating#27, cnt3#36, cd_dep_count#28, cnt4#37, cd_dep_employed_count#29, cnt5#38, cd_dep_college_count#30, cnt6#39] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * ColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan parquet spark_catalog.default.date_dim (48) +BroadcastExchange (50) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(48) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(49) CometFilter +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Condition : (((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2002)) AND (d_moy#11 >= 1)) AND (d_moy#11 <= 4)) AND isnotnull(d_date_sk#9)) -(50) CometProject +(48) CometProject Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Arguments: [d_date_sk#9], [d_date_sk#9] -(51) ColumnarToRow [codegen id : 1] +(49) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(52) BroadcastExchange +(50) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt index a3ab6bbbb..efd4b187d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q10/simplified.txt @@ -1,73 +1,71 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + InputAdapter + Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_county] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt index ac0d82f80..0346efa54 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/explain.txt @@ -9,7 +9,7 @@ : : +- CometBroadcastHashJoin (33) : : :- CometFilter (16) : : : +- CometHashAggregate (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometHashAggregate (13) : : : +- CometProject (12) : : : +- CometBroadcastHashJoin (11) @@ -25,7 +25,7 @@ : : : +- CometScan parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometProject (28) : : +- CometBroadcastHashJoin (27) @@ -42,7 +42,7 @@ : +- CometBroadcastExchange (49) : +- CometFilter (48) : +- CometHashAggregate (47) - : +- CometColumnarExchange (46) + : +- CometExchange (46) : +- CometHashAggregate (45) : +- CometProject (44) : +- CometBroadcastHashJoin (43) @@ -56,7 +56,7 @@ : +- ReusedExchange (42) +- CometBroadcastExchange (65) +- CometHashAggregate (64) - +- CometColumnarExchange (63) + +- CometExchange (63) +- CometHashAggregate (62) +- CometProject (61) +- CometBroadcastHashJoin (60) @@ -135,9 +135,9 @@ Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_fl Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -(14) CometColumnarExchange +(14) CometExchange Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] @@ -213,9 +213,9 @@ Input [10]: [c_customer_id#20, c_first_name#21, c_last_name#22, c_preferred_cust Keys [8]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#29 - ss_ext_discount_amt#28)))] -(30) CometColumnarExchange +(30) CometExchange Input [9]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, sum#34] -Arguments: hashpartitioning(c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate Input [9]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, sum#34] @@ -288,9 +288,9 @@ Input [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust Keys [8]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#52] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#48 - ws_ext_discount_amt#47)))] -(46) CometColumnarExchange +(46) CometExchange Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#52, sum#53] -Arguments: hashpartitioning(c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#52, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (47) CometHashAggregate Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#52, sum#53] @@ -367,9 +367,9 @@ Input [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust Keys [8]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#70] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#66 - ws_ext_discount_amt#65)))] -(63) CometColumnarExchange +(63) CometExchange Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#70, sum#71] -Arguments: hashpartitioning(c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#70, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#70, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (64) CometHashAggregate Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#70, sum#71] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt index 83ffbd0a6..e509c8c3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q11/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_preferred_cust_flag,year_total] CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -33,7 +33,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #5 CometHashAggregate [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -57,7 +57,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -72,7 +72,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt index 40c6660a9..572f500c9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (22) +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -92,18 +92,18 @@ Input [6]: [ws_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt index 3e0fd16a4..e81a50713 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q12/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c ColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometColumnarExchange [i_class] #1 + CometExchange [i_class] #1 CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt index b05df2927..7d035666a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (33) +- CometHashAggregate (32) - +- CometColumnarExchange (31) + +- CometExchange (31) +- CometHashAggregate (30) +- CometProject (29) +- CometBroadcastHashJoin (28) @@ -179,9 +179,9 @@ Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] Keys: [] Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] -(31) CometColumnarExchange +(31) CometExchange Input [7]: [sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [7]: [sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt index 0f438f153..46ed9b636 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q13/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum,avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost))] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,count,sum,count,sum,count,sum,ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt index 81c7b9c42..c758aeb6f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/explain.txt @@ -2,14 +2,14 @@ * ColumnarToRow (102) +- CometTakeOrderedAndProject (101) +- CometHashAggregate (100) - +- CometColumnarExchange (99) + +- CometExchange (99) +- CometHashAggregate (98) +- CometExpand (97) +- CometUnion (96) :- CometProject (65) : +- CometFilter (64) : +- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometProject (60) : +- CometBroadcastHashJoin (59) @@ -26,7 +26,7 @@ : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) - : : : : +- CometColumnarExchange (31) + : : : : +- CometExchange (31) : : : : +- CometHashAggregate (30) : : : : +- CometProject (29) : : : : +- CometBroadcastHashJoin (28) @@ -74,7 +74,7 @@ :- CometProject (80) : +- CometFilter (79) : +- CometHashAggregate (78) - : +- CometColumnarExchange (77) + : +- CometExchange (77) : +- CometHashAggregate (76) : +- CometProject (75) : +- CometBroadcastHashJoin (74) @@ -89,7 +89,7 @@ +- CometProject (95) +- CometFilter (94) +- CometHashAggregate (93) - +- CometColumnarExchange (92) + +- CometExchange (92) +- CometHashAggregate (91) +- CometProject (90) +- CometBroadcastHashJoin (89) @@ -252,9 +252,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -393,9 +393,9 @@ Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_cate Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(62) CometColumnarExchange +(62) CometExchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] @@ -459,9 +459,9 @@ Input [5]: [cs_quantity#58, cs_list_price#59, i_brand_id#64, i_class_id#65, i_ca Keys [3]: [i_brand_id#64, i_class_id#65, i_category_id#66] Functions [2]: [partial_sum((cast(cs_quantity#58 as decimal(10,0)) * cs_list_price#59)), partial_count(1)] -(77) CometColumnarExchange +(77) CometExchange Input [6]: [i_brand_id#64, i_class_id#65, i_category_id#66, sum#68, isEmpty#69, count#70] -Arguments: hashpartitioning(i_brand_id#64, i_class_id#65, i_category_id#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_brand_id#64, i_class_id#65, i_category_id#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (78) CometHashAggregate Input [6]: [i_brand_id#64, i_class_id#65, i_category_id#66, sum#68, isEmpty#69, count#70] @@ -525,9 +525,9 @@ Input [5]: [ws_quantity#75, ws_list_price#76, i_brand_id#81, i_class_id#82, i_ca Keys [3]: [i_brand_id#81, i_class_id#82, i_category_id#83] Functions [2]: [partial_sum((cast(ws_quantity#75 as decimal(10,0)) * ws_list_price#76)), partial_count(1)] -(92) CometColumnarExchange +(92) CometExchange Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#85, isEmpty#86, count#87] -Arguments: hashpartitioning(i_brand_id#81, i_class_id#82, i_category_id#83, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_brand_id#81, i_class_id#82, i_category_id#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (93) CometHashAggregate Input [6]: [i_brand_id#81, i_class_id#82, i_category_id#83, sum#85, isEmpty#86, count#87] @@ -556,9 +556,9 @@ Input [7]: [sales#49, number_sales#50, channel#91, i_brand_id#92, i_class_id#93, Keys [5]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95] Functions [2]: [partial_sum(sales#49), partial_sum(number_sales#50)] -(99) CometColumnarExchange +(99) CometExchange Input [8]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95, sum#96, isEmpty#97, sum#98] -Arguments: hashpartitioning(channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (100) CometHashAggregate Input [8]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, spark_grouping_id#95, sum#96, isEmpty#97, sum#98] @@ -577,7 +577,7 @@ Input [6]: [channel#91, i_brand_id#92, i_class_id#93, i_category_id#94, sum(sale Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#51, [id=#52] * ColumnarToRow (119) +- CometHashAggregate (118) - +- CometColumnarExchange (117) + +- CometExchange (117) +- CometHashAggregate (116) +- CometUnion (115) :- CometProject (106) @@ -661,9 +661,9 @@ Input [2]: [quantity#106, list_price#107] Keys: [] Functions [1]: [partial_avg((cast(quantity#106 as decimal(10,0)) * list_price#107))] -(117) CometColumnarExchange +(117) CometExchange Input [2]: [sum#122, count#123] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (118) CometHashAggregate Input [2]: [sum#122, count#123] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt index bb69fc92a..852af8ad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14a/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),spark_grouping_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] - CometColumnarExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 + CometExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum,sales,number_sales] CometExpand [channel,i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] @@ -14,7 +14,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometColumnarExchange #15 + CometExchange #15 CometHashAggregate [sum,count,quantity,list_price] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] @@ -33,7 +33,7 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #2 + CometExchange [i_brand_id,i_class_id,i_category_id] #2 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -58,7 +58,7 @@ WholeStageCodegen (1) CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #6 + CometExchange [brand_id,class_id,category_id] #6 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -117,7 +117,7 @@ WholeStageCodegen (1) CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #16 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,cs_quantity,cs_list_price] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -134,7 +134,7 @@ WholeStageCodegen (1) CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #17 + CometExchange [i_brand_id,i_class_id,i_category_id] #17 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ws_quantity,ws_list_price] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt index 7bb99e447..95ed68f76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/explain.txt @@ -4,7 +4,7 @@ +- CometBroadcastHashJoin (83) :- CometFilter (64) : +- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometProject (60) : +- CometBroadcastHashJoin (59) @@ -21,7 +21,7 @@ : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) - : : : : +- CometColumnarExchange (31) + : : : : +- CometExchange (31) : : : : +- CometHashAggregate (30) : : : : +- CometProject (29) : : : : +- CometBroadcastHashJoin (28) @@ -69,7 +69,7 @@ +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) - +- CometColumnarExchange (79) + +- CometExchange (79) +- CometHashAggregate (78) +- CometProject (77) +- CometBroadcastHashJoin (76) @@ -235,9 +235,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -376,9 +376,9 @@ Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_cate Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(62) CometColumnarExchange +(62) CometExchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] @@ -454,9 +454,9 @@ Input [5]: [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_ca Keys [3]: [i_brand_id#62, i_class_id#63, i_category_id#64] Functions [2]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), partial_count(1)] -(79) CometColumnarExchange +(79) CometExchange Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] -Arguments: hashpartitioning(i_brand_id#62, i_class_id#63, i_category_id#64, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_brand_id#62, i_class_id#63, i_category_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (80) CometHashAggregate Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] @@ -488,7 +488,7 @@ Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#5 Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#53, [id=#54] * ColumnarToRow (102) +- CometHashAggregate (101) - +- CometColumnarExchange (100) + +- CometExchange (100) +- CometHashAggregate (99) +- CometUnion (98) :- CometProject (89) @@ -572,9 +572,9 @@ Input [2]: [quantity#80, list_price#81] Keys: [] Functions [1]: [partial_avg((cast(quantity#80 as decimal(10,0)) * list_price#81))] -(100) CometColumnarExchange +(100) CometExchange Input [2]: [sum#96, count#97] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (101) CometHashAggregate Input [2]: [sum#96, count#97] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt index a51d1d007..bb4be4d3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q14b/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometColumnarExchange #14 + CometExchange #14 CometHashAggregate [sum,count,quantity,list_price] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] @@ -28,7 +28,7 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #1 + CometExchange [i_brand_id,i_class_id,i_category_id] #1 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -60,7 +60,7 @@ WholeStageCodegen (1) CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #5 + CometExchange [brand_id,class_id,category_id] #5 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -120,7 +120,7 @@ WholeStageCodegen (1) CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #16 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt index af9c0909f..842c3e576 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (23) +- CometTakeOrderedAndProject (22) +- CometHashAggregate (21) - +- CometColumnarExchange (20) + +- CometExchange (20) +- CometHashAggregate (19) +- CometProject (18) +- CometBroadcastHashJoin (17) @@ -117,9 +117,9 @@ Input [2]: [cs_sales_price#2, ca_zip#9] Keys [1]: [ca_zip#9] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] -(20) CometColumnarExchange +(20) CometExchange Input [2]: [ca_zip#9, sum#13] -Arguments: hashpartitioning(ca_zip#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ca_zip#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [2]: [ca_zip#9, sum#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt index 989ea0856..d417f1c68 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q15/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [ca_zip,sum(cs_sales_price)] CometHashAggregate [ca_zip,sum(cs_sales_price),sum,sum(UnscaledValue(cs_sales_price))] - CometColumnarExchange [ca_zip] #1 + CometExchange [ca_zip] #1 CometHashAggregate [ca_zip,sum,cs_sales_price] CometProject [cs_sales_price,ca_zip] CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt index ebff95d06..4890b4436 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/explain.txt @@ -1,51 +1,49 @@ == Physical Plan == -* HashAggregate (47) -+- * ColumnarToRow (46) - +- CometColumnarExchange (45) - +- RowToColumnar (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * SortMergeJoin LeftAnti (19) - : : : :- * Project (13) - : : : : +- * SortMergeJoin LeftSemi (12) - : : : : :- * ColumnarToRow (6) - : : : : : +- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : +- * ColumnarToRow (11) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (7) - : : : +- * ColumnarToRow (18) - : : : +- CometSort (17) - : : : +- CometColumnarExchange (16) - : : : +- CometProject (15) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (14) - : : +- BroadcastExchange (24) - : : +- * ColumnarToRow (23) - : : +- CometProject (22) - : : +- CometFilter (21) - : : +- CometScan parquet spark_catalog.default.date_dim (20) - : +- BroadcastExchange (31) - : +- * ColumnarToRow (30) - : +- CometProject (29) - : +- CometFilter (28) - : +- CometScan parquet spark_catalog.default.customer_address (27) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan parquet spark_catalog.default.call_center (34) +* HashAggregate (45) ++- Exchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (33) + : +- * BroadcastHashJoin Inner BuildRight (32) + : :- * Project (26) + : : +- * BroadcastHashJoin Inner BuildRight (25) + : : :- * SortMergeJoin LeftAnti (19) + : : : :- * Project (13) + : : : : +- * SortMergeJoin LeftSemi (12) + : : : : :- * ColumnarToRow (6) + : : : : : +- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- * ColumnarToRow (11) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (7) + : : : +- * ColumnarToRow (18) + : : : +- CometSort (17) + : : : +- CometExchange (16) + : : : +- CometProject (15) + : : : +- CometScan parquet spark_catalog.default.catalog_returns (14) + : : +- BroadcastExchange (24) + : : +- * ColumnarToRow (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometScan parquet spark_catalog.default.date_dim (20) + : +- BroadcastExchange (31) + : +- * ColumnarToRow (30) + : +- CometProject (29) + : +- CometFilter (28) + : +- CometScan parquet spark_catalog.default.customer_address (27) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan parquet spark_catalog.default.call_center (34) (1) Scan parquet spark_catalog.default.catalog_sales @@ -63,9 +61,9 @@ Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(4) CometColumnarExchange +(4) CometExchange Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] @@ -84,9 +82,9 @@ ReadSchema: struct Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order_number#10] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [cs_warehouse_sk#9, cs_order_number#10] @@ -115,9 +113,9 @@ ReadSchema: struct Input [2]: [cr_order_number#12, cr_returned_date_sk#13] Arguments: [cr_order_number#12], [cr_order_number#12] -(16) CometColumnarExchange +(16) CometExchange Input [1]: [cr_order_number#12] -Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (17) CometSort Input [1]: [cr_order_number#12] @@ -249,17 +247,11 @@ Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] Results [3]: [sum#22, sum#23, count#25] -(44) RowToColumnar +(44) Exchange Input [3]: [sum#22, sum#23, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(45) CometColumnarExchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(46) ColumnarToRow [codegen id : 9] -Input [3]: [sum#22, sum#23, count#25] - -(47) HashAggregate [codegen id : 9] +(45) HashAggregate [codegen id : 9] Input [3]: [sum#22, sum#23, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt index 605ce3703..18ba39326 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q16/simplified.txt @@ -1,70 +1,68 @@ WholeStageCodegen (9) HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (8) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - Project [cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk] - Project [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk] - SortMergeJoin [cs_order_number,cr_order_number] - InputAdapter - WholeStageCodegen (3) - Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometColumnarExchange [cs_order_number] #2 - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [cs_warehouse_sk,cs_order_number] - CometColumnarExchange [cs_order_number] #3 - CometProject [cs_warehouse_sk,cs_order_number] - CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [cr_order_number] - CometColumnarExchange [cr_order_number] #4 - CometProject [cr_order_number] - CometScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (8) + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + Project [cs_order_number,cs_ext_ship_cost,cs_net_profit] + BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] + Project [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk] + Project [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + BroadcastHashJoin [cs_ship_date_sk,d_date_sk] + SortMergeJoin [cs_order_number,cr_order_number] + InputAdapter + WholeStageCodegen (3) + Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometExchange [cs_order_number] #2 + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + InputAdapter + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometSort [cs_warehouse_sk,cs_order_number] + CometExchange [cs_order_number] #3 + CometProject [cs_warehouse_sk,cs_order_number] + CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + InputAdapter + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [cr_order_number] + CometExchange [cr_order_number] #4 + CometProject [cr_order_number] + CometScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) + BroadcastExchange #5 + WholeStageCodegen (5) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) + BroadcastExchange #6 + WholeStageCodegen (6) ColumnarToRow InputAdapter - CometProject [cc_call_center_sk] - CometFilter [cc_call_center_sk,cc_county] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometProject [cc_call_center_sk] + CometFilter [cc_call_center_sk,cc_county] + CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt index ddf62fe94..69f720e6f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (42) +- CometTakeOrderedAndProject (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -226,9 +226,9 @@ Input [6]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#25, i_ Keys [3]: [i_item_id#27, i_item_desc#28, s_state#25] Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#11), partial_avg(sr_return_quantity#11), partial_stddev_samp(cast(sr_return_quantity#11 as double)), partial_count(cs_quantity#16), partial_avg(cs_quantity#16), partial_stddev_samp(cast(cs_quantity#16 as double))] -(39) CometColumnarExchange +(39) CometExchange Input [21]: [i_item_id#27, i_item_desc#28, s_state#25, count#29, sum#30, count#31, n#32, avg#33, m2#34, count#35, sum#36, count#37, n#38, avg#39, m2#40, count#41, sum#42, count#43, n#44, avg#45, m2#46] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_state#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_state#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate Input [21]: [i_item_id#27, i_item_desc#28, s_state#25, count#29, sum#30, count#31, n#32, avg#33, m2#34, count#35, sum#36, count#37, n#38, avg#39, m2#40, count#41, sum#42, count#43, n#44, avg#45, m2#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt index e350dfe51..a3adaf172 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q17/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] CometHashAggregate [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] - CometColumnarExchange [i_item_id,i_item_desc,s_state] #1 + CometExchange [i_item_id,i_item_desc,s_state] #1 CometHashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,ss_quantity,sr_return_quantity,cs_quantity] CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt index 6c969939d..bd8c5eb57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (41) +- CometTakeOrderedAndProject (40) +- CometHashAggregate (39) - +- CometColumnarExchange (38) + +- CometExchange (38) +- CometHashAggregate (37) +- CometExpand (36) +- CometProject (35) @@ -219,9 +219,9 @@ Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, Keys [5]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33] Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#19 as decimal(12,2))), partial_avg(cast(cd_dep_count#14 as decimal(12,2)))] -(38) CometColumnarExchange +(38) CometExchange Input [19]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, sum#34, count#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47] -Arguments: hashpartitioning(i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (39) CometHashAggregate Input [19]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, sum#34, count#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/simplified.txt index 8182da775..e26491c76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q18/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] - CometColumnarExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 + CometExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/explain.txt index 2b3480d80..2191317fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (34) +- CometTakeOrderedAndProject (33) +- CometHashAggregate (32) - +- CometColumnarExchange (31) + +- CometExchange (31) +- CometHashAggregate (30) +- CometProject (29) +- CometBroadcastHashJoin (28) @@ -180,9 +180,9 @@ Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] -(31) CometColumnarExchange +(31) CometExchange Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#21] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/simplified.txt index 8f81e32ff..46b0d650a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q19/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [brand_id,brand,i_manufact_id,i_manufact,ext_price] CometHashAggregate [brand_id,brand,i_manufact_id,i_manufact,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 + CometExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 CometHashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/explain.txt index d37261260..35e00a7d2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/explain.txt @@ -7,7 +7,7 @@ :- CometProject (19) : +- CometBroadcastHashJoin (18) : :- CometHashAggregate (13) - : : +- CometColumnarExchange (12) + : : +- CometExchange (12) : : +- CometHashAggregate (11) : : +- CometProject (10) : : +- CometBroadcastHashJoin (9) @@ -89,9 +89,9 @@ Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] Keys [1]: [d_week_seq#10] Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -(12) CometColumnarExchange +(12) CometExchange Input [8]: [d_week_seq#10, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (13) CometHashAggregate Input [8]: [d_week_seq#10, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt index c5c06a0b6..ed772bade 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q2/simplified.txt @@ -8,7 +8,7 @@ WholeStageCodegen (1) CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] CometHashAggregate [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] - CometColumnarExchange [d_week_seq] #2 + CometExchange [d_week_seq] #2 CometHashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum,d_day_name,sales_price] CometProject [sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt index c6982f073..1af8f20f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (22) +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -92,18 +92,18 @@ Input [6]: [cs_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt index 9c30689f6..9a906085b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q20/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c ColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometColumnarExchange [i_class] #1 + CometExchange [i_class] #1 CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt index ff0f63b74..dce72db26 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (23) +- CometFilter (22) +- CometHashAggregate (21) - +- CometColumnarExchange (20) + +- CometExchange (20) +- CometHashAggregate (19) +- CometProject (18) +- CometBroadcastHashJoin (17) @@ -118,9 +118,9 @@ Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#9, d_date#12] Keys [2]: [w_warehouse_name#7, i_item_id#9] Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -(20) CometColumnarExchange +(20) CometExchange Input [4]: [w_warehouse_name#7, i_item_id#9, sum#13, sum#14] -Arguments: hashpartitioning(w_warehouse_name#7, i_item_id#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(w_warehouse_name#7, i_item_id#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [4]: [w_warehouse_name#7, i_item_id#9, sum#13, sum#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/simplified.txt index 1a4f175c4..6e483f604 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q21/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] CometFilter [w_warehouse_name,i_item_id,inv_before,inv_after] CometHashAggregate [w_warehouse_name,i_item_id,inv_before,inv_after,sum,sum,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] - CometColumnarExchange [w_warehouse_name,i_item_id] #1 + CometExchange [w_warehouse_name,i_item_id] #1 CometHashAggregate [w_warehouse_name,i_item_id,sum,sum,d_date,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/explain.txt index 1f7209c62..c3f3c8002 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (24) +- CometTakeOrderedAndProject (23) +- CometHashAggregate (22) - +- CometColumnarExchange (21) + +- CometExchange (21) +- CometHashAggregate (20) +- CometExpand (19) +- CometProject (18) @@ -122,9 +122,9 @@ Input [6]: [inv_quantity_on_hand#3, i_product_name#14, i_brand#15, i_class#16, i Keys [5]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -(21) CometColumnarExchange +(21) CometExchange Input [7]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, sum#19, count#20] -Arguments: hashpartitioning(i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate Input [7]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, sum#19, count#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/simplified.txt index 05d02283b..ed997b5d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q22/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,spark_grouping_id,sum,count,avg(inv_quantity_on_hand)] - CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + CometExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 CometHashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count,inv_quantity_on_hand] CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/explain.txt index 2af9c6082..c4d6d30d1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (67) +- CometHashAggregate (66) - +- CometColumnarExchange (65) + +- CometExchange (65) +- CometHashAggregate (64) +- CometUnion (63) :- CometProject (46) @@ -9,7 +9,7 @@ : :- CometProject (40) : : +- CometSortMergeJoin (39) : : :- CometSort (24) - : : : +- CometColumnarExchange (23) + : : : +- CometExchange (23) : : : +- CometProject (22) : : : +- CometBroadcastHashJoin (21) : : : :- CometScan parquet spark_catalog.default.catalog_sales (1) @@ -17,7 +17,7 @@ : : : +- CometProject (19) : : : +- CometFilter (18) : : : +- CometHashAggregate (17) - : : : +- CometColumnarExchange (16) + : : : +- CometExchange (16) : : : +- CometHashAggregate (15) : : : +- CometProject (14) : : : +- CometBroadcastHashJoin (13) @@ -36,7 +36,7 @@ : : +- CometProject (37) : : +- CometFilter (36) : : +- CometHashAggregate (35) - : : +- CometColumnarExchange (34) + : : +- CometExchange (34) : : +- CometHashAggregate (33) : : +- CometProject (32) : : +- CometBroadcastHashJoin (31) @@ -55,7 +55,7 @@ :- CometProject (59) : +- CometSortMergeJoin (58) : :- CometSort (52) - : : +- CometColumnarExchange (51) + : : +- CometExchange (51) : : +- CometProject (50) : : +- CometBroadcastHashJoin (49) : : :- CometScan parquet spark_catalog.default.web_sales (47) @@ -144,9 +144,9 @@ Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] Functions [1]: [partial_count(1)] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] @@ -174,9 +174,9 @@ Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(23) CometColumnarExchange +(23) CometExchange Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -226,9 +226,9 @@ Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] Keys [1]: [c_customer_sk#23] Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -(34) CometColumnarExchange +(34) CometExchange Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (35) CometHashAggregate Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] @@ -303,9 +303,9 @@ Arguments: [ws_item_sk#33], [item_sk#39], LeftSemi, BuildRight Input [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] Arguments: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -(51) CometColumnarExchange +(51) CometExchange Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: hashpartitioning(ws_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ws_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (52) CometSort Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] @@ -361,9 +361,9 @@ Input [1]: [sales#32] Keys: [] Functions [1]: [partial_sum(sales#32)] -(65) CometColumnarExchange +(65) CometExchange Input [2]: [sum#48, isEmpty#49] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (66) CometHashAggregate Input [2]: [sum#48, isEmpty#49] @@ -438,10 +438,10 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:3 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#27, [id=#28] * ColumnarToRow (95) +- CometHashAggregate (94) - +- CometColumnarExchange (93) + +- CometExchange (93) +- CometHashAggregate (92) +- CometHashAggregate (91) - +- CometColumnarExchange (90) + +- CometExchange (90) +- CometHashAggregate (89) +- CometProject (88) +- CometBroadcastHashJoin (87) @@ -513,9 +513,9 @@ Input [3]: [ss_quantity#52, ss_sales_price#53, c_customer_sk#56] Keys [1]: [c_customer_sk#56] Functions [1]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_sales_price#53))] -(90) CometColumnarExchange +(90) CometExchange Input [3]: [c_customer_sk#56, sum#59, isEmpty#60] -Arguments: hashpartitioning(c_customer_sk#56, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(c_customer_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (91) CometHashAggregate Input [3]: [c_customer_sk#56, sum#59, isEmpty#60] @@ -527,9 +527,9 @@ Input [1]: [csales#61] Keys: [] Functions [1]: [partial_max(csales#61)] -(93) CometColumnarExchange +(93) CometExchange Input [1]: [max#62] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (94) CometHashAggregate Input [1]: [max#62] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt index 18cf2b2c6..7606f105f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23a/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [sum(sales),sum,isEmpty,sum(sales)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,isEmpty,sales] CometUnion [sales] CometProject [cs_quantity,cs_list_price] [sales] @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometProject [cs_quantity,cs_list_price,cs_sold_date_sk] CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometColumnarExchange [cs_bill_customer_sk] #2 + CometExchange [cs_bill_customer_sk] #2 CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] @@ -26,7 +26,7 @@ WholeStageCodegen (1) CometProject [item_sk] CometFilter [item_sk,cnt] CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometColumnarExchange [_groupingexpression,i_item_sk,d_date] #5 + CometExchange [_groupingexpression,i_item_sk,d_date] #5 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] @@ -57,10 +57,10 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [tpcds_cmax,max,max(csales)] - CometColumnarExchange #11 + CometExchange #11 CometHashAggregate [max,csales] CometHashAggregate [csales,c_customer_sk,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometColumnarExchange [c_customer_sk] #12 + CometExchange [c_customer_sk] #12 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] @@ -82,7 +82,7 @@ WholeStageCodegen (1) CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometColumnarExchange [c_customer_sk] #9 + CometExchange [c_customer_sk] #9 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] @@ -101,7 +101,7 @@ WholeStageCodegen (1) CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometColumnarExchange [ws_bill_customer_sk] #16 + CometExchange [ws_bill_customer_sk] #16 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt index ee11e7897..4de12b9b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (86) +- CometUnion (85) :- CometHashAggregate (62) - : +- CometColumnarExchange (61) + : +- CometExchange (61) : +- CometHashAggregate (60) : +- CometProject (59) : +- CometBroadcastHashJoin (58) @@ -11,7 +11,7 @@ : : +- CometBroadcastHashJoin (52) : : :- CometSortMergeJoin (40) : : : :- CometSort (25) - : : : : +- CometColumnarExchange (24) + : : : : +- CometExchange (24) : : : : +- CometProject (23) : : : : +- CometBroadcastHashJoin (22) : : : : :- CometFilter (2) @@ -20,7 +20,7 @@ : : : : +- CometProject (20) : : : : +- CometFilter (19) : : : : +- CometHashAggregate (18) - : : : : +- CometColumnarExchange (17) + : : : : +- CometExchange (17) : : : : +- CometHashAggregate (16) : : : : +- CometProject (15) : : : : +- CometBroadcastHashJoin (14) @@ -39,7 +39,7 @@ : : : +- CometProject (38) : : : +- CometFilter (37) : : : +- CometHashAggregate (36) - : : : +- CometColumnarExchange (35) + : : : +- CometExchange (35) : : : +- CometHashAggregate (34) : : : +- CometProject (33) : : : +- CometBroadcastHashJoin (32) @@ -52,7 +52,7 @@ : : +- CometBroadcastExchange (51) : : +- CometSortMergeJoin (50) : : :- CometSort (44) - : : : +- CometColumnarExchange (43) + : : : +- CometExchange (43) : : : +- CometFilter (42) : : : +- CometScan parquet spark_catalog.default.customer (41) : : +- CometSort (49) @@ -65,7 +65,7 @@ : +- CometFilter (55) : +- CometScan parquet spark_catalog.default.date_dim (54) +- CometHashAggregate (84) - +- CometColumnarExchange (83) + +- CometExchange (83) +- CometHashAggregate (82) +- CometProject (81) +- CometBroadcastHashJoin (80) @@ -73,7 +73,7 @@ : +- CometBroadcastHashJoin (77) : :- CometSortMergeJoin (75) : : :- CometSort (69) - : : : +- CometColumnarExchange (68) + : : : +- CometExchange (68) : : : +- CometProject (67) : : : +- CometBroadcastHashJoin (66) : : : :- CometFilter (64) @@ -169,9 +169,9 @@ Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] Functions [1]: [partial_count(1)] -(17) CometColumnarExchange +(17) CometExchange Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] @@ -199,9 +199,9 @@ Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(24) CometColumnarExchange +(24) CometExchange Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (25) CometSort Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -251,9 +251,9 @@ Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] Keys [1]: [c_customer_sk#23] Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -(35) CometColumnarExchange +(35) CometExchange Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (36) CometHashAggregate Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] @@ -288,9 +288,9 @@ ReadSchema: struct Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] Condition : isnotnull(c_customer_sk#29) -(43) CometColumnarExchange +(43) CometExchange Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: hashpartitioning(c_customer_sk#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_sk#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (44) CometSort Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] @@ -367,9 +367,9 @@ Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -(61) CometColumnarExchange +(61) CometExchange Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] -Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (62) CometHashAggregate Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] @@ -400,9 +400,9 @@ Arguments: [ws_item_sk#37], [item_sk#43], LeftSemi, BuildRight Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] -(68) CometColumnarExchange +(68) CometExchange Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] -Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (69) CometSort Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] @@ -462,9 +462,9 @@ Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#51, c_last_name#52] Keys [2]: [c_last_name#52, c_first_name#51] Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] -(83) CometColumnarExchange +(83) CometExchange Input [4]: [c_last_name#52, c_first_name#51, sum#54, isEmpty#55] -Arguments: hashpartitioning(c_last_name#52, c_first_name#51, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(c_last_name#52, c_first_name#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (84) CometHashAggregate Input [4]: [c_last_name#52, c_first_name#51, sum#54, isEmpty#55] @@ -547,10 +547,10 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:3 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#27, [id=#28] * ColumnarToRow (115) +- CometHashAggregate (114) - +- CometColumnarExchange (113) + +- CometExchange (113) +- CometHashAggregate (112) +- CometHashAggregate (111) - +- CometColumnarExchange (110) + +- CometExchange (110) +- CometHashAggregate (109) +- CometProject (108) +- CometBroadcastHashJoin (107) @@ -622,9 +622,9 @@ Input [3]: [ss_quantity#59, ss_sales_price#60, c_customer_sk#63] Keys [1]: [c_customer_sk#63] Functions [1]: [partial_sum((cast(ss_quantity#59 as decimal(10,0)) * ss_sales_price#60))] -(110) CometColumnarExchange +(110) CometExchange Input [3]: [c_customer_sk#63, sum#66, isEmpty#67] -Arguments: hashpartitioning(c_customer_sk#63, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(c_customer_sk#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (111) CometHashAggregate Input [3]: [c_customer_sk#63, sum#66, isEmpty#67] @@ -636,9 +636,9 @@ Input [1]: [csales#68] Keys: [] Functions [1]: [partial_max(csales#68)] -(113) CometColumnarExchange +(113) CometExchange Input [1]: [max#69] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (114) CometHashAggregate Input [1]: [max#69] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt index 55f17ca34..d503bb68a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q23b/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [c_last_name,c_first_name,sales] CometUnion [c_last_name,c_first_name,sales] CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] - CometColumnarExchange [c_last_name,c_first_name] #1 + CometExchange [c_last_name,c_first_name] #1 CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,cs_quantity,cs_list_price] CometProject [cs_quantity,cs_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name,d_date_sk] @@ -12,7 +12,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometColumnarExchange [cs_bill_customer_sk] #2 + CometExchange [cs_bill_customer_sk] #2 CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] @@ -29,7 +29,7 @@ WholeStageCodegen (1) CometProject [item_sk] CometFilter [item_sk,cnt] CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometColumnarExchange [_groupingexpression,i_item_sk,d_date] #5 + CometExchange [_groupingexpression,i_item_sk,d_date] #5 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] @@ -60,10 +60,10 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [tpcds_cmax,max,max(csales)] - CometColumnarExchange #11 + CometExchange #11 CometHashAggregate [max,csales] CometHashAggregate [csales,c_customer_sk,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometColumnarExchange [c_customer_sk] #12 + CometExchange [c_customer_sk] #12 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] @@ -85,7 +85,7 @@ WholeStageCodegen (1) CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometColumnarExchange [c_customer_sk] #9 + CometExchange [c_customer_sk] #9 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] @@ -98,7 +98,7 @@ WholeStageCodegen (1) CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #15 CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] CometSort [c_customer_sk,c_first_name,c_last_name] - CometColumnarExchange [c_customer_sk] #16 + CometExchange [c_customer_sk] #16 CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometSort [c_customer_sk] @@ -112,7 +112,7 @@ WholeStageCodegen (1) CometFilter [d_date_sk,d_year,d_moy] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] - CometColumnarExchange [c_last_name,c_first_name] #18 + CometExchange [c_last_name,c_first_name] #18 CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,ws_quantity,ws_list_price] CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] @@ -120,7 +120,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometColumnarExchange [ws_bill_customer_sk] #19 + CometExchange [ws_bill_customer_sk] #19 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt index 7872cef74..3094c4e07 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/explain.txt @@ -1,50 +1,46 @@ == Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometColumnarExchange (38) - +- RowToColumnar (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) +* Filter (42) ++- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (21) + : : +- CometFilter (20) + : : +- CometScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.customer (24) + +- BroadcastExchange (33) + +- * ColumnarToRow (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.customer_address (30) (1) Scan parquet spark_catalog.default.store_sales @@ -62,9 +58,9 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(4) CometColumnarExchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -85,9 +81,9 @@ Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -216,216 +212,188 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#28] Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -(37) RowToColumnar +(37) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(38) CometColumnarExchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(39) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(40) HashAggregate [codegen id : 3] +(38) HashAggregate [codegen id : 3] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [partial_sum(netpaid#31)] Aggregate Attributes [2]: [sum#32, isEmpty#33] Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -(42) RowToColumnar -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(43) CometColumnarExchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [sum(netpaid#31)] Aggregate Attributes [1]: [sum(netpaid#31)#36] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (77) -+- * ColumnarToRow (76) - +- CometColumnarExchange (75) - +- RowToColumnar (74) - +- * HashAggregate (73) - +- * HashAggregate (72) - +- * ColumnarToRow (71) - +- CometColumnarExchange (70) - +- RowToColumnar (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * ColumnarToRow (64) - : +- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometProject (52) - : : : : +- CometSortMergeJoin (51) - : : : : :- CometSort (48) - : : : : : +- ReusedExchange (47) - : : : : +- CometSort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- CometBroadcastExchange (58) - : : +- CometFilter (57) - : : +- CometScan parquet spark_catalog.default.item (56) - : +- ReusedExchange (61) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 4] +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] +* HashAggregate (69) ++- Exchange (68) + +- * HashAggregate (67) + +- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * BroadcastHashJoin Inner BuildRight (62) + :- * ColumnarToRow (60) + : +- CometProject (59) + : +- CometBroadcastHashJoin (58) + : :- CometProject (56) + : : +- CometBroadcastHashJoin (55) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometProject (48) + : : : : +- CometSortMergeJoin (47) + : : : : :- CometSort (44) + : : : : : +- ReusedExchange (43) + : : : : +- CometSort (46) + : : : : +- ReusedExchange (45) + : : : +- ReusedExchange (49) + : : +- CometBroadcastExchange (54) + : : +- CometFilter (53) + : : +- CometScan parquet spark_catalog.default.item (52) + : +- ReusedExchange (57) + +- ReusedExchange (61) + + +(43) ReusedExchange [Reuses operator id: 4] Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -(48) CometSort +(44) CometSort Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] -(49) ReusedExchange [Reuses operator id: 9] +(45) ReusedExchange [Reuses operator id: 9] Output [2]: [sr_item_sk#45, sr_ticket_number#46] -(50) CometSort +(46) CometSort Input [2]: [sr_item_sk#45, sr_ticket_number#46] Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] -(51) CometSortMergeJoin +(47) CometSortMergeJoin Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] Right output [2]: [sr_item_sk#45, sr_ticket_number#46] Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner -(52) CometProject +(48) CometProject Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -(53) ReusedExchange [Reuses operator id: 16] +(49) ReusedExchange [Reuses operator id: 16] Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight -(55) CometProject +(51) CometProject Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -(56) Scan parquet spark_catalog.default.item +(52) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) CometFilter +(53) CometFilter Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Condition : isnotnull(i_item_sk#51) -(58) CometBroadcastExchange +(54) CometBroadcastExchange Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -(59) CometBroadcastHashJoin +(55) CometBroadcastHashJoin Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight -(60) CometProject +(56) CometProject Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -(61) ReusedExchange [Reuses operator id: 26] +(57) ReusedExchange [Reuses operator id: 26] Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -(62) CometBroadcastHashJoin +(58) CometBroadcastHashJoin Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight -(63) CometProject +(59) CometProject Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] -(64) ColumnarToRow [codegen id : 2] +(60) ColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] -(65) ReusedExchange [Reuses operator id: 33] +(61) ReusedExchange [Reuses operator id: 33] Output [3]: [ca_state#61, ca_zip#62, ca_country#63] -(66) BroadcastHashJoin [codegen id : 2] +(62) BroadcastHashJoin [codegen id : 2] Left keys [2]: [c_birth_country#60, s_zip#50] Right keys [2]: [upper(ca_country#63), ca_zip#62] Join type: Inner Join condition: None -(67) Project [codegen id : 2] +(63) Project [codegen id : 2] Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] -(68) HashAggregate [codegen id : 2] +(64) HashAggregate [codegen id : 2] Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] Aggregate Attributes [1]: [sum#64] Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -(69) RowToColumnar +(65) Exchange Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] +Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(70) CometColumnarExchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(71) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(72) HashAggregate [codegen id : 3] +(66) HashAggregate [codegen id : 3] Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] -(73) HashAggregate [codegen id : 3] +(67) HashAggregate [codegen id : 3] Input [1]: [netpaid#66] Keys: [] Functions [1]: [partial_avg(netpaid#66)] Aggregate Attributes [2]: [sum#67, count#68] Results [2]: [sum#69, count#70] -(74) RowToColumnar -Input [2]: [sum#69, count#70] - -(75) CometColumnarExchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(76) ColumnarToRow [codegen id : 4] +(68) Exchange Input [2]: [sum#69, count#70] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(77) HashAggregate [codegen id : 4] +(69) HashAggregate [codegen id : 4] Input [2]: [sum#69, count#70] Keys: [] Functions [1]: [avg(netpaid#66)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt index 0b53b6fc9..bd14d8250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24a/simplified.txt @@ -3,92 +3,84 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #9 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 - RowToColumnar + InputAdapter + Exchange #9 WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [netpaid] [sum,count,sum,count] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (1) + ColumnarToRow InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt index b5f3faefb..846c7a14a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/explain.txt @@ -1,50 +1,46 @@ == Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometColumnarExchange (38) - +- RowToColumnar (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) +* Filter (42) ++- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (21) + : : +- CometFilter (20) + : : +- CometScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.customer (24) + +- BroadcastExchange (33) + +- * ColumnarToRow (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.customer_address (30) (1) Scan parquet spark_catalog.default.store_sales @@ -62,9 +58,9 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(4) CometColumnarExchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -85,9 +81,9 @@ Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -216,216 +212,188 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#28] Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -(37) RowToColumnar +(37) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(38) CometColumnarExchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(39) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(40) HashAggregate [codegen id : 3] +(38) HashAggregate [codegen id : 3] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [partial_sum(netpaid#31)] Aggregate Attributes [2]: [sum#32, isEmpty#33] Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -(42) RowToColumnar -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(43) CometColumnarExchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [sum(netpaid#31)] Aggregate Attributes [1]: [sum(netpaid#31)#36] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (77) -+- * ColumnarToRow (76) - +- CometColumnarExchange (75) - +- RowToColumnar (74) - +- * HashAggregate (73) - +- * HashAggregate (72) - +- * ColumnarToRow (71) - +- CometColumnarExchange (70) - +- RowToColumnar (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * ColumnarToRow (64) - : +- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometProject (52) - : : : : +- CometSortMergeJoin (51) - : : : : :- CometSort (48) - : : : : : +- ReusedExchange (47) - : : : : +- CometSort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- CometBroadcastExchange (58) - : : +- CometFilter (57) - : : +- CometScan parquet spark_catalog.default.item (56) - : +- ReusedExchange (61) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 4] +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] +* HashAggregate (69) ++- Exchange (68) + +- * HashAggregate (67) + +- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * BroadcastHashJoin Inner BuildRight (62) + :- * ColumnarToRow (60) + : +- CometProject (59) + : +- CometBroadcastHashJoin (58) + : :- CometProject (56) + : : +- CometBroadcastHashJoin (55) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometProject (48) + : : : : +- CometSortMergeJoin (47) + : : : : :- CometSort (44) + : : : : : +- ReusedExchange (43) + : : : : +- CometSort (46) + : : : : +- ReusedExchange (45) + : : : +- ReusedExchange (49) + : : +- CometBroadcastExchange (54) + : : +- CometFilter (53) + : : +- CometScan parquet spark_catalog.default.item (52) + : +- ReusedExchange (57) + +- ReusedExchange (61) + + +(43) ReusedExchange [Reuses operator id: 4] Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -(48) CometSort +(44) CometSort Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] -(49) ReusedExchange [Reuses operator id: 9] +(45) ReusedExchange [Reuses operator id: 9] Output [2]: [sr_item_sk#45, sr_ticket_number#46] -(50) CometSort +(46) CometSort Input [2]: [sr_item_sk#45, sr_ticket_number#46] Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] -(51) CometSortMergeJoin +(47) CometSortMergeJoin Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] Right output [2]: [sr_item_sk#45, sr_ticket_number#46] Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner -(52) CometProject +(48) CometProject Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -(53) ReusedExchange [Reuses operator id: 16] +(49) ReusedExchange [Reuses operator id: 16] Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight -(55) CometProject +(51) CometProject Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -(56) Scan parquet spark_catalog.default.item +(52) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) CometFilter +(53) CometFilter Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Condition : isnotnull(i_item_sk#51) -(58) CometBroadcastExchange +(54) CometBroadcastExchange Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -(59) CometBroadcastHashJoin +(55) CometBroadcastHashJoin Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight -(60) CometProject +(56) CometProject Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -(61) ReusedExchange [Reuses operator id: 26] +(57) ReusedExchange [Reuses operator id: 26] Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -(62) CometBroadcastHashJoin +(58) CometBroadcastHashJoin Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight -(63) CometProject +(59) CometProject Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] -(64) ColumnarToRow [codegen id : 2] +(60) ColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] -(65) ReusedExchange [Reuses operator id: 33] +(61) ReusedExchange [Reuses operator id: 33] Output [3]: [ca_state#61, ca_zip#62, ca_country#63] -(66) BroadcastHashJoin [codegen id : 2] +(62) BroadcastHashJoin [codegen id : 2] Left keys [2]: [c_birth_country#60, s_zip#50] Right keys [2]: [upper(ca_country#63), ca_zip#62] Join type: Inner Join condition: None -(67) Project [codegen id : 2] +(63) Project [codegen id : 2] Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] -(68) HashAggregate [codegen id : 2] +(64) HashAggregate [codegen id : 2] Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] Aggregate Attributes [1]: [sum#64] Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -(69) RowToColumnar +(65) Exchange Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] +Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(70) CometColumnarExchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(71) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(72) HashAggregate [codegen id : 3] +(66) HashAggregate [codegen id : 3] Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] -(73) HashAggregate [codegen id : 3] +(67) HashAggregate [codegen id : 3] Input [1]: [netpaid#66] Keys: [] Functions [1]: [partial_avg(netpaid#66)] Aggregate Attributes [2]: [sum#67, count#68] Results [2]: [sum#69, count#70] -(74) RowToColumnar -Input [2]: [sum#69, count#70] - -(75) CometColumnarExchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(76) ColumnarToRow [codegen id : 4] +(68) Exchange Input [2]: [sum#69, count#70] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(77) HashAggregate [codegen id : 4] +(69) HashAggregate [codegen id : 4] Input [2]: [sum#69, count#70] Keys: [] Functions [1]: [avg(netpaid#66)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt index 0b53b6fc9..bd14d8250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q24b/simplified.txt @@ -3,92 +3,84 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #9 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 - RowToColumnar + InputAdapter + Exchange #9 WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [netpaid] [sum,count,sum,count] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (1) + ColumnarToRow InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt index f12717c3f..0d47bdcf1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (42) +- CometTakeOrderedAndProject (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -226,9 +226,9 @@ Input [7]: [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#27, s_ Keys [4]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28] Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#11)), partial_sum(UnscaledValue(cs_net_profit#16))] -(39) CometColumnarExchange +(39) CometExchange Input [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate Input [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, sum#32, sum#33, sum#34] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt index c2c652e2b..13cb05053 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q25/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit))] - CometColumnarExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum,ss_net_profit,sr_net_loss,cs_net_profit] CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt index cb4f53a0f..5ba7d13f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (30) +- CometTakeOrderedAndProject (29) +- CometHashAggregate (28) - +- CometColumnarExchange (27) + +- CometExchange (27) +- CometHashAggregate (26) +- CometProject (25) +- CometBroadcastHashJoin (24) @@ -156,9 +156,9 @@ Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] -(27) CometColumnarExchange +(27) CometExchange Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] -Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/simplified.txt index 08745014d..f643ff7b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q26/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] CometHashAggregate [i_item_id,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/explain.txt index 129139b03..150b31fd3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (30) +- CometTakeOrderedAndProject (29) +- CometHashAggregate (28) - +- CometColumnarExchange (27) + +- CometExchange (27) +- CometHashAggregate (26) +- CometExpand (25) +- CometProject (24) @@ -156,9 +156,9 @@ Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i Keys [3]: [i_item_id#20, s_state#21, spark_grouping_id#22] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -(27) CometColumnarExchange +(27) CometExchange Input [11]: [i_item_id#20, s_state#21, spark_grouping_id#22, sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Arguments: hashpartitioning(i_item_id#20, s_state#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#20, s_state#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate Input [11]: [i_item_id#20, s_state#21, spark_grouping_id#22, sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/simplified.txt index 0c2fe68c6..121f1be7c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q27/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_item_id,s_state,spark_grouping_id] #1 + CometExchange [i_item_id,s_state,spark_grouping_id] #1 CometHashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/explain.txt index 8881b95aa..29af7273c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/explain.txt @@ -1,86 +1,74 @@ == Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (82) -:- * BroadcastNestedLoopJoin Inner BuildRight (68) -: :- * BroadcastNestedLoopJoin Inner BuildRight (54) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (40) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (26) -: : : : :- * ColumnarToRow (12) -: : : : : +- CometHashAggregate (11) -: : : : : +- CometColumnarExchange (10) -: : : : : +- RowToColumnar (9) -: : : : : +- * HashAggregate (8) -: : : : : +- * HashAggregate (7) -: : : : : +- * ColumnarToRow (6) -: : : : : +- CometColumnarExchange (5) -: : : : : +- CometHashAggregate (4) -: : : : : +- CometProject (3) -: : : : : +- CometFilter (2) -: : : : : +- CometScan parquet spark_catalog.default.store_sales (1) -: : : : +- BroadcastExchange (25) -: : : : +- * ColumnarToRow (24) -: : : : +- CometHashAggregate (23) -: : : : +- CometColumnarExchange (22) -: : : : +- RowToColumnar (21) -: : : : +- * HashAggregate (20) -: : : : +- * HashAggregate (19) -: : : : +- * ColumnarToRow (18) -: : : : +- CometColumnarExchange (17) -: : : : +- CometHashAggregate (16) -: : : : +- CometProject (15) -: : : : +- CometFilter (14) -: : : : +- CometScan parquet spark_catalog.default.store_sales (13) -: : : +- BroadcastExchange (39) -: : : +- * ColumnarToRow (38) -: : : +- CometHashAggregate (37) -: : : +- CometColumnarExchange (36) -: : : +- RowToColumnar (35) -: : : +- * HashAggregate (34) -: : : +- * HashAggregate (33) -: : : +- * ColumnarToRow (32) -: : : +- CometColumnarExchange (31) -: : : +- CometHashAggregate (30) -: : : +- CometProject (29) -: : : +- CometFilter (28) -: : : +- CometScan parquet spark_catalog.default.store_sales (27) -: : +- BroadcastExchange (53) -: : +- * ColumnarToRow (52) -: : +- CometHashAggregate (51) -: : +- CometColumnarExchange (50) -: : +- RowToColumnar (49) -: : +- * HashAggregate (48) -: : +- * HashAggregate (47) -: : +- * ColumnarToRow (46) -: : +- CometColumnarExchange (45) -: : +- CometHashAggregate (44) -: : +- CometProject (43) -: : +- CometFilter (42) -: : +- CometScan parquet spark_catalog.default.store_sales (41) -: +- BroadcastExchange (67) -: +- * ColumnarToRow (66) -: +- CometHashAggregate (65) -: +- CometColumnarExchange (64) -: +- RowToColumnar (63) -: +- * HashAggregate (62) -: +- * HashAggregate (61) -: +- * ColumnarToRow (60) -: +- CometColumnarExchange (59) -: +- CometHashAggregate (58) -: +- CometProject (57) -: +- CometFilter (56) -: +- CometScan parquet spark_catalog.default.store_sales (55) -+- BroadcastExchange (81) - +- * ColumnarToRow (80) - +- CometHashAggregate (79) - +- CometColumnarExchange (78) - +- RowToColumnar (77) - +- * HashAggregate (76) - +- * HashAggregate (75) - +- * ColumnarToRow (74) - +- CometColumnarExchange (73) - +- CometHashAggregate (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan parquet spark_catalog.default.store_sales (69) +* BroadcastNestedLoopJoin Inner BuildRight (70) +:- * BroadcastNestedLoopJoin Inner BuildRight (58) +: :- * BroadcastNestedLoopJoin Inner BuildRight (46) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) +: : : : :- * HashAggregate (10) +: : : : : +- Exchange (9) +: : : : : +- * HashAggregate (8) +: : : : : +- * HashAggregate (7) +: : : : : +- * ColumnarToRow (6) +: : : : : +- CometExchange (5) +: : : : : +- CometHashAggregate (4) +: : : : : +- CometProject (3) +: : : : : +- CometFilter (2) +: : : : : +- CometScan parquet spark_catalog.default.store_sales (1) +: : : : +- BroadcastExchange (21) +: : : : +- * HashAggregate (20) +: : : : +- Exchange (19) +: : : : +- * HashAggregate (18) +: : : : +- * HashAggregate (17) +: : : : +- * ColumnarToRow (16) +: : : : +- CometExchange (15) +: : : : +- CometHashAggregate (14) +: : : : +- CometProject (13) +: : : : +- CometFilter (12) +: : : : +- CometScan parquet spark_catalog.default.store_sales (11) +: : : +- BroadcastExchange (33) +: : : +- * HashAggregate (32) +: : : +- Exchange (31) +: : : +- * HashAggregate (30) +: : : +- * HashAggregate (29) +: : : +- * ColumnarToRow (28) +: : : +- CometExchange (27) +: : : +- CometHashAggregate (26) +: : : +- CometProject (25) +: : : +- CometFilter (24) +: : : +- CometScan parquet spark_catalog.default.store_sales (23) +: : +- BroadcastExchange (45) +: : +- * HashAggregate (44) +: : +- Exchange (43) +: : +- * HashAggregate (42) +: : +- * HashAggregate (41) +: : +- * ColumnarToRow (40) +: : +- CometExchange (39) +: : +- CometHashAggregate (38) +: : +- CometProject (37) +: : +- CometFilter (36) +: : +- CometScan parquet spark_catalog.default.store_sales (35) +: +- BroadcastExchange (57) +: +- * HashAggregate (56) +: +- Exchange (55) +: +- * HashAggregate (54) +: +- * HashAggregate (53) +: +- * ColumnarToRow (52) +: +- CometExchange (51) +: +- CometHashAggregate (50) +: +- CometProject (49) +: +- CometFilter (48) +: +- CometScan parquet spark_catalog.default.store_sales (47) ++- BroadcastExchange (69) + +- * HashAggregate (68) + +- Exchange (67) + +- * HashAggregate (66) + +- * HashAggregate (65) + +- * ColumnarToRow (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometFilter (60) + +- CometScan parquet spark_catalog.default.store_sales (59) (1) Scan parquet spark_catalog.default.store_sales @@ -103,9 +91,9 @@ Input [1]: [ss_list_price#3] Keys [1]: [ss_list_price#3] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)] -(5) CometColumnarExchange +(5) CometExchange Input [4]: [ss_list_price#3, sum#6, count#7, count#8] -Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (6) ColumnarToRow [codegen id : 1] Input [4]: [ss_list_price#3, sum#6, count#7, count#8] @@ -124,338 +112,314 @@ Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_p Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] Results [4]: [sum#6, count#7, count#8, count#12] -(9) RowToColumnar +(9) Exchange Input [4]: [sum#6, count#7, count#8, count#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(10) CometColumnarExchange -Input [4]: [sum#6, count#7, count#8, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(11) CometHashAggregate +(10) HashAggregate [codegen id : 12] Input [4]: [sum#6, count#7, count#8, count#12] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#9 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#10 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] -(12) ColumnarToRow [codegen id : 12] -Input [3]: [B1_LP#13, B1_CNT#14, B1_CNTD#15] - -(13) Scan parquet spark_catalog.default.store_sales +(11) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] ReadSchema: struct -(14) CometFilter +(12) CometFilter Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) -(15) CometProject +(13) CometProject Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Arguments: [ss_list_price#18], [ss_list_price#18] -(16) CometHashAggregate +(14) CometHashAggregate Input [1]: [ss_list_price#18] Keys [1]: [ss_list_price#18] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] -(17) CometColumnarExchange +(15) CometExchange Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(18) ColumnarToRow [codegen id : 2] +(16) ColumnarToRow [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -(19) HashAggregate [codegen id : 2] +(17) HashAggregate [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys [1]: [ss_list_price#18] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25] Results [4]: [ss_list_price#18, sum#21, count#22, count#23] -(20) HashAggregate [codegen id : 2] +(18) HashAggregate [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] Results [4]: [sum#21, count#22, count#23, count#27] -(21) RowToColumnar +(19) Exchange Input [4]: [sum#21, count#22, count#23, count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] -(22) CometColumnarExchange -Input [4]: [sum#21, count#22, count#23, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(23) CometHashAggregate +(20) HashAggregate [codegen id : 3] Input [4]: [sum#21, count#22, count#23, count#27] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#24 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#25 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] -(24) ColumnarToRow [codegen id : 3] -Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] - -(25) BroadcastExchange +(21) BroadcastExchange Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] Arguments: IdentityBroadcastMode, [plan_id=5] -(26) BroadcastNestedLoopJoin [codegen id : 12] +(22) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(27) Scan parquet spark_catalog.default.store_sales +(23) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] ReadSchema: struct -(28) CometFilter +(24) CometFilter Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) -(29) CometProject +(25) CometProject Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Arguments: [ss_list_price#33], [ss_list_price#33] -(30) CometHashAggregate +(26) CometHashAggregate Input [1]: [ss_list_price#33] Keys [1]: [ss_list_price#33] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] -(31) CometColumnarExchange +(27) CometExchange Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(32) ColumnarToRow [codegen id : 4] +(28) ColumnarToRow [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -(33) HashAggregate [codegen id : 4] +(29) HashAggregate [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys [1]: [ss_list_price#33] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40] Results [4]: [ss_list_price#33, sum#36, count#37, count#38] -(34) HashAggregate [codegen id : 4] +(30) HashAggregate [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] Results [4]: [sum#36, count#37, count#38, count#42] -(35) RowToColumnar -Input [4]: [sum#36, count#37, count#38, count#42] - -(36) CometColumnarExchange +(31) Exchange Input [4]: [sum#36, count#37, count#38, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(37) CometHashAggregate +(32) HashAggregate [codegen id : 5] Input [4]: [sum#36, count#37, count#38, count#42] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#39 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#40 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] -(38) ColumnarToRow [codegen id : 5] -Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] - -(39) BroadcastExchange +(33) BroadcastExchange Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] Arguments: IdentityBroadcastMode, [plan_id=8] -(40) BroadcastNestedLoopJoin [codegen id : 12] +(34) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(41) Scan parquet spark_catalog.default.store_sales +(35) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] ReadSchema: struct -(42) CometFilter +(36) CometFilter Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) -(43) CometProject +(37) CometProject Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Arguments: [ss_list_price#48], [ss_list_price#48] -(44) CometHashAggregate +(38) CometHashAggregate Input [1]: [ss_list_price#48] Keys [1]: [ss_list_price#48] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] -(45) CometColumnarExchange +(39) CometExchange Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(46) ColumnarToRow [codegen id : 6] +(40) ColumnarToRow [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -(47) HashAggregate [codegen id : 6] +(41) HashAggregate [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys [1]: [ss_list_price#48] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55] Results [4]: [ss_list_price#48, sum#51, count#52, count#53] -(48) HashAggregate [codegen id : 6] +(42) HashAggregate [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] Results [4]: [sum#51, count#52, count#53, count#57] -(49) RowToColumnar +(43) Exchange Input [4]: [sum#51, count#52, count#53, count#57] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] -(50) CometColumnarExchange -Input [4]: [sum#51, count#52, count#53, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] - -(51) CometHashAggregate +(44) HashAggregate [codegen id : 7] Input [4]: [sum#51, count#52, count#53, count#57] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#54 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#55 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] -(52) ColumnarToRow [codegen id : 7] -Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] - -(53) BroadcastExchange +(45) BroadcastExchange Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] Arguments: IdentityBroadcastMode, [plan_id=11] -(54) BroadcastNestedLoopJoin [codegen id : 12] +(46) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(55) Scan parquet spark_catalog.default.store_sales +(47) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] ReadSchema: struct -(56) CometFilter +(48) CometFilter Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) -(57) CometProject +(49) CometProject Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Arguments: [ss_list_price#63], [ss_list_price#63] -(58) CometHashAggregate +(50) CometHashAggregate Input [1]: [ss_list_price#63] Keys [1]: [ss_list_price#63] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] -(59) CometColumnarExchange +(51) CometExchange Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(60) ColumnarToRow [codegen id : 8] +(52) ColumnarToRow [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -(61) HashAggregate [codegen id : 8] +(53) HashAggregate [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys [1]: [ss_list_price#63] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70] Results [4]: [ss_list_price#63, sum#66, count#67, count#68] -(62) HashAggregate [codegen id : 8] +(54) HashAggregate [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] Results [4]: [sum#66, count#67, count#68, count#72] -(63) RowToColumnar +(55) Exchange Input [4]: [sum#66, count#67, count#68, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] -(64) CometColumnarExchange -Input [4]: [sum#66, count#67, count#68, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] - -(65) CometHashAggregate +(56) HashAggregate [codegen id : 9] Input [4]: [sum#66, count#67, count#68, count#72] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#69 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#70 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] -(66) ColumnarToRow [codegen id : 9] -Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] - -(67) BroadcastExchange +(57) BroadcastExchange Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] Arguments: IdentityBroadcastMode, [plan_id=14] -(68) BroadcastNestedLoopJoin [codegen id : 12] +(58) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(69) Scan parquet spark_catalog.default.store_sales +(59) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] ReadSchema: struct -(70) CometFilter +(60) CometFilter Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) -(71) CometProject +(61) CometProject Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Arguments: [ss_list_price#78], [ss_list_price#78] -(72) CometHashAggregate +(62) CometHashAggregate Input [1]: [ss_list_price#78] Keys [1]: [ss_list_price#78] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] -(73) CometColumnarExchange +(63) CometExchange Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] +Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(74) ColumnarToRow [codegen id : 10] +(64) ColumnarToRow [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -(75) HashAggregate [codegen id : 10] +(65) HashAggregate [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys [1]: [ss_list_price#78] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85] Results [4]: [ss_list_price#78, sum#81, count#82, count#83] -(76) HashAggregate [codegen id : 10] +(66) HashAggregate [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] Results [4]: [sum#81, count#82, count#83, count#87] -(77) RowToColumnar -Input [4]: [sum#81, count#82, count#83, count#87] - -(78) CometColumnarExchange +(67) Exchange Input [4]: [sum#81, count#82, count#83, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] -(79) CometHashAggregate +(68) HashAggregate [codegen id : 11] Input [4]: [sum#81, count#82, count#83, count#87] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#84 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#85 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] -(80) ColumnarToRow [codegen id : 11] -Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] - -(81) BroadcastExchange +(69) BroadcastExchange Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] Arguments: IdentityBroadcastMode, [plan_id=17] -(82) BroadcastNestedLoopJoin [codegen id : 12] +(70) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/simplified.txt index 73343f39e..4a547c4e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q28/simplified.txt @@ -4,108 +4,96 @@ WholeStageCodegen (12) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - ColumnarToRow + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] InputAdapter - CometHashAggregate [B1_LP,B1_CNT,B1_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #2 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + Exchange #1 + WholeStageCodegen (1) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometExchange [ss_list_price] #2 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (3) - ColumnarToRow + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] InputAdapter - CometHashAggregate [B2_LP,B2_CNT,B2_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #4 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #5 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometHashAggregate [B3_LP,B3_CNT,B3_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #7 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #8 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometHashAggregate [B4_LP,B4_CNT,B4_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #10 - RowToColumnar - WholeStageCodegen (6) + Exchange #4 + WholeStageCodegen (2) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] ColumnarToRow InputAdapter - CometColumnarExchange [ss_list_price] #11 + CometExchange [ss_list_price] #5 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (9) - ColumnarToRow - InputAdapter - CometHashAggregate [B5_LP,B5_CNT,B5_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #13 - RowToColumnar - WholeStageCodegen (8) + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (5) + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] + InputAdapter + Exchange #7 + WholeStageCodegen (4) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] ColumnarToRow InputAdapter - CometColumnarExchange [ss_list_price] #14 + CometExchange [ss_list_price] #8 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometHashAggregate [B6_LP,B6_CNT,B6_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #16 - RowToColumnar - WholeStageCodegen (10) + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (7) + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] + InputAdapter + Exchange #10 + WholeStageCodegen (6) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] ColumnarToRow InputAdapter - CometColumnarExchange [ss_list_price] #17 + CometExchange [ss_list_price] #11 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (9) + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] + InputAdapter + Exchange #13 + WholeStageCodegen (8) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometExchange [ss_list_price] #14 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #15 + WholeStageCodegen (11) + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] + InputAdapter + Exchange #16 + WholeStageCodegen (10) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometExchange [ss_list_price] #17 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/explain.txt index a3346549e..6248ddee4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (45) +- CometTakeOrderedAndProject (44) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometProject (40) +- CometBroadcastHashJoin (39) @@ -245,9 +245,9 @@ Input [7]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#28, Keys [4]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29] Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#11), partial_sum(cs_quantity#16)] -(42) CometColumnarExchange +(42) CometExchange Input [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, sum#33, sum#34, sum#35] -Arguments: hashpartitioning(i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (43) CometHashAggregate Input [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, sum#33, sum#34, sum#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/simplified.txt index 3ca13e7b4..9398fcdf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q29/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] - CometColumnarExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum,ss_quantity,sr_return_quantity,cs_quantity] CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/explain.txt index 9079e0886..9a63fffa1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -93,9 +93,9 @@ Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/simplified.txt index 65c80baed..f82fd24bd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q3/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [d_year,brand_id,brand,sum_agg] CometHashAggregate [d_year,brand_id,brand,sum_agg,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [d_year,i_brand,i_brand_id] #1 + CometExchange [d_year,i_brand,i_brand_id] #1 CometHashAggregate [d_year,i_brand,i_brand_id,sum,ss_ext_sales_price] CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt index a14fa6074..3719c24f6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/explain.txt @@ -9,7 +9,7 @@ : : +- CometBroadcastHashJoin (34) : : :- CometFilter (17) : : : +- CometHashAggregate (16) - : : : +- CometColumnarExchange (15) + : : : +- CometExchange (15) : : : +- CometHashAggregate (14) : : : +- CometProject (13) : : : +- CometBroadcastHashJoin (12) @@ -27,10 +27,10 @@ : : +- CometBroadcastExchange (33) : : +- CometFilter (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometHashAggregate (28) - : : +- CometColumnarExchange (27) + : : +- CometExchange (27) : : +- CometHashAggregate (26) : : +- CometProject (25) : : +- CometBroadcastHashJoin (24) @@ -118,9 +118,9 @@ Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] @@ -172,9 +172,9 @@ Input [3]: [wr_returning_customer_sk#14, wr_return_amt#16, ca_state#21] Keys [2]: [wr_returning_customer_sk#14, ca_state#21] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#16))] -(27) CometColumnarExchange +(27) CometExchange Input [3]: [wr_returning_customer_sk#14, ca_state#21, sum#22] -Arguments: hashpartitioning(wr_returning_customer_sk#14, ca_state#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(wr_returning_customer_sk#14, ca_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [3]: [wr_returning_customer_sk#14, ca_state#21, sum#22] @@ -186,9 +186,9 @@ Input [2]: [ctr_state#23, ctr_total_return#24] Keys [1]: [ctr_state#23] Functions [1]: [partial_avg(ctr_total_return#24)] -(30) CometColumnarExchange +(30) CometExchange Input [3]: [ctr_state#23, sum#25, count#26] -Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (31) CometHashAggregate Input [3]: [ctr_state#23, sum#25, count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/simplified.txt index 576a8779f..824cc5116 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q30/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] CometHashAggregate [ctr_customer_sk,ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(wr_return_amt))] - CometColumnarExchange [wr_returning_customer_sk,ca_state] #1 + CometExchange [wr_returning_customer_sk,ca_state] #1 CometHashAggregate [wr_returning_customer_sk,ca_state,sum,wr_return_amt] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] @@ -36,10 +36,10 @@ WholeStageCodegen (1) CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #5 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_state,sum,count,avg(ctr_total_return)] - CometColumnarExchange [ctr_state] #6 + CometExchange [ctr_state] #6 CometHashAggregate [ctr_state,sum,count,ctr_total_return] CometHashAggregate [ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(wr_return_amt))] - CometColumnarExchange [wr_returning_customer_sk,ca_state] #7 + CometExchange [wr_returning_customer_sk,ca_state] #7 CometHashAggregate [wr_returning_customer_sk,ca_state,sum,wr_return_amt] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/explain.txt index 7284a48b6..e5bc4e7ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/explain.txt @@ -11,7 +11,7 @@ : : : +- CometBroadcastHashJoin (45) : : : :- CometBroadcastHashJoin (30) : : : : :- CometHashAggregate (15) - : : : : : +- CometColumnarExchange (14) + : : : : : +- CometExchange (14) : : : : : +- CometHashAggregate (13) : : : : : +- CometProject (12) : : : : : +- CometBroadcastHashJoin (11) @@ -27,7 +27,7 @@ : : : : : +- CometScan parquet spark_catalog.default.customer_address (8) : : : : +- CometBroadcastExchange (29) : : : : +- CometHashAggregate (28) - : : : : +- CometColumnarExchange (27) + : : : : +- CometExchange (27) : : : : +- CometHashAggregate (26) : : : : +- CometProject (25) : : : : +- CometBroadcastHashJoin (24) @@ -41,7 +41,7 @@ : : : : +- ReusedExchange (23) : : : +- CometBroadcastExchange (44) : : : +- CometHashAggregate (43) - : : : +- CometColumnarExchange (42) + : : : +- CometExchange (42) : : : +- CometHashAggregate (41) : : : +- CometProject (40) : : : +- CometBroadcastHashJoin (39) @@ -55,7 +55,7 @@ : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (58) : : +- CometHashAggregate (57) - : : +- CometColumnarExchange (56) + : : +- CometExchange (56) : : +- CometHashAggregate (55) : : +- CometProject (54) : : +- CometBroadcastHashJoin (53) @@ -67,7 +67,7 @@ : : +- ReusedExchange (52) : +- CometBroadcastExchange (71) : +- CometHashAggregate (70) - : +- CometColumnarExchange (69) + : +- CometExchange (69) : +- CometHashAggregate (68) : +- CometProject (67) : +- CometBroadcastHashJoin (66) @@ -79,7 +79,7 @@ : +- ReusedExchange (65) +- CometBroadcastExchange (85) +- CometHashAggregate (84) - +- CometColumnarExchange (83) + +- CometExchange (83) +- CometHashAggregate (82) +- CometProject (81) +- CometBroadcastHashJoin (80) @@ -156,9 +156,9 @@ Input [4]: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9] Keys [3]: [ca_county#9, d_qoy#7, d_year#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(14) CometColumnarExchange +(14) CometExchange Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] -Arguments: hashpartitioning(ca_county#9, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ca_county#9, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] @@ -218,9 +218,9 @@ Input [4]: [ss_ext_sales_price#12, d_year#16, d_qoy#17, ca_county#19] Keys [3]: [ca_county#19, d_qoy#17, d_year#16] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#12))] -(27) CometColumnarExchange +(27) CometExchange Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] -Arguments: hashpartitioning(ca_county#19, d_qoy#17, d_year#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ca_county#19, d_qoy#17, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] @@ -289,9 +289,9 @@ Input [4]: [ss_ext_sales_price#24, d_year#28, d_qoy#29, ca_county#31] Keys [3]: [ca_county#31, d_qoy#29, d_year#28] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#24))] -(42) CometColumnarExchange +(42) CometExchange Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#32] -Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (43) CometHashAggregate Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#32] @@ -352,9 +352,9 @@ Input [4]: [ws_ext_sales_price#35, d_year#39, d_qoy#40, ca_county#42] Keys [3]: [ca_county#42, d_qoy#40, d_year#39] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#35))] -(56) CometColumnarExchange +(56) CometExchange Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#43] -Arguments: hashpartitioning(ca_county#42, d_qoy#40, d_year#39, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ca_county#42, d_qoy#40, d_year#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (57) CometHashAggregate Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#43] @@ -411,9 +411,9 @@ Input [4]: [ws_ext_sales_price#46, d_year#50, d_qoy#51, ca_county#53] Keys [3]: [ca_county#53, d_qoy#51, d_year#50] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#46))] -(69) CometColumnarExchange +(69) CometExchange Input [4]: [ca_county#53, d_qoy#51, d_year#50, sum#54] -Arguments: hashpartitioning(ca_county#53, d_qoy#51, d_year#50, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ca_county#53, d_qoy#51, d_year#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (70) CometHashAggregate Input [4]: [ca_county#53, d_qoy#51, d_year#50, sum#54] @@ -474,9 +474,9 @@ Input [4]: [ws_ext_sales_price#57, d_year#61, d_qoy#62, ca_county#64] Keys [3]: [ca_county#64, d_qoy#62, d_year#61] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#57))] -(83) CometColumnarExchange +(83) CometExchange Input [4]: [ca_county#64, d_qoy#62, d_year#61, sum#65] -Arguments: hashpartitioning(ca_county#64, d_qoy#62, d_year#61, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ca_county#64, d_qoy#62, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (84) CometHashAggregate Input [4]: [ca_county#64, d_qoy#62, d_year#61, sum#65] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt index 6dcf5714d..d6615fe1c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q31/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales,ca_county,store_sales] CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales] CometHashAggregate [ca_county,d_year,store_sales,d_qoy,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #2 + CometExchange [ca_county,d_qoy,d_year] #2 CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -35,7 +35,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [ca_county,store_sales] #6 CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #7 + CometExchange [ca_county,d_qoy,d_year] #7 CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -56,7 +56,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,store_sales] #10 CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #11 + CometExchange [ca_county,d_qoy,d_year] #11 CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -77,7 +77,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,web_sales] #14 CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #15 + CometExchange [ca_county,d_qoy,d_year] #15 CometHashAggregate [ca_county,d_qoy,d_year,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -90,7 +90,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,web_sales] #16 CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #17 + CometExchange [ca_county,d_qoy,d_year] #17 CometHashAggregate [ca_county,d_qoy,d_year,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -103,7 +103,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,web_sales] #18 CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #19 + CometExchange [ca_county,d_qoy,d_year] #19 CometHashAggregate [ca_county,d_qoy,d_year,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt index 701a75e24..a0b604678 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (30) +- CometHashAggregate (29) - +- CometColumnarExchange (28) + +- CometExchange (28) +- CometHashAggregate (27) +- CometProject (26) +- CometBroadcastHashJoin (25) @@ -18,7 +18,7 @@ : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) - : +- CometColumnarExchange (18) + : +- CometExchange (18) : +- CometHashAggregate (17) : +- CometProject (16) : +- CometBroadcastHashJoin (15) @@ -116,9 +116,9 @@ Input [2]: [cs_item_sk#7, cs_ext_discount_amt#8] Keys [1]: [cs_item_sk#7] Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#8))] -(18) CometColumnarExchange +(18) CometExchange Input [3]: [cs_item_sk#7, sum#13, count#14] -Arguments: hashpartitioning(cs_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (19) CometHashAggregate Input [3]: [cs_item_sk#7, sum#13, count#14] @@ -159,9 +159,9 @@ Input [1]: [cs_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] -(28) CometColumnarExchange +(28) CometExchange Input [1]: [sum#17] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate Input [1]: [sum#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt index 19be8718e..14b2a34a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q32/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [excess discount amount,sum,sum(UnscaledValue(cs_ext_discount_amt))] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,cs_ext_discount_amt] CometProject [cs_ext_discount_amt] CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] @@ -27,7 +27,7 @@ WholeStageCodegen (1) CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #4 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] CometHashAggregate [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,sum,count,avg(UnscaledValue(cs_ext_discount_amt))] - CometColumnarExchange [cs_item_sk] #5 + CometExchange [cs_item_sk] #5 CometHashAggregate [cs_item_sk,sum,count,cs_ext_discount_amt] CometProject [cs_item_sk,cs_ext_discount_amt] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt index 6fb8b05fb..a432ecd29 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/explain.txt @@ -2,11 +2,11 @@ * ColumnarToRow (61) +- CometTakeOrderedAndProject (60) +- CometHashAggregate (59) - +- CometColumnarExchange (58) + +- CometExchange (58) +- CometHashAggregate (57) +- CometUnion (56) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -33,7 +33,7 @@ : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.item (17) :- CometHashAggregate (41) - : +- CometColumnarExchange (40) + : +- CometExchange (40) : +- CometHashAggregate (39) : +- CometProject (38) : +- CometBroadcastHashJoin (37) @@ -47,7 +47,7 @@ : : +- ReusedExchange (33) : +- ReusedExchange (36) +- CometHashAggregate (55) - +- CometColumnarExchange (54) + +- CometExchange (54) +- CometHashAggregate (53) +- CometProject (52) +- CometBroadcastHashJoin (51) @@ -183,9 +183,9 @@ Input [2]: [ss_ext_sales_price#3, i_manufact_id#12] Keys [1]: [i_manufact_id#12] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(26) CometColumnarExchange +(26) CometExchange Input [2]: [i_manufact_id#12, sum#15] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [2]: [i_manufact_id#12, sum#15] @@ -245,9 +245,9 @@ Input [2]: [cs_ext_sales_price#18, i_manufact_id#24] Keys [1]: [i_manufact_id#24] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] -(40) CometColumnarExchange +(40) CometExchange Input [2]: [i_manufact_id#24, sum#25] -Arguments: hashpartitioning(i_manufact_id#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_manufact_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (41) CometHashAggregate Input [2]: [i_manufact_id#24, sum#25] @@ -307,9 +307,9 @@ Input [2]: [ws_ext_sales_price#28, i_manufact_id#34] Keys [1]: [i_manufact_id#34] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] -(54) CometColumnarExchange +(54) CometExchange Input [2]: [i_manufact_id#34, sum#35] -Arguments: hashpartitioning(i_manufact_id#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_manufact_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (55) CometHashAggregate Input [2]: [i_manufact_id#34, sum#35] @@ -326,9 +326,9 @@ Input [2]: [i_manufact_id#12, total_sales#36] Keys [1]: [i_manufact_id#12] Functions [1]: [partial_sum(total_sales#36)] -(58) CometColumnarExchange +(58) CometExchange Input [3]: [i_manufact_id#12, sum#39, isEmpty#40] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (59) CometHashAggregate Input [3]: [i_manufact_id#12, sum#39, isEmpty#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt index 964f250ba..9556aa785 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q33/simplified.txt @@ -3,11 +3,11 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_manufact_id,total_sales] CometHashAggregate [i_manufact_id,total_sales,sum,isEmpty,sum(total_sales)] - CometColumnarExchange [i_manufact_id] #1 + CometExchange [i_manufact_id] #1 CometHashAggregate [i_manufact_id,sum,isEmpty,total_sales] CometUnion [i_manufact_id,total_sales] CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_manufact_id] #2 + CometExchange [i_manufact_id] #2 CometHashAggregate [i_manufact_id,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] @@ -42,7 +42,7 @@ WholeStageCodegen (1) CometFilter [i_category,i_manufact_id] CometScan parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_manufact_id] #8 + CometExchange [i_manufact_id] #8 CometHashAggregate [i_manufact_id,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] @@ -57,7 +57,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk] #5 ReusedExchange [i_item_sk,i_manufact_id] #6 CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_manufact_id] #9 + CometExchange [i_manufact_id] #9 CometHashAggregate [i_manufact_id,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt index c33735fa4..dccad5750 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/explain.txt @@ -6,7 +6,7 @@ +- CometBroadcastHashJoin (28) :- CometFilter (24) : +- CometHashAggregate (23) - : +- CometColumnarExchange (22) + : +- CometExchange (22) : +- CometHashAggregate (21) : +- CometProject (20) : +- CometBroadcastHashJoin (19) @@ -134,9 +134,9 @@ Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt index c131bd9c0..779d4cbdd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q34/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometColumnarExchange [ss_ticket_number,ss_customer_sk] #2 + CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt index b35338d18..c06c1dd16 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/explain.txt @@ -1,50 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- Exchange (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * ColumnarToRow (31) + : +- CometFilter (30) + : +- CometScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometFilter (36) + +- CometScan parquet spark_catalog.default.customer_demographics (35) (1) Scan parquet spark_catalog.default.customer @@ -240,56 +238,50 @@ Functions [10]: [partial_count(1), partial_min(cd_dep_count#25), partial_max(cd_ Aggregate Attributes [13]: [count#28, min#29, max#30, sum#31, count#32, min#33, max#34, sum#35, count#36, min#37, max#38, sum#39, count#40] Results [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] -(42) RowToColumnar +(42) Exchange Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) CometColumnarExchange -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] -Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 6] -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] - -(45) HashAggregate [codegen id : 6] +(43) HashAggregate [codegen id : 6] Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] Keys [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [10]: [count(1), min(cd_dep_count#25), max(cd_dep_count#25), avg(cd_dep_count#25), min(cd_dep_employed_count#26), max(cd_dep_employed_count#26), avg(cd_dep_employed_count#26), min(cd_dep_college_count#27), max(cd_dep_college_count#27), avg(cd_dep_college_count#27)] Aggregate Attributes [10]: [count(1)#54, min(cd_dep_count#25)#55, max(cd_dep_count#25)#56, avg(cd_dep_count#25)#57, min(cd_dep_employed_count#26)#58, max(cd_dep_employed_count#26)#59, avg(cd_dep_employed_count#26)#60, min(cd_dep_college_count#27)#61, max(cd_dep_college_count#27)#62, avg(cd_dep_college_count#27)#63] Results [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, count(1)#54 AS cnt1#64, min(cd_dep_count#25)#55 AS min(cd_dep_count)#65, max(cd_dep_count#25)#56 AS max(cd_dep_count)#66, avg(cd_dep_count#25)#57 AS avg(cd_dep_count)#67, cd_dep_employed_count#26, count(1)#54 AS cnt2#68, min(cd_dep_employed_count#26)#58 AS min(cd_dep_employed_count)#69, max(cd_dep_employed_count#26)#59 AS max(cd_dep_employed_count)#70, avg(cd_dep_employed_count#26)#60 AS avg(cd_dep_employed_count)#71, cd_dep_college_count#27, count(1)#54 AS cnt3#72, min(cd_dep_college_count#27)#61 AS min(cd_dep_college_count)#73, max(cd_dep_college_count#27)#62 AS max(cd_dep_college_count)#74, avg(cd_dep_college_count#27)#63 AS avg(cd_dep_college_count)#75, cd_dep_count#25] -(46) TakeOrderedAndProject +(44) TakeOrderedAndProject Input [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75, cd_dep_count#25] Arguments: 100, [ca_state#21 ASC NULLS FIRST, cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [ca_state#21, cd_gender#23, cd_marital_status#24, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (51) -+- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.date_dim (47) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan parquet spark_catalog.default.date_dim (45) -(47) Scan parquet spark_catalog.default.date_dim +(45) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(48) CometFilter +(46) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) -(49) CometProject +(47) CometProject Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Arguments: [d_date_sk#9], [d_date_sk#9] -(50) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(51) BroadcastExchange +(49) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt index f9ef660d5..9deb51342 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q35/simplified.txt @@ -1,72 +1,70 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + InputAdapter + Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt index 9f22e37d6..68397f3a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (29) +- Window (27) +- * ColumnarToRow (26) +- CometSort (25) - +- CometColumnarExchange (24) + +- CometExchange (24) +- CometHashAggregate (23) - +- CometColumnarExchange (22) + +- CometExchange (22) +- CometHashAggregate (21) +- CometExpand (20) +- CometProject (19) @@ -131,18 +131,18 @@ Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#14, i_class#15, sp Keys [3]: [i_category#14, i_class#15, spark_grouping_id#16] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(22) CometColumnarExchange +(22) CometExchange Input [5]: [i_category#14, i_class#15, spark_grouping_id#16, sum#17, sum#18] -Arguments: hashpartitioning(i_category#14, i_class#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#14, i_class#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [5]: [i_category#14, i_class#15, spark_grouping_id#16, sum#17, sum#18] Keys [3]: [i_category#14, i_class#15, spark_grouping_id#16] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -(24) CometColumnarExchange +(24) CometExchange Input [7]: [gross_margin#19, i_category#14, i_class#15, lochierarchy#20, _w0#21, _w1#22, _w2#23] -Arguments: hashpartitioning(_w1#22, _w2#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(_w1#22, _w2#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (25) CometSort Input [7]: [gross_margin#19, i_category#14, i_class#15, lochierarchy#20, _w0#21, _w1#22, _w2#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/simplified.txt index d7beb4422..421db8dad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q36/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i ColumnarToRow InputAdapter CometSort [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2] - CometColumnarExchange [_w1,_w2] #1 + CometExchange [_w1,_w2] #1 CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2,spark_grouping_id,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_category,i_class,spark_grouping_id] #2 + CometExchange [i_category,i_class,spark_grouping_id] #2 CometHashAggregate [i_category,i_class,spark_grouping_id,sum,sum,ss_net_profit,ss_ext_sales_price] CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/explain.txt index 42dab0b38..bd5ca5ef6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometBroadcastHashJoin (20) @@ -132,9 +132,9 @@ Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Functions: [] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt index 39edfb519..85e86ca8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q37/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometColumnarExchange [i_item_id,i_item_desc,i_current_price] #1 + CometExchange [i_item_id,i_item_desc,i_current_price] #1 CometHashAggregate [i_item_id,i_item_desc,i_current_price] CometProject [i_item_id,i_item_desc,i_current_price] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,cs_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt index 091883bd6..598dccaf0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == * ColumnarToRow (47) +- CometHashAggregate (46) - +- CometColumnarExchange (45) + +- CometExchange (45) +- CometHashAggregate (44) +- CometProject (43) +- CometBroadcastHashJoin (42) :- CometBroadcastHashJoin (29) : :- CometHashAggregate (16) - : : +- CometColumnarExchange (15) + : : +- CometExchange (15) : : +- CometHashAggregate (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) @@ -24,7 +24,7 @@ : : +- CometScan parquet spark_catalog.default.customer (9) : +- CometBroadcastExchange (28) : +- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -36,7 +36,7 @@ : +- ReusedExchange (22) +- CometBroadcastExchange (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -117,9 +117,9 @@ Input [3]: [c_last_name#9, c_first_name#8, d_date#5] Keys [3]: [c_last_name#9, c_first_name#8, d_date#5] Functions: [] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [c_last_name#9, c_first_name#8, d_date#5] -Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [c_last_name#9, c_first_name#8, d_date#5] @@ -167,9 +167,9 @@ Input [3]: [c_last_name#17, c_first_name#16, d_date#14] Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(26) CometColumnarExchange +(26) CometExchange Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate Input [3]: [c_last_name#17, c_first_name#16, d_date#14] @@ -226,9 +226,9 @@ Input [3]: [c_last_name#25, c_first_name#24, d_date#22] Keys [3]: [c_last_name#25, c_first_name#24, d_date#22] Functions: [] -(39) CometColumnarExchange +(39) CometExchange Input [3]: [c_last_name#25, c_first_name#24, d_date#22] -Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (40) CometHashAggregate Input [3]: [c_last_name#25, c_first_name#24, d_date#22] @@ -252,9 +252,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(45) CometColumnarExchange +(45) CometExchange Input [1]: [count#26] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (46) CometHashAggregate Input [1]: [count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt index 77970e8d3..8272260f5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q38/simplified.txt @@ -2,13 +2,13 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [count(1),count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #2 + CometExchange [c_last_name,c_first_name,d_date] #2 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -33,7 +33,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #7 + CometExchange [c_last_name,c_first_name,d_date] #7 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -46,7 +46,7 @@ WholeStageCodegen (1) ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 CometBroadcastExchange [c_last_name,c_first_name,d_date] #8 CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #9 + CometExchange [c_last_name,c_first_name,d_date] #9 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt index 60c02c542..58e0e6649 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/explain.txt @@ -6,7 +6,7 @@ :- CometProject (23) : +- CometFilter (22) : +- CometHashAggregate (21) - : +- CometColumnarExchange (20) + : +- CometExchange (20) : +- CometHashAggregate (19) : +- CometProject (18) : +- CometBroadcastHashJoin (17) @@ -30,7 +30,7 @@ +- CometProject (42) +- CometFilter (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -141,9 +141,9 @@ Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_n Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -(20) CometColumnarExchange +(20) CometExchange Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] @@ -227,9 +227,9 @@ Input [5]: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehous Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#22 as double)), partial_avg(inv_quantity_on_hand#22)] -(39) CometColumnarExchange +(39) CometExchange Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) CometHashAggregate Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/simplified.txt index a4d1753f9..b789f3a8d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39a/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] @@ -39,7 +39,7 @@ WholeStageCodegen (1) CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/explain.txt index d95f2c4cc..c246e93a5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/explain.txt @@ -6,7 +6,7 @@ :- CometProject (23) : +- CometFilter (22) : +- CometHashAggregate (21) - : +- CometColumnarExchange (20) + : +- CometExchange (20) : +- CometHashAggregate (19) : +- CometProject (18) : +- CometBroadcastHashJoin (17) @@ -30,7 +30,7 @@ +- CometProject (42) +- CometFilter (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -141,9 +141,9 @@ Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_n Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -(20) CometColumnarExchange +(20) CometExchange Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] @@ -227,9 +227,9 @@ Input [5]: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehous Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#22 as double)), partial_avg(inv_quantity_on_hand#22)] -(39) CometColumnarExchange +(39) CometExchange Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) CometHashAggregate Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/simplified.txt index a4d1753f9..b789f3a8d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q39b/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] @@ -39,7 +39,7 @@ WholeStageCodegen (1) CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/explain.txt index 960cb130c..b6387767d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/explain.txt @@ -12,7 +12,7 @@ : : : :- CometBroadcastHashJoin (33) : : : : :- CometFilter (16) : : : : : +- CometHashAggregate (15) - : : : : : +- CometColumnarExchange (14) + : : : : : +- CometExchange (14) : : : : : +- CometHashAggregate (13) : : : : : +- CometProject (12) : : : : : +- CometBroadcastHashJoin (11) @@ -28,7 +28,7 @@ : : : : : +- CometScan parquet spark_catalog.default.date_dim (8) : : : : +- CometBroadcastExchange (32) : : : : +- CometHashAggregate (31) - : : : : +- CometColumnarExchange (30) + : : : : +- CometExchange (30) : : : : +- CometHashAggregate (29) : : : : +- CometProject (28) : : : : +- CometBroadcastHashJoin (27) @@ -45,7 +45,7 @@ : : : +- CometBroadcastExchange (48) : : : +- CometFilter (47) : : : +- CometHashAggregate (46) - : : : +- CometColumnarExchange (45) + : : : +- CometExchange (45) : : : +- CometHashAggregate (44) : : : +- CometProject (43) : : : +- CometBroadcastHashJoin (42) @@ -59,7 +59,7 @@ : : : +- ReusedExchange (41) : : +- CometBroadcastExchange (64) : : +- CometHashAggregate (63) - : : +- CometColumnarExchange (62) + : : +- CometExchange (62) : : +- CometHashAggregate (61) : : +- CometProject (60) : : +- CometBroadcastHashJoin (59) @@ -74,7 +74,7 @@ : +- CometBroadcastExchange (81) : +- CometFilter (80) : +- CometHashAggregate (79) - : +- CometColumnarExchange (78) + : +- CometExchange (78) : +- CometHashAggregate (77) : +- CometProject (76) : +- CometBroadcastHashJoin (75) @@ -88,7 +88,7 @@ : +- ReusedExchange (74) +- CometBroadcastExchange (97) +- CometHashAggregate (96) - +- CometColumnarExchange (95) + +- CometExchange (95) +- CometHashAggregate (94) +- CometProject (93) +- CometBroadcastHashJoin (92) @@ -167,9 +167,9 @@ Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_fl Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17] Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -(14) CometColumnarExchange +(14) CometExchange Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, sum#18, isEmpty#19] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, sum#18, isEmpty#19] @@ -245,9 +245,9 @@ Input [12]: [c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust Keys [8]: [c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26, c_birth_country#27, c_login#28, c_email_address#29, d_year#38] Functions [1]: [partial_sum(((((ss_ext_list_price#34 - ss_ext_wholesale_cost#33) - ss_ext_discount_amt#31) + ss_ext_sales_price#32) / 2))] -(30) CometColumnarExchange +(30) CometExchange Input [10]: [c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26, c_birth_country#27, c_login#28, c_email_address#29, d_year#38, sum#39, isEmpty#40] -Arguments: hashpartitioning(c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26, c_birth_country#27, c_login#28, c_email_address#29, d_year#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26, c_birth_country#27, c_login#28, c_email_address#29, d_year#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate Input [10]: [c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26, c_birth_country#27, c_login#28, c_email_address#29, d_year#38, sum#39, isEmpty#40] @@ -316,9 +316,9 @@ Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#65] Functions [1]: [partial_sum(((((cs_ext_list_price#61 - cs_ext_wholesale_cost#60) - cs_ext_discount_amt#58) + cs_ext_sales_price#59) / 2))] -(45) CometColumnarExchange +(45) CometExchange Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#65, sum#66, isEmpty#67] -Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#65, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometHashAggregate Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#65, sum#66, isEmpty#67] @@ -395,9 +395,9 @@ Input [12]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust Keys [8]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#86] Functions [1]: [partial_sum(((((cs_ext_list_price#82 - cs_ext_wholesale_cost#81) - cs_ext_discount_amt#79) + cs_ext_sales_price#80) / 2))] -(62) CometColumnarExchange +(62) CometExchange Input [10]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#86, sum#87, isEmpty#88] -Arguments: hashpartitioning(c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#86, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (63) CometHashAggregate Input [10]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#86, sum#87, isEmpty#88] @@ -470,9 +470,9 @@ Input [12]: [c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust Keys [8]: [c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust_flag#95, c_birth_country#96, c_login#97, c_email_address#98, d_year#107] Functions [1]: [partial_sum(((((ws_ext_list_price#103 - ws_ext_wholesale_cost#102) - ws_ext_discount_amt#100) + ws_ext_sales_price#101) / 2))] -(78) CometColumnarExchange +(78) CometExchange Input [10]: [c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust_flag#95, c_birth_country#96, c_login#97, c_email_address#98, d_year#107, sum#108, isEmpty#109] -Arguments: hashpartitioning(c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust_flag#95, c_birth_country#96, c_login#97, c_email_address#98, d_year#107, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust_flag#95, c_birth_country#96, c_login#97, c_email_address#98, d_year#107, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (79) CometHashAggregate Input [10]: [c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust_flag#95, c_birth_country#96, c_login#97, c_email_address#98, d_year#107, sum#108, isEmpty#109] @@ -549,9 +549,9 @@ Input [12]: [c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_c Keys [8]: [c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_cust_flag#116, c_birth_country#117, c_login#118, c_email_address#119, d_year#128] Functions [1]: [partial_sum(((((ws_ext_list_price#124 - ws_ext_wholesale_cost#123) - ws_ext_discount_amt#121) + ws_ext_sales_price#122) / 2))] -(95) CometColumnarExchange +(95) CometExchange Input [10]: [c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_cust_flag#116, c_birth_country#117, c_login#118, c_email_address#119, d_year#128, sum#129, isEmpty#130] -Arguments: hashpartitioning(c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_cust_flag#116, c_birth_country#117, c_login#118, c_email_address#119, d_year#128, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_cust_flag#116, c_birth_country#117, c_login#118, c_email_address#119, d_year#128, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (96) CometHashAggregate Input [10]: [c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_cust_flag#116, c_birth_country#117, c_login#118, c_email_address#119, d_year#128, sum#129, isEmpty#130] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt index 7e6b12575..d6651fadc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q4/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -60,7 +60,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] @@ -75,7 +75,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] @@ -91,7 +91,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #16 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #17 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #17 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -106,7 +106,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #19 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #20 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #20 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt index 00967faa4..224e100ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (32) +- CometTakeOrderedAndProject (31) +- CometHashAggregate (30) - +- CometColumnarExchange (29) + +- CometExchange (29) +- CometHashAggregate (28) +- CometProject (27) +- CometBroadcastHashJoin (26) @@ -13,11 +13,11 @@ : : :- CometProject (11) : : : +- CometSortMergeJoin (10) : : : :- CometSort (4) - : : : : +- CometColumnarExchange (3) + : : : : +- CometExchange (3) : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometSort (9) - : : : +- CometColumnarExchange (8) + : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) : : : +- CometScan parquet spark_catalog.default.catalog_returns (5) @@ -45,9 +45,9 @@ ReadSchema: struct= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -(29) CometColumnarExchange +(29) CometExchange Input [6]: [w_state#12, i_item_id#14, sum#18, isEmpty#19, sum#20, isEmpty#21] -Arguments: hashpartitioning(w_state#12, i_item_id#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(w_state#12, i_item_id#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (30) CometHashAggregate Input [6]: [w_state#12, i_item_id#14, sum#18, isEmpty#19, sum#20, isEmpty#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt index 6e47fa59f..eac613819 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q40/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] CometHashAggregate [w_state,i_item_id,sales_before,sales_after,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] - CometColumnarExchange [w_state,i_item_id] #1 + CometExchange [w_state,i_item_id] #1 CometHashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty,d_date,cs_sales_price,cr_refunded_cash] CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] @@ -14,7 +14,7 @@ WholeStageCodegen (1) CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometColumnarExchange [cs_order_number,cs_item_sk] #2 + CometExchange [cs_order_number,cs_item_sk] #2 CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 @@ -25,7 +25,7 @@ WholeStageCodegen (1) CometFilter [d_date_sk,d_date] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] - CometColumnarExchange [cr_order_number,cr_item_sk] #4 + CometExchange [cr_order_number,cr_item_sk] #4 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt index 5c2b28b9d..c4059e5f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -13,7 +13,7 @@ +- CometProject (11) +- CometFilter (10) +- CometHashAggregate (9) - +- CometColumnarExchange (8) + +- CometExchange (8) +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) @@ -55,9 +55,9 @@ Input [1]: [i_manufact#5] Keys [1]: [i_manufact#5] Functions [1]: [partial_count(1)] -(8) CometColumnarExchange +(8) CometExchange Input [2]: [i_manufact#5, count#9] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (9) CometHashAggregate Input [2]: [i_manufact#5, count#9] @@ -90,9 +90,9 @@ Input [1]: [i_product_name#3] Keys [1]: [i_product_name#3] Functions: [] -(16) CometColumnarExchange +(16) CometExchange Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (17) CometHashAggregate Input [1]: [i_product_name#3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt index 732e8679b..007103499 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q41/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_product_name] CometHashAggregate [i_product_name] - CometColumnarExchange [i_product_name] #1 + CometExchange [i_product_name] #1 CometHashAggregate [i_product_name] CometProject [i_product_name] CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] @@ -14,7 +14,7 @@ WholeStageCodegen (1) CometProject [i_manufact] CometFilter [item_cnt,i_manufact] CometHashAggregate [item_cnt,i_manufact,count,count(1)] - CometColumnarExchange [i_manufact] #3 + CometExchange [i_manufact] #3 CometHashAggregate [i_manufact,count] CometProject [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt index ae0d6f333..c2c2c8bf3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -93,9 +93,9 @@ Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] Keys [3]: [d_year#2, i_category_id#8, i_category#9] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#11] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt index bdc71f6da..40528f52d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q42/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [d_year,i_category_id,i_category,sum(ss_ext_sales_price)] CometHashAggregate [d_year,i_category_id,i_category,sum(ss_ext_sales_price),sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [d_year,i_category_id,i_category] #1 + CometExchange [d_year,i_category_id,i_category] #1 CometHashAggregate [d_year,i_category_id,i_category,sum,ss_ext_sales_price] CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt index 41a564299..d6cb50a4f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -93,9 +93,9 @@ Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] Keys [2]: [s_store_name#9, s_store_id#8] Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -(16) CometColumnarExchange +(16) CometExchange Input [9]: [s_store_name#9, s_store_id#8, sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [9]: [s_store_name#9, s_store_id#8, sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt index 6e1b8d6bf..447131b77 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q43/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] CometHashAggregate [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] - CometColumnarExchange [s_store_name,s_store_id] #1 + CometExchange [s_store_name,s_store_id] #1 CometHashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum,d_day_name,ss_sales_price] CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt index 9ee93656a..57e6b0e89 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/explain.txt @@ -1,50 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * Project (45) - +- * BroadcastHashJoin Inner BuildRight (44) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (36) - : : +- * SortMergeJoin Inner (35) - : : :- * Sort (19) - : : : +- * Project (18) - : : : +- * Filter (17) - : : : +- Window (16) - : : : +- WindowGroupLimit (15) - : : : +- * ColumnarToRow (14) - : : : +- CometSort (13) - : : : +- CometColumnarExchange (12) - : : : +- RowToColumnar (11) - : : : +- WindowGroupLimit (10) - : : : +- * ColumnarToRow (9) - : : : +- CometSort (8) - : : : +- CometFilter (7) - : : : +- CometHashAggregate (6) - : : : +- CometColumnarExchange (5) - : : : +- CometHashAggregate (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- * Sort (34) - : : +- * Project (33) - : : +- * Filter (32) - : : +- Window (31) - : : +- WindowGroupLimit (30) - : : +- * ColumnarToRow (29) - : : +- CometSort (28) - : : +- CometColumnarExchange (27) - : : +- RowToColumnar (26) - : : +- WindowGroupLimit (25) - : : +- * ColumnarToRow (24) - : : +- CometSort (23) - : : +- CometFilter (22) - : : +- CometHashAggregate (21) - : : +- ReusedExchange (20) - : +- BroadcastExchange (40) - : +- * ColumnarToRow (39) - : +- CometFilter (38) - : +- CometScan parquet spark_catalog.default.item (37) - +- ReusedExchange (43) +TakeOrderedAndProject (42) ++- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (38) + : +- * BroadcastHashJoin Inner BuildRight (37) + : :- * Project (32) + : : +- * SortMergeJoin Inner (31) + : : :- * Sort (17) + : : : +- * Project (16) + : : : +- * Filter (15) + : : : +- Window (14) + : : : +- WindowGroupLimit (13) + : : : +- * Sort (12) + : : : +- Exchange (11) + : : : +- WindowGroupLimit (10) + : : : +- * ColumnarToRow (9) + : : : +- CometSort (8) + : : : +- CometFilter (7) + : : : +- CometHashAggregate (6) + : : : +- CometExchange (5) + : : : +- CometHashAggregate (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- * Sort (30) + : : +- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- WindowGroupLimit (26) + : : +- * Sort (25) + : : +- Exchange (24) + : : +- WindowGroupLimit (23) + : : +- * ColumnarToRow (22) + : : +- CometSort (21) + : : +- CometFilter (20) + : : +- CometHashAggregate (19) + : : +- ReusedExchange (18) + : +- BroadcastExchange (36) + : +- * ColumnarToRow (35) + : +- CometFilter (34) + : +- CometScan parquet spark_catalog.default.item (33) + +- ReusedExchange (39) (1) Scan parquet spark_catalog.default.store_sales @@ -67,9 +63,9 @@ Input [2]: [ss_item_sk#1, ss_net_profit#3] Keys [1]: [ss_item_sk#1] Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] -(5) CometColumnarExchange +(5) CometExchange Input [3]: [ss_item_sk#1, sum#5, count#6] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (6) CometHashAggregate Input [3]: [ss_item_sk#1, sum#5, count#6] @@ -91,196 +87,184 @@ Input [2]: [item_sk#7, rank_col#8] Input [2]: [item_sk#7, rank_col#8] Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Partial -(11) RowToColumnar +(11) Exchange Input [2]: [item_sk#7, rank_col#8] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(12) CometColumnarExchange +(12) Sort [codegen id : 2] Input [2]: [item_sk#7, rank_col#8] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: [rank_col#8 ASC NULLS FIRST], false, 0 -(13) CometSort -Input [2]: [item_sk#7, rank_col#8] -Arguments: [item_sk#7, rank_col#8], [rank_col#8 ASC NULLS FIRST] - -(14) ColumnarToRow [codegen id : 2] -Input [2]: [item_sk#7, rank_col#8] - -(15) WindowGroupLimit +(13) WindowGroupLimit Input [2]: [item_sk#7, rank_col#8] Arguments: [rank_col#8 ASC NULLS FIRST], rank(rank_col#8), 10, Final -(16) Window +(14) Window Input [2]: [item_sk#7, rank_col#8] Arguments: [rank(rank_col#8) windowspecdefinition(rank_col#8 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#11], [rank_col#8 ASC NULLS FIRST] -(17) Filter [codegen id : 3] +(15) Filter [codegen id : 3] Input [3]: [item_sk#7, rank_col#8, rnk#11] Condition : ((rnk#11 < 11) AND isnotnull(item_sk#7)) -(18) Project [codegen id : 3] +(16) Project [codegen id : 3] Output [2]: [item_sk#7, rnk#11] Input [3]: [item_sk#7, rank_col#8, rnk#11] -(19) Sort [codegen id : 3] +(17) Sort [codegen id : 3] Input [2]: [item_sk#7, rnk#11] Arguments: [rnk#11 ASC NULLS FIRST], false, 0 -(20) ReusedExchange [Reuses operator id: 5] +(18) ReusedExchange [Reuses operator id: 5] Output [3]: [ss_item_sk#12, sum#13, count#14] -(21) CometHashAggregate +(19) CometHashAggregate Input [3]: [ss_item_sk#12, sum#13, count#14] Keys [1]: [ss_item_sk#12] Functions [1]: [avg(UnscaledValue(ss_net_profit#15))] -(22) CometFilter +(20) CometFilter Input [2]: [item_sk#16, rank_col#17] Condition : (isnotnull(rank_col#17) AND (cast(rank_col#17 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#9, [id=#10]))) -(23) CometSort +(21) CometSort Input [2]: [item_sk#16, rank_col#17] Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] -(24) ColumnarToRow [codegen id : 4] +(22) ColumnarToRow [codegen id : 4] Input [2]: [item_sk#16, rank_col#17] -(25) WindowGroupLimit +(23) WindowGroupLimit Input [2]: [item_sk#16, rank_col#17] Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Partial -(26) RowToColumnar +(24) Exchange Input [2]: [item_sk#16, rank_col#17] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] -(27) CometColumnarExchange -Input [2]: [item_sk#16, rank_col#17] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(28) CometSort -Input [2]: [item_sk#16, rank_col#17] -Arguments: [item_sk#16, rank_col#17], [rank_col#17 DESC NULLS LAST] - -(29) ColumnarToRow [codegen id : 5] +(25) Sort [codegen id : 5] Input [2]: [item_sk#16, rank_col#17] +Arguments: [rank_col#17 DESC NULLS LAST], false, 0 -(30) WindowGroupLimit +(26) WindowGroupLimit Input [2]: [item_sk#16, rank_col#17] Arguments: [rank_col#17 DESC NULLS LAST], rank(rank_col#17), 10, Final -(31) Window +(27) Window Input [2]: [item_sk#16, rank_col#17] Arguments: [rank(rank_col#17) windowspecdefinition(rank_col#17 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#18], [rank_col#17 DESC NULLS LAST] -(32) Filter [codegen id : 6] +(28) Filter [codegen id : 6] Input [3]: [item_sk#16, rank_col#17, rnk#18] Condition : ((rnk#18 < 11) AND isnotnull(item_sk#16)) -(33) Project [codegen id : 6] +(29) Project [codegen id : 6] Output [2]: [item_sk#16, rnk#18] Input [3]: [item_sk#16, rank_col#17, rnk#18] -(34) Sort [codegen id : 6] +(30) Sort [codegen id : 6] Input [2]: [item_sk#16, rnk#18] Arguments: [rnk#18 ASC NULLS FIRST], false, 0 -(35) SortMergeJoin [codegen id : 9] +(31) SortMergeJoin [codegen id : 9] Left keys [1]: [rnk#11] Right keys [1]: [rnk#18] Join type: Inner Join condition: None -(36) Project [codegen id : 9] +(32) Project [codegen id : 9] Output [3]: [item_sk#7, rnk#11, item_sk#16] Input [4]: [item_sk#7, rnk#11, item_sk#16, rnk#18] -(37) Scan parquet spark_catalog.default.item +(33) Scan parquet spark_catalog.default.item Output [2]: [i_item_sk#19, i_product_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(38) CometFilter +(34) CometFilter Input [2]: [i_item_sk#19, i_product_name#20] Condition : isnotnull(i_item_sk#19) -(39) ColumnarToRow [codegen id : 7] +(35) ColumnarToRow [codegen id : 7] Input [2]: [i_item_sk#19, i_product_name#20] -(40) BroadcastExchange +(36) BroadcastExchange Input [2]: [i_item_sk#19, i_product_name#20] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] -(41) BroadcastHashJoin [codegen id : 9] +(37) BroadcastHashJoin [codegen id : 9] Left keys [1]: [item_sk#7] Right keys [1]: [i_item_sk#19] Join type: Inner Join condition: None -(42) Project [codegen id : 9] +(38) Project [codegen id : 9] Output [3]: [rnk#11, item_sk#16, i_product_name#20] Input [5]: [item_sk#7, rnk#11, item_sk#16, i_item_sk#19, i_product_name#20] -(43) ReusedExchange [Reuses operator id: 40] +(39) ReusedExchange [Reuses operator id: 36] Output [2]: [i_item_sk#21, i_product_name#22] -(44) BroadcastHashJoin [codegen id : 9] +(40) BroadcastHashJoin [codegen id : 9] Left keys [1]: [item_sk#16] Right keys [1]: [i_item_sk#21] Join type: Inner Join condition: None -(45) Project [codegen id : 9] +(41) Project [codegen id : 9] Output [3]: [rnk#11, i_product_name#20 AS best_performing#23, i_product_name#22 AS worst_performing#24] Input [5]: [rnk#11, item_sk#16, i_product_name#20, i_item_sk#21, i_product_name#22] -(46) TakeOrderedAndProject +(42) TakeOrderedAndProject Input [3]: [rnk#11, best_performing#23, worst_performing#24] Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#23, worst_performing#24] ===== Subqueries ===== Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] -* ColumnarToRow (53) -+- CometHashAggregate (52) - +- CometColumnarExchange (51) - +- CometHashAggregate (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.store_sales (47) +* ColumnarToRow (49) ++- CometHashAggregate (48) + +- CometExchange (47) + +- CometHashAggregate (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan parquet spark_catalog.default.store_sales (43) -(47) Scan parquet spark_catalog.default.store_sales +(43) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] ReadSchema: struct -(48) CometFilter +(44) CometFilter Input [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] Condition : ((isnotnull(ss_store_sk#26) AND (ss_store_sk#26 = 4)) AND isnull(ss_addr_sk#25)) -(49) CometProject +(45) CometProject Input [4]: [ss_addr_sk#25, ss_store_sk#26, ss_net_profit#27, ss_sold_date_sk#28] Arguments: [ss_store_sk#26, ss_net_profit#27], [ss_store_sk#26, ss_net_profit#27] -(50) CometHashAggregate +(46) CometHashAggregate Input [2]: [ss_store_sk#26, ss_net_profit#27] Keys [1]: [ss_store_sk#26] Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#27))] -(51) CometColumnarExchange +(47) CometExchange Input [3]: [ss_store_sk#26, sum#29, count#30] -Arguments: hashpartitioning(ss_store_sk#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ss_store_sk#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(52) CometHashAggregate +(48) CometHashAggregate Input [3]: [ss_store_sk#26, sum#29, count#30] Keys [1]: [ss_store_sk#26] Functions [1]: [avg(UnscaledValue(ss_net_profit#27))] -(53) ColumnarToRow [codegen id : 1] +(49) ColumnarToRow [codegen id : 1] Input [1]: [rank_col#31] -Subquery:2 Hosting operator id = 22 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] +Subquery:2 Hosting operator id = 20 Hosting Expression = ReusedSubquery Subquery scalar-subquery#9, [id=#10] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt index 886ee5e0d..58687dc84 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q44/simplified.txt @@ -15,33 +15,31 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] Window [rank_col] WindowGroupLimit [rank_col] WholeStageCodegen (2) - ColumnarToRow + Sort [rank_col] InputAdapter - CometSort [item_sk,rank_col] - CometColumnarExchange #1 - RowToColumnar - WindowGroupLimit [rank_col] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item_sk,rank_col] - CometFilter [item_sk,rank_col] - Subquery #1 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [rank_col,ss_store_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - CometColumnarExchange [ss_store_sk] #3 - CometHashAggregate [ss_store_sk,sum,count,ss_net_profit] - CometProject [ss_store_sk,ss_net_profit] - CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - CometColumnarExchange [ss_item_sk] #2 - CometHashAggregate [ss_item_sk,sum,count,ss_net_profit] - CometProject [ss_item_sk,ss_net_profit] - CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + Exchange #1 + WindowGroupLimit [rank_col] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometFilter [item_sk,rank_col] + Subquery #1 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometHashAggregate [rank_col,ss_store_sk,sum,count,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_store_sk] #3 + CometHashAggregate [ss_store_sk,sum,count,ss_net_profit] + CometProject [ss_store_sk,ss_net_profit] + CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] + CometExchange [ss_item_sk] #2 + CometHashAggregate [ss_item_sk,sum,count,ss_net_profit] + CometProject [ss_item_sk,ss_net_profit] + CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] InputAdapter WholeStageCodegen (6) Sort [rnk] @@ -51,20 +49,18 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] Window [rank_col] WindowGroupLimit [rank_col] WholeStageCodegen (5) - ColumnarToRow + Sort [rank_col] InputAdapter - CometSort [item_sk,rank_col] - CometColumnarExchange #4 - RowToColumnar - WindowGroupLimit [rank_col] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item_sk,rank_col] - CometFilter [item_sk,rank_col] - ReusedSubquery [rank_col] #1 - CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - ReusedExchange [ss_item_sk,sum,count] #2 + Exchange #4 + WindowGroupLimit [rank_col] + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [item_sk,rank_col] + CometFilter [item_sk,rank_col] + ReusedSubquery [rank_col] #1 + CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] + ReusedExchange [ss_item_sk,sum,count] #2 InputAdapter BroadcastExchange #5 WholeStageCodegen (7) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt index 044247540..f128499e3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/explain.txt @@ -1,42 +1,40 @@ == Physical Plan == -TakeOrderedAndProject (38) -+- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometColumnarExchange (35) - +- RowToColumnar (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) - :- * ColumnarToRow (24) - : +- CometProject (23) - : +- CometBroadcastHashJoin (22) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.customer (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.customer_address (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.date_dim (13) - : +- CometBroadcastExchange (21) - : +- CometFilter (20) - : +- CometScan parquet spark_catalog.default.item (19) - +- BroadcastExchange (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.item (25) +TakeOrderedAndProject (36) ++- * HashAggregate (35) + +- Exchange (34) + +- * HashAggregate (33) + +- * Project (32) + +- * Filter (31) + +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) + :- * ColumnarToRow (24) + : +- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.customer (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (16) + : : +- CometProject (15) + : : +- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.date_dim (13) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometScan parquet spark_catalog.default.item (19) + +- BroadcastExchange (29) + +- * ColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.item (25) (1) Scan parquet spark_catalog.default.web_sales @@ -197,56 +195,50 @@ Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] Aggregate Attributes [1]: [sum#19] Results [3]: [ca_zip#11, ca_city#10, sum#20] -(34) RowToColumnar +(34) Exchange Input [3]: [ca_zip#11, ca_city#10, sum#20] +Arguments: hashpartitioning(ca_zip#11, ca_city#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(35) CometColumnarExchange -Input [3]: [ca_zip#11, ca_city#10, sum#20] -Arguments: hashpartitioning(ca_zip#11, ca_city#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(36) ColumnarToRow [codegen id : 3] -Input [3]: [ca_zip#11, ca_city#10, sum#20] - -(37) HashAggregate [codegen id : 3] +(35) HashAggregate [codegen id : 3] Input [3]: [ca_zip#11, ca_city#10, sum#20] Keys [2]: [ca_zip#11, ca_city#10] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#21] Results [3]: [ca_zip#11, ca_city#10, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#21,17,2) AS sum(ws_sales_price)#22] -(38) TakeOrderedAndProject +(36) TakeOrderedAndProject Input [3]: [ca_zip#11, ca_city#10, sum(ws_sales_price)#22] Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#10 ASC NULLS FIRST], [ca_zip#11, ca_city#10, sum(ws_sales_price)#22] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (43) -+- * ColumnarToRow (42) - +- CometProject (41) - +- CometFilter (40) - +- CometScan parquet spark_catalog.default.date_dim (39) +BroadcastExchange (41) ++- * ColumnarToRow (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan parquet spark_catalog.default.date_dim (37) -(39) Scan parquet spark_catalog.default.date_dim +(37) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(40) CometFilter +(38) CometFilter Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) -(41) CometProject +(39) CometProject Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] Arguments: [d_date_sk#12], [d_date_sk#12] -(42) ColumnarToRow [codegen id : 1] +(40) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#12] -(43) BroadcastExchange +(41) BroadcastExchange Input [1]: [d_date_sk#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/simplified.txt index 8550b91c6..f60fdb18a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q45/simplified.txt @@ -1,53 +1,51 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] WholeStageCodegen (3) HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_zip,ca_city] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - ColumnarToRow - InputAdapter - CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] - CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 - CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #4 - CometFilter [ca_address_sk,ca_city,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #6 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_id] - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + Exchange [ca_zip,ca_city] #1 + WholeStageCodegen (2) + HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] + Project [ws_sales_price,ca_city,ca_zip] + Filter [ca_zip,exists] + BroadcastHashJoin [i_item_id,i_item_id] + ColumnarToRow + InputAdapter + CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #4 + CometFilter [ca_address_sk,ca_city,ca_zip] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/explain.txt index b81931528..75334c745 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/explain.txt @@ -6,7 +6,7 @@ :- CometProject (33) : +- CometBroadcastHashJoin (32) : :- CometHashAggregate (28) - : : +- CometColumnarExchange (27) + : : +- CometExchange (27) : : +- CometHashAggregate (26) : : +- CometProject (25) : : +- CometBroadcastHashJoin (24) @@ -164,9 +164,9 @@ Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -(27) CometColumnarExchange +(27) CometExchange Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/simplified.txt index 1d28a8ede..e1b53bf4d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q46/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometProject [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometHashAggregate [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,ss_addr_sk,ca_city,sum,sum,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] - CometColumnarExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,ss_coupon_amt,ss_net_profit] CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/explain.txt index 3fe3003ce..8221d32a3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/explain.txt @@ -11,9 +11,9 @@ TakeOrderedAndProject (45) : : +- Window (24) : : +- * ColumnarToRow (23) : : +- CometSort (22) - : : +- CometColumnarExchange (21) + : : +- CometExchange (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -135,18 +135,18 @@ Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#10, d_moy#11, s_st Keys [6]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -(19) CometColumnarExchange +(19) CometExchange Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (20) CometHashAggregate Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#15] Keys [6]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -(21) CometColumnarExchange +(21) CometExchange Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (22) CometSort Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#16, _w0#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/simplified.txt index 91c2cd65d..2c8192f43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q47/simplified.txt @@ -16,9 +16,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra ColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum,ss_sales_price] CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/explain.txt index 3393afedf..40181264b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (28) +- CometHashAggregate (27) - +- CometColumnarExchange (26) + +- CometExchange (26) +- CometHashAggregate (25) +- CometProject (24) +- CometBroadcastHashJoin (23) @@ -150,9 +150,9 @@ Input [1]: [ss_quantity#4] Keys: [] Functions [1]: [partial_sum(ss_quantity#4)] -(26) CometColumnarExchange +(26) CometExchange Input [1]: [sum#18] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [1]: [sum#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt index 8bb0cede6..a46d8d603 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q48/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [sum(ss_quantity),sum,sum(ss_quantity)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,ss_quantity] CometProject [ss_quantity] CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt index 4b0d5aed0..f7e750d47 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/explain.txt @@ -1,83 +1,81 @@ == Physical Plan == -* ColumnarToRow (79) -+- CometTakeOrderedAndProject (78) - +- CometHashAggregate (77) - +- CometColumnarExchange (76) - +- RowToColumnar (75) - +- * HashAggregate (74) - +- Union (73) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * ColumnarToRow (21) - : +- CometSort (20) - : +- CometColumnarExchange (19) - : +- CometHashAggregate (18) - : +- CometColumnarExchange (17) - : +- CometHashAggregate (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometScan parquet spark_catalog.default.web_returns (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.date_dim (10) - :- * Project (49) - : +- * Filter (48) - : +- Window (47) - : +- * Sort (46) - : +- Window (45) - : +- * ColumnarToRow (44) - : +- CometSort (43) - : +- CometColumnarExchange (42) - : +- CometHashAggregate (41) - : +- CometColumnarExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometBroadcastExchange (30) - : : : +- CometProject (29) - : : : +- CometFilter (28) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) - : : +- CometProject (33) - : : +- CometFilter (32) - : : +- CometScan parquet spark_catalog.default.catalog_returns (31) - : +- ReusedExchange (36) - +- * Project (72) - +- * Filter (71) - +- Window (70) - +- * Sort (69) - +- Window (68) - +- * ColumnarToRow (67) - +- CometSort (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometColumnarExchange (63) - +- CometHashAggregate (62) - +- CometProject (61) - +- CometBroadcastHashJoin (60) - :- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometBroadcastExchange (53) - : : +- CometProject (52) - : : +- CometFilter (51) - : : +- CometScan parquet spark_catalog.default.store_sales (50) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometScan parquet spark_catalog.default.store_returns (54) - +- ReusedExchange (59) +TakeOrderedAndProject (77) ++- * HashAggregate (76) + +- Exchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * ColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * ColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * ColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) (1) Scan parquet spark_catalog.default.web_sales @@ -157,18 +155,18 @@ Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, w Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -(19) CometColumnarExchange +(19) CometExchange Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort Input [3]: [item#22, return_ratio#23, currency_ratio#24] @@ -258,18 +256,18 @@ Input [5]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37 Keys [1]: [cs_item_sk#28] Functions [4]: [partial_sum(coalesce(cr_return_quantity#37, 0)), partial_sum(coalesce(cs_quantity#30, 0)), partial_sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] -(40) CometColumnarExchange +(40) CometExchange Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] Keys [1]: [cs_item_sk#28] Functions [4]: [sum(coalesce(cr_return_quantity#37, 0)), sum(coalesce(cs_quantity#30, 0)), sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] -(42) CometColumnarExchange +(42) CometExchange Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort Input [3]: [item#47, return_ratio#48, currency_ratio#49] @@ -359,18 +357,18 @@ Input [5]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62 Keys [1]: [ss_item_sk#53] Functions [4]: [partial_sum(coalesce(sr_return_quantity#62, 0)), partial_sum(coalesce(ss_quantity#55, 0)), partial_sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] -(63) CometColumnarExchange +(63) CometExchange Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (64) CometHashAggregate Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] Keys [1]: [ss_item_sk#53] Functions [4]: [sum(coalesce(sr_return_quantity#62, 0)), sum(coalesce(ss_quantity#55, 0)), sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] -(65) CometColumnarExchange +(65) CometExchange Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (66) CometSort Input [3]: [item#72, return_ratio#73, currency_ratio#74] @@ -408,54 +406,50 @@ Functions: [] Aggregate Attributes: [] Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -(75) RowToColumnar +(75) Exchange Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(76) CometColumnarExchange -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(77) CometHashAggregate +(76) HashAggregate [codegen id : 11] Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -(78) CometTakeOrderedAndProject -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#27 ASC NULLS FIRST,return_rank#25 ASC NULLS FIRST,currency_rank#26 ASC NULLS FIRST], output=[channel#27,item#22,return_ratio#23,return_rank#25,currency_rank#26]), [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26], 100, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] - -(79) ColumnarToRow [codegen id : 11] +(77) TakeOrderedAndProject Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Arguments: 100, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (84) -+- * ColumnarToRow (83) - +- CometProject (82) - +- CometFilter (81) - +- CometScan parquet spark_catalog.default.date_dim (80) +BroadcastExchange (82) ++- * ColumnarToRow (81) + +- CometProject (80) + +- CometFilter (79) + +- CometScan parquet spark_catalog.default.date_dim (78) -(80) Scan parquet spark_catalog.default.date_dim +(78) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(81) CometFilter +(79) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) -(82) CometProject +(80) CometProject Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Arguments: [d_date_sk#13], [d_date_sk#13] -(83) ColumnarToRow [codegen id : 1] +(81) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#13] -(84) BroadcastExchange +(82) BroadcastExchange Input [1]: [d_date_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt index 0d02a9c39..d98b2b0a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q49/simplified.txt @@ -1,111 +1,109 @@ -WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] - CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] - CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (3) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (2) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #2 - CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [ws_item_sk] #3 - CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (6) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (5) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #7 - CometHashAggregate [item,return_ratio,currency_ratio,cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [cs_item_sk] #8 - CometHashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - WholeStageCodegen (9) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (8) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #10 - CometHashAggregate [item,return_ratio,currency_ratio,ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [ss_item_sk] #11 - CometHashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 +TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] + WholeStageCodegen (11) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #7 + CometHashAggregate [item,return_ratio,currency_ratio,cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #8 + CometHashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #10 + CometHashAggregate [item,return_ratio,currency_ratio,ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #11 + CometHashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt index ed1f8feda..82b5c6690 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/explain.txt @@ -1,76 +1,74 @@ == Physical Plan == -TakeOrderedAndProject (72) -+- * HashAggregate (71) - +- * ColumnarToRow (70) - +- CometColumnarExchange (69) - +- RowToColumnar (68) - +- * HashAggregate (67) - +- * Expand (66) - +- Union (65) - :- * HashAggregate (22) - : +- * ColumnarToRow (21) - : +- CometColumnarExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometUnion (7) - : : : :- CometProject (3) - : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometScan parquet spark_catalog.default.store_returns (4) - : : +- CometBroadcastExchange (11) - : : +- CometProject (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.date_dim (8) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.store (14) - :- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometColumnarExchange (39) - : +- CometHashAggregate (38) - : +- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometProject (32) - : : +- CometBroadcastHashJoin (31) - : : :- CometUnion (29) - : : : :- CometProject (25) - : : : : +- CometFilter (24) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) - : : : +- CometProject (28) - : : : +- CometFilter (27) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) - : : +- ReusedExchange (30) - : +- CometBroadcastExchange (35) - : +- CometFilter (34) - : +- CometScan parquet spark_catalog.default.catalog_page (33) - +- * HashAggregate (64) - +- * ColumnarToRow (63) - +- CometColumnarExchange (62) - +- CometHashAggregate (61) - +- CometProject (60) - +- CometBroadcastHashJoin (59) - :- CometProject (55) - : +- CometBroadcastHashJoin (54) - : :- CometUnion (52) - : : :- CometProject (44) - : : : +- CometFilter (43) - : : : +- CometScan parquet spark_catalog.default.web_sales (42) - : : +- CometProject (51) - : : +- CometBroadcastHashJoin (50) - : : :- CometBroadcastExchange (46) - : : : +- CometScan parquet spark_catalog.default.web_returns (45) - : : +- CometProject (49) - : : +- CometFilter (48) - : : +- CometScan parquet spark_catalog.default.web_sales (47) - : +- ReusedExchange (53) - +- CometBroadcastExchange (58) - +- CometFilter (57) - +- CometScan parquet spark_catalog.default.web_site (56) +TakeOrderedAndProject (70) ++- * HashAggregate (69) + +- Exchange (68) + +- * HashAggregate (67) + +- * Expand (66) + +- Union (65) + :- * HashAggregate (22) + : +- * ColumnarToRow (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometUnion (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.date_dim (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan parquet spark_catalog.default.store (14) + :- * HashAggregate (41) + : +- * ColumnarToRow (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometUnion (29) + : : : :- CometProject (25) + : : : : +- CometFilter (24) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) + : : : +- CometProject (28) + : : : +- CometFilter (27) + : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) + : : +- ReusedExchange (30) + : +- CometBroadcastExchange (35) + : +- CometFilter (34) + : +- CometScan parquet spark_catalog.default.catalog_page (33) + +- * HashAggregate (64) + +- * ColumnarToRow (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometUnion (52) + : : :- CometProject (44) + : : : +- CometFilter (43) + : : : +- CometScan parquet spark_catalog.default.web_sales (42) + : : +- CometProject (51) + : : +- CometBroadcastHashJoin (50) + : : :- CometBroadcastExchange (46) + : : : +- CometScan parquet spark_catalog.default.web_returns (45) + : : +- CometProject (49) + : : +- CometFilter (48) + : : +- CometScan parquet spark_catalog.default.web_sales (47) + : +- ReusedExchange (53) + +- CometBroadcastExchange (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.web_site (56) (1) Scan parquet spark_catalog.default.store_sales @@ -166,9 +164,9 @@ Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#25] Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] -(20) CometColumnarExchange +(20) CometExchange Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] -Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) ColumnarToRow [codegen id : 1] Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] @@ -257,9 +255,9 @@ Input [5]: [sales_price#46, profit#47, return_amt#48, net_loss#49, cp_catalog_pa Keys [1]: [cp_catalog_page_id#62] Functions [4]: [partial_sum(UnscaledValue(sales_price#46)), partial_sum(UnscaledValue(return_amt#48)), partial_sum(UnscaledValue(profit#47)), partial_sum(UnscaledValue(net_loss#49))] -(39) CometColumnarExchange +(39) CometExchange Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] -Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) ColumnarToRow [codegen id : 2] Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] @@ -367,9 +365,9 @@ Input [5]: [sales_price#83, profit#84, return_amt#85, net_loss#86, web_site_id#1 Keys [1]: [web_site_id#104] Functions [4]: [partial_sum(UnscaledValue(sales_price#83)), partial_sum(UnscaledValue(return_amt#85)), partial_sum(UnscaledValue(profit#84)), partial_sum(UnscaledValue(net_loss#86))] -(62) CometColumnarExchange +(62) CometExchange Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] -Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (63) ColumnarToRow [codegen id : 3] Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] @@ -394,56 +392,50 @@ Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(prof Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] Results [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -(68) RowToColumnar +(68) Exchange Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(69) CometColumnarExchange -Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(70) ColumnarToRow [codegen id : 5] -Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] - -(71) HashAggregate [codegen id : 5] +(69) HashAggregate [codegen id : 5] Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] Keys [3]: [channel#118, id#119, spark_grouping_id#120] Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] Aggregate Attributes [3]: [sum(sales#34)#133, sum(returns#35)#134, sum(profit#36)#135] Results [5]: [channel#118, id#119, sum(sales#34)#133 AS sales#136, sum(returns#35)#134 AS returns#137, sum(profit#36)#135 AS profit#138] -(72) TakeOrderedAndProject +(70) TakeOrderedAndProject Input [5]: [channel#118, id#119, sales#136, returns#137, profit#138] Arguments: 100, [channel#118 ASC NULLS FIRST, id#119 ASC NULLS FIRST], [channel#118, id#119, sales#136, returns#137, profit#138] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (77) -+- * ColumnarToRow (76) - +- CometProject (75) - +- CometFilter (74) - +- CometScan parquet spark_catalog.default.date_dim (73) +BroadcastExchange (75) ++- * ColumnarToRow (74) + +- CometProject (73) + +- CometFilter (72) + +- CometScan parquet spark_catalog.default.date_dim (71) -(73) Scan parquet spark_catalog.default.date_dim +(71) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] ReadSchema: struct -(74) CometFilter +(72) CometFilter Input [2]: [d_date_sk#22, d_date#23] Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-06)) AND isnotnull(d_date_sk#22)) -(75) CometProject +(73) CometProject Input [2]: [d_date_sk#22, d_date#23] Arguments: [d_date_sk#22], [d_date_sk#22] -(76) ColumnarToRow [codegen id : 1] +(74) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#22] -(77) BroadcastExchange +(75) BroadcastExchange Input [1]: [d_date_sk#22] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/simplified.txt index 7b4b411a4..c6f9f60af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q5/simplified.txt @@ -1,95 +1,93 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (5) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_id] #2 - CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #5 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [cp_catalog_page_id] #6 - CometHashAggregate [cp_catalog_page_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [page_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [web_site_id] #8 - CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [web_site_sk,web_site_id] #10 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + InputAdapter + Exchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #6 + CometHashAggregate [cp_catalog_page_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [web_site_id] #8 + CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/explain.txt index 1e9d7743c..84eef7b55 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (28) +- CometTakeOrderedAndProject (27) +- CometHashAggregate (26) - +- CometColumnarExchange (25) + +- CometExchange (25) +- CometHashAggregate (24) +- CometProject (23) +- CometBroadcastHashJoin (22) @@ -147,9 +147,9 @@ Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_compan Keys [10]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(25) CometColumnarExchange +(25) CometExchange Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, sum#26, sum#27, sum#28, sum#29, sum#30] -Arguments: hashpartitioning(s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometHashAggregate Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, sum#26, sum#27, sum#28, sum#29, sum#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt index 075786f9b..533334464 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q50/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] CometHashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] - CometColumnarExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 + CometExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 CometHashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum,sr_returned_date_sk,ss_sold_date_sk] CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt index 3bd86214d..fcd43fe3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/explain.txt @@ -1,47 +1,44 @@ == Physical Plan == -* ColumnarToRow (43) -+- CometTakeOrderedAndProject (42) - +- CometFilter (41) - +- CometWindowExec (40) - +- CometSort (39) - +- CometColumnarExchange (38) - +- CometProject (37) - +- CometSortMergeJoin (36) - :- CometSort (19) - : +- CometColumnarExchange (18) - : +- RowToColumnar (17) - : +- * Project (16) - : +- Window (15) - : +- * ColumnarToRow (14) - : +- CometSort (13) - : +- CometColumnarExchange (12) - : +- CometHashAggregate (11) - : +- CometColumnarExchange (10) - : +- CometHashAggregate (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.web_sales (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.date_dim (3) - +- CometSort (35) - +- CometColumnarExchange (34) - +- RowToColumnar (33) - +- * Project (32) - +- Window (31) - +- * ColumnarToRow (30) - +- CometSort (29) - +- CometColumnarExchange (28) - +- CometHashAggregate (27) - +- CometColumnarExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometFilter (21) - : +- CometScan parquet spark_catalog.default.store_sales (20) - +- ReusedExchange (22) +TakeOrderedAndProject (40) ++- * Filter (39) + +- Window (38) + +- * Sort (37) + +- Exchange (36) + +- * Project (35) + +- * SortMergeJoin FullOuter (34) + :- * Sort (18) + : +- Exchange (17) + : +- * Project (16) + : +- Window (15) + : +- * ColumnarToRow (14) + : +- CometSort (13) + : +- CometExchange (12) + : +- CometHashAggregate (11) + : +- CometExchange (10) + : +- CometHashAggregate (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.date_dim (3) + +- * Sort (33) + +- Exchange (32) + +- * Project (31) + +- Window (30) + +- * ColumnarToRow (29) + +- CometSort (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometFilter (20) + : +- CometScan parquet spark_catalog.default.store_sales (19) + +- ReusedExchange (21) (1) Scan parquet spark_catalog.default.web_sales @@ -89,18 +86,18 @@ Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6] Keys [2]: [ws_item_sk#1, d_date#6] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -(10) CometColumnarExchange +(10) CometExchange Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate Input [3]: [ws_item_sk#1, d_date#6, sum#8] Keys [2]: [ws_item_sk#1, d_date#6] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -(12) CometColumnarExchange +(12) CometExchange Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (13) CometSort Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] @@ -117,18 +114,15 @@ Arguments: [sum(_w0#10) windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FI Output [3]: [item_sk#9, d_date#6, cume_sales#11] Input [5]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1, cume_sales#11] -(17) RowToColumnar +(17) Exchange Input [3]: [item_sk#9, d_date#6, cume_sales#11] +Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(18) CometColumnarExchange +(18) Sort [codegen id : 3] Input [3]: [item_sk#9, d_date#6, cume_sales#11] -Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0 -(19) CometSort -Input [3]: [item_sk#9, d_date#6, cume_sales#11] -Arguments: [item_sk#9, d_date#6, cume_sales#11], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] - -(20) Scan parquet spark_catalog.default.store_sales +(19) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -136,130 +130,125 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#14), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(21) CometFilter +(20) CometFilter Input [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] Condition : isnotnull(ss_item_sk#12) -(22) ReusedExchange [Reuses operator id: 6] +(21) ReusedExchange [Reuses operator id: 6] Output [2]: [d_date_sk#16, d_date#17] -(23) CometBroadcastHashJoin +(22) CometBroadcastHashJoin Left output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] Right output [2]: [d_date_sk#16, d_date#17] Arguments: [ss_sold_date_sk#14], [d_date_sk#16], Inner, BuildRight -(24) CometProject +(23) CometProject Input [5]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#16, d_date#17] Arguments: [ss_item_sk#12, ss_sales_price#13, d_date#17], [ss_item_sk#12, ss_sales_price#13, d_date#17] -(25) CometHashAggregate +(24) CometHashAggregate Input [3]: [ss_item_sk#12, ss_sales_price#13, d_date#17] Keys [2]: [ss_item_sk#12, d_date#17] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))] -(26) CometColumnarExchange +(25) CometExchange Input [3]: [ss_item_sk#12, d_date#17, sum#18] -Arguments: hashpartitioning(ss_item_sk#12, d_date#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ss_item_sk#12, d_date#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(27) CometHashAggregate +(26) CometHashAggregate Input [3]: [ss_item_sk#12, d_date#17, sum#18] Keys [2]: [ss_item_sk#12, d_date#17] Functions [1]: [sum(UnscaledValue(ss_sales_price#13))] -(28) CometColumnarExchange +(27) CometExchange Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] -Arguments: hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(29) CometSort +(28) CometSort Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] Arguments: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12], [ss_item_sk#12 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST] -(30) ColumnarToRow [codegen id : 3] +(29) ColumnarToRow [codegen id : 4] Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] -(31) Window +(30) Window Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] Arguments: [sum(_w0#20) windowspecdefinition(ss_item_sk#12, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#21], [ss_item_sk#12], [d_date#17 ASC NULLS FIRST] -(32) Project [codegen id : 4] +(31) Project [codegen id : 5] Output [3]: [item_sk#19, d_date#17, cume_sales#21] Input [5]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12, cume_sales#21] -(33) RowToColumnar -Input [3]: [item_sk#19, d_date#17, cume_sales#21] - -(34) CometColumnarExchange +(32) Exchange Input [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: hashpartitioning(item_sk#19, d_date#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(item_sk#19, d_date#17, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(35) CometSort +(33) Sort [codegen id : 6] Input [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#19, d_date#17, cume_sales#21], [item_sk#19 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST] +Arguments: [item_sk#19 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], false, 0 -(36) CometSortMergeJoin -Left output [3]: [item_sk#9, d_date#6, cume_sales#11] -Right output [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#9, d_date#6], [item_sk#19, d_date#17], FullOuter +(34) SortMergeJoin [codegen id : 7] +Left keys [2]: [item_sk#9, d_date#6] +Right keys [2]: [item_sk#19, d_date#17] +Join type: FullOuter +Join condition: None -(37) CometProject +(35) Project [codegen id : 7] +Output [4]: [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#19 END AS item_sk#22, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#17 END AS d_date#23, cume_sales#11 AS web_sales#24, cume_sales#21 AS store_sales#25] Input [6]: [item_sk#9, d_date#6, cume_sales#11, item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#19 END AS item_sk#22, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#17 END AS d_date#23, cume_sales#11 AS web_sales#24, cume_sales#21 AS store_sales#25] -(38) CometColumnarExchange +(36) Exchange Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: hashpartitioning(item_sk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(item_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(39) CometSort +(37) Sort [codegen id : 8] Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST] +Arguments: [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], false, 0 -(40) CometWindowExec +(38) Window Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27], [max(web_sales#24) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#26, max(store_sales#25) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#27], [item_sk#22], [d_date#23 ASC NULLS FIRST] +Arguments: [max(web_sales#24) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#26, max(store_sales#25) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#27], [item_sk#22], [d_date#23 ASC NULLS FIRST] -(41) CometFilter +(39) Filter [codegen id : 9] Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] Condition : ((isnotnull(web_cumulative#26) AND isnotnull(store_cumulative#27)) AND (web_cumulative#26 > store_cumulative#27)) -(42) CometTakeOrderedAndProject -Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_sk#22 ASC NULLS FIRST,d_date#23 ASC NULLS FIRST], output=[item_sk#22,d_date#23,web_sales#24,store_sales#25,web_cumulative#26,store_cumulative#27]), [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27], 100, [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] - -(43) ColumnarToRow [codegen id : 5] +(40) TakeOrderedAndProject Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] +Arguments: 100, [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (48) -+- * ColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan parquet spark_catalog.default.date_dim (44) +BroadcastExchange (45) ++- * ColumnarToRow (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan parquet spark_catalog.default.date_dim (41) -(44) Scan parquet spark_catalog.default.date_dim +(41) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(45) CometFilter +(42) CometFilter Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5)) -(46) CometProject +(43) CometProject Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] -(47) ColumnarToRow [codegen id : 1] +(44) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#5, d_date#6] -(48) BroadcastExchange +(45) BroadcastExchange Input [2]: [d_date_sk#5, d_date#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#4 +Subquery:2 Hosting operator id = 19 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt index 0e5a8e3d7..f972c31c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q51/simplified.txt @@ -1,62 +1,68 @@ -WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - CometFilter [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - CometWindowExec [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - CometSort [item_sk,d_date,web_sales,store_sales] - CometColumnarExchange [item_sk] #1 - CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] - CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #2 - RowToColumnar - WholeStageCodegen (2) - Project [item_sk,d_date,cume_sales] +TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] + WholeStageCodegen (9) + Filter [web_cumulative,store_cumulative] + InputAdapter + Window [web_sales,item_sk,d_date,store_sales] + WholeStageCodegen (8) + Sort [item_sk,d_date] + InputAdapter + Exchange [item_sk] #1 + WholeStageCodegen (7) + Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] + SortMergeJoin [item_sk,d_date,item_sk,d_date] + InputAdapter + WholeStageCodegen (3) + Sort [item_sk,d_date] InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (1) - ColumnarToRow + Exchange [item_sk,d_date] #2 + WholeStageCodegen (2) + Project [item_sk,d_date,cume_sales] InputAdapter - CometSort [item_sk,d_date,_w0,ws_item_sk] - CometColumnarExchange [ws_item_sk] #3 - CometHashAggregate [item_sk,d_date,_w0,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] - CometColumnarExchange [ws_item_sk,d_date] #4 - CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #6 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #7 - RowToColumnar - WholeStageCodegen (4) - Project [item_sk,d_date,cume_sales] + Window [_w0,ws_item_sk,d_date] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ws_item_sk] + CometExchange [ws_item_sk] #3 + CometHashAggregate [item_sk,d_date,_w0,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] + CometExchange [ws_item_sk,d_date] #4 + CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + InputAdapter + WholeStageCodegen (6) + Sort [item_sk,d_date] InputAdapter - Window [_w0,ss_item_sk,d_date] - WholeStageCodegen (3) - ColumnarToRow + Exchange [item_sk,d_date] #7 + WholeStageCodegen (5) + Project [item_sk,d_date,cume_sales] InputAdapter - CometSort [item_sk,d_date,_w0,ss_item_sk] - CometColumnarExchange [ss_item_sk] #8 - CometHashAggregate [item_sk,d_date,_w0,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [ss_item_sk,d_date] #9 - CometHashAggregate [ss_item_sk,d_date,sum,ss_sales_price] - CometProject [ss_item_sk,ss_sales_price,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #6 + Window [_w0,ss_item_sk,d_date] + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ss_item_sk] + CometExchange [ss_item_sk] #8 + CometHashAggregate [item_sk,d_date,_w0,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_item_sk,d_date] #9 + CometHashAggregate [ss_item_sk,d_date,sum,ss_sales_price] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_date] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt index f8beff744..eee36d229 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -93,9 +93,9 @@ Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt index 99f3baf91..af5223b69 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q52/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [d_year,brand_id,brand,ext_price] CometHashAggregate [d_year,brand_id,brand,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [d_year,i_brand,i_brand_id] #1 + CometExchange [d_year,i_brand,i_brand_id] #1 CometHashAggregate [d_year,i_brand,i_brand_id,sum,ss_ext_sales_price] CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt index 71c84c040..8156e275b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/explain.txt @@ -5,9 +5,9 @@ TakeOrderedAndProject (29) +- Window (26) +- * ColumnarToRow (25) +- CometSort (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) - +- CometColumnarExchange (21) + +- CometExchange (21) +- CometHashAggregate (20) +- CometProject (19) +- CometBroadcastHashJoin (18) @@ -127,18 +127,18 @@ Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#17] Keys [2]: [i_manufact_id#5, d_qoy#17] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -(21) CometColumnarExchange +(21) CometExchange Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] Keys [2]: [i_manufact_id#5, d_qoy#17] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] -Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt index a0d09de86..363b84b35 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q53/simplified.txt @@ -8,9 +8,9 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] ColumnarToRow InputAdapter CometSort [i_manufact_id,sum_sales,_w0] - CometColumnarExchange [i_manufact_id] #1 + CometExchange [i_manufact_id] #1 CometHashAggregate [i_manufact_id,sum_sales,_w0,d_qoy,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_manufact_id,d_qoy] #2 + CometExchange [i_manufact_id,d_qoy] #2 CometHashAggregate [i_manufact_id,d_qoy,sum,ss_sales_price] CometProject [i_manufact_id,ss_sales_price,d_qoy] CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt index cf1531550..d5bd8e387 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/explain.txt @@ -2,10 +2,10 @@ * ColumnarToRow (56) +- CometTakeOrderedAndProject (55) +- CometHashAggregate (54) - +- CometColumnarExchange (53) + +- CometExchange (53) +- CometHashAggregate (52) +- CometHashAggregate (51) - +- CometColumnarExchange (50) + +- CometExchange (50) +- CometHashAggregate (49) +- CometProject (48) +- CometBroadcastHashJoin (47) @@ -16,7 +16,7 @@ : : :- CometProject (32) : : : +- CometBroadcastHashJoin (31) : : : :- CometHashAggregate (27) - : : : : +- CometColumnarExchange (26) + : : : : +- CometExchange (26) : : : : +- CometHashAggregate (25) : : : : +- CometProject (24) : : : : +- CometBroadcastHashJoin (23) @@ -178,9 +178,9 @@ Input [2]: [c_customer_sk#20, c_current_addr_sk#21] Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] Functions: [] -(26) CometColumnarExchange +(26) CometExchange Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: hashpartitioning(c_customer_sk#20, c_current_addr_sk#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_sk#20, c_current_addr_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [2]: [c_customer_sk#20, c_current_addr_sk#21] @@ -293,9 +293,9 @@ Input [2]: [c_customer_sk#20, ss_ext_sales_price#23] Keys [1]: [c_customer_sk#20] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#23))] -(50) CometColumnarExchange +(50) CometExchange Input [2]: [c_customer_sk#20, sum#37] -Arguments: hashpartitioning(c_customer_sk#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (51) CometHashAggregate Input [2]: [c_customer_sk#20, sum#37] @@ -307,9 +307,9 @@ Input [1]: [segment#38] Keys [1]: [segment#38] Functions [1]: [partial_count(1)] -(53) CometColumnarExchange +(53) CometExchange Input [2]: [segment#38, count#39] -Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (54) CometHashAggregate Input [2]: [segment#38, count#39] @@ -390,7 +390,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:4 Hosting operator id = 63 Hosting Expression = Subquery scalar-subquery#33, [id=#34] * ColumnarToRow (73) +- CometHashAggregate (72) - +- CometColumnarExchange (71) + +- CometExchange (71) +- CometHashAggregate (70) +- CometProject (69) +- CometFilter (68) @@ -417,9 +417,9 @@ Input [1]: [(d_month_seq + 1)#45] Keys [1]: [(d_month_seq + 1)#45] Functions: [] -(71) CometColumnarExchange +(71) CometExchange Input [1]: [(d_month_seq + 1)#45] -Arguments: hashpartitioning((d_month_seq + 1)#45, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning((d_month_seq + 1)#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (72) CometHashAggregate Input [1]: [(d_month_seq + 1)#45] @@ -432,7 +432,7 @@ Input [1]: [(d_month_seq + 1)#45] Subquery:5 Hosting operator id = 63 Hosting Expression = Subquery scalar-subquery#35, [id=#36] * ColumnarToRow (80) +- CometHashAggregate (79) - +- CometColumnarExchange (78) + +- CometExchange (78) +- CometHashAggregate (77) +- CometProject (76) +- CometFilter (75) @@ -459,9 +459,9 @@ Input [1]: [(d_month_seq + 3)#49] Keys [1]: [(d_month_seq + 3)#49] Functions: [] -(78) CometColumnarExchange +(78) CometExchange Input [1]: [(d_month_seq + 3)#49] -Arguments: hashpartitioning((d_month_seq + 3)#49, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning((d_month_seq + 3)#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (79) CometHashAggregate Input [1]: [(d_month_seq + 3)#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt index d3a89a6c6..129018668 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q54/simplified.txt @@ -3,10 +3,10 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [segment,num_customers,segment_base] CometHashAggregate [segment,num_customers,segment_base,count,count(1)] - CometColumnarExchange [segment] #1 + CometExchange [segment] #1 CometHashAggregate [segment,count] CometHashAggregate [segment,c_customer_sk,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [c_customer_sk] #2 + CometExchange [c_customer_sk] #2 CometHashAggregate [c_customer_sk,sum,ss_ext_sales_price] CometProject [c_customer_sk,ss_ext_sales_price] CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] @@ -17,7 +17,7 @@ WholeStageCodegen (1) CometProject [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometHashAggregate [c_customer_sk,c_current_addr_sk] - CometColumnarExchange [c_customer_sk,c_current_addr_sk] #3 + CometExchange [c_customer_sk,c_current_addr_sk] #3 CometHashAggregate [c_customer_sk,c_current_addr_sk] CometProject [c_customer_sk,c_current_addr_sk] CometBroadcastHashJoin [customer_sk,c_customer_sk,c_current_addr_sk] @@ -67,7 +67,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [(d_month_seq + 1)] - CometColumnarExchange [(d_month_seq + 1)] #10 + CometExchange [(d_month_seq + 1)] #10 CometHashAggregate [(d_month_seq + 1)] CometProject [d_month_seq] [(d_month_seq + 1)] CometFilter [d_month_seq,d_year,d_moy] @@ -77,7 +77,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [(d_month_seq + 3)] - CometColumnarExchange [(d_month_seq + 3)] #11 + CometExchange [(d_month_seq + 3)] #11 CometHashAggregate [(d_month_seq + 3)] CometProject [d_month_seq] [(d_month_seq + 3)] CometFilter [d_month_seq,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt index ee250ed9b..6eff9f52d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -93,9 +93,9 @@ Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] Keys [2]: [i_brand#9, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(16) CometColumnarExchange +(16) CometExchange Input [3]: [i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [3]: [i_brand#9, i_brand_id#8, sum#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt index 45200509f..999c8a6c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q55/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [brand_id,brand,ext_price] CometHashAggregate [brand_id,brand,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_brand,i_brand_id] #1 + CometExchange [i_brand,i_brand_id] #1 CometHashAggregate [i_brand,i_brand_id,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt index 8061fb91e..d9af63246 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/explain.txt @@ -2,11 +2,11 @@ * ColumnarToRow (61) +- CometTakeOrderedAndProject (60) +- CometHashAggregate (59) - +- CometColumnarExchange (58) + +- CometExchange (58) +- CometHashAggregate (57) +- CometUnion (56) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -33,7 +33,7 @@ : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.item (17) :- CometHashAggregate (41) - : +- CometColumnarExchange (40) + : +- CometExchange (40) : +- CometHashAggregate (39) : +- CometProject (38) : +- CometBroadcastHashJoin (37) @@ -47,7 +47,7 @@ : : +- ReusedExchange (33) : +- ReusedExchange (36) +- CometHashAggregate (55) - +- CometColumnarExchange (54) + +- CometExchange (54) +- CometHashAggregate (53) +- CometProject (52) +- CometBroadcastHashJoin (51) @@ -183,9 +183,9 @@ Input [2]: [ss_ext_sales_price#3, i_item_id#12] Keys [1]: [i_item_id#12] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(26) CometColumnarExchange +(26) CometExchange Input [2]: [i_item_id#12, sum#15] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [2]: [i_item_id#12, sum#15] @@ -245,9 +245,9 @@ Input [2]: [cs_ext_sales_price#18, i_item_id#24] Keys [1]: [i_item_id#24] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] -(40) CometColumnarExchange +(40) CometExchange Input [2]: [i_item_id#24, sum#25] -Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (41) CometHashAggregate Input [2]: [i_item_id#24, sum#25] @@ -307,9 +307,9 @@ Input [2]: [ws_ext_sales_price#28, i_item_id#34] Keys [1]: [i_item_id#34] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] -(54) CometColumnarExchange +(54) CometExchange Input [2]: [i_item_id#34, sum#35] -Arguments: hashpartitioning(i_item_id#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (55) CometHashAggregate Input [2]: [i_item_id#34, sum#35] @@ -326,9 +326,9 @@ Input [2]: [i_item_id#12, total_sales#36] Keys [1]: [i_item_id#12] Functions [1]: [partial_sum(total_sales#36)] -(58) CometColumnarExchange +(58) CometExchange Input [3]: [i_item_id#12, sum#39, isEmpty#40] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (59) CometHashAggregate Input [3]: [i_item_id#12, sum#39, isEmpty#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt index 4bb76eecc..2603a2781 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q56/simplified.txt @@ -3,11 +3,11 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,total_sales] CometHashAggregate [i_item_id,total_sales,sum,isEmpty,sum(total_sales)] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,isEmpty,total_sales] CometUnion [i_item_id,total_sales] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_item_id] #2 + CometExchange [i_item_id] #2 CometHashAggregate [i_item_id,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] @@ -42,7 +42,7 @@ WholeStageCodegen (1) CometFilter [i_item_id,i_color] CometScan parquet spark_catalog.default.item [i_item_id,i_color] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_item_id] #8 + CometExchange [i_item_id] #8 CometHashAggregate [i_item_id,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] @@ -57,7 +57,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk] #5 ReusedExchange [i_item_sk,i_item_id] #6 CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_item_id] #9 + CometExchange [i_item_id] #9 CometHashAggregate [i_item_id,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt index b5372cffc..e8a40a0f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/explain.txt @@ -11,9 +11,9 @@ TakeOrderedAndProject (45) : : +- Window (24) : : +- * ColumnarToRow (23) : : +- CometSort (22) - : : +- CometColumnarExchange (21) + : : +- CometExchange (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -135,18 +135,18 @@ Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#10, d_moy#11, cc_n Keys [5]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -(19) CometColumnarExchange +(19) CometExchange Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (20) CometHashAggregate Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#14] Keys [5]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -(21) CometColumnarExchange +(21) CometExchange Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#15, _w0#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (22) CometSort Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#15, _w0#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt index 454d12fae..96a3fdfa0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q57/simplified.txt @@ -16,9 +16,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ ColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,cc_name] #1 + CometExchange [i_category,i_brand,cc_name] #1 CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(cs_sales_price))] - CometColumnarExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum,cs_sales_price] CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt index 06e3fb959..f05dbd720 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/explain.txt @@ -7,7 +7,7 @@ : +- CometBroadcastHashJoin (33) : :- CometFilter (19) : : +- CometHashAggregate (18) - : : +- CometColumnarExchange (17) + : : +- CometExchange (17) : : +- CometHashAggregate (16) : : +- CometProject (15) : : +- CometBroadcastHashJoin (14) @@ -27,7 +27,7 @@ : +- CometBroadcastExchange (32) : +- CometFilter (31) : +- CometHashAggregate (30) - : +- CometColumnarExchange (29) + : +- CometExchange (29) : +- CometHashAggregate (28) : +- CometProject (27) : +- CometBroadcastHashJoin (26) @@ -40,7 +40,7 @@ +- CometBroadcastExchange (47) +- CometFilter (46) +- CometHashAggregate (45) - +- CometColumnarExchange (44) + +- CometExchange (44) +- CometHashAggregate (43) +- CometProject (42) +- CometBroadcastHashJoin (41) @@ -129,9 +129,9 @@ Input [2]: [ss_ext_sales_price#2, i_item_id#6] Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [2]: [i_item_id#6, sum#10] -Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [2]: [i_item_id#6, sum#10] @@ -183,9 +183,9 @@ Input [2]: [cs_ext_sales_price#14, i_item_id#18] Keys [1]: [i_item_id#18] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#14))] -(29) CometColumnarExchange +(29) CometExchange Input [2]: [i_item_id#18, sum#20] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (30) CometHashAggregate Input [2]: [i_item_id#18, sum#20] @@ -250,9 +250,9 @@ Input [2]: [ws_ext_sales_price#24, i_item_id#28] Keys [1]: [i_item_id#28] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#24))] -(44) CometColumnarExchange +(44) CometExchange Input [2]: [i_item_id#28, sum#30] -Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (45) CometHashAggregate Input [2]: [i_item_id#28, sum#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt index 326fdcb78..36fc8ac57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q58/simplified.txt @@ -8,7 +8,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [item_id,ss_item_rev,item_id,cs_item_rev] CometFilter [item_id,ss_item_rev] CometHashAggregate [item_id,ss_item_rev,i_item_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,d_date_sk] @@ -48,7 +48,7 @@ WholeStageCodegen (1) CometBroadcastExchange [item_id,cs_item_rev] #6 CometFilter [item_id,cs_item_rev] CometHashAggregate [item_id,cs_item_rev,i_item_id,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_item_id] #7 + CometExchange [i_item_id] #7 CometHashAggregate [i_item_id,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] @@ -62,7 +62,7 @@ WholeStageCodegen (1) CometBroadcastExchange [item_id,ws_item_rev] #8 CometFilter [item_id,ws_item_rev] CometHashAggregate [item_id,ws_item_rev,i_item_id,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_item_id] #9 + CometExchange [i_item_id] #9 CometHashAggregate [i_item_id,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt index 51afdb3c7..a3f89c9dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/explain.txt @@ -8,7 +8,7 @@ : :- CometProject (15) : : +- CometBroadcastHashJoin (14) : : :- CometHashAggregate (10) - : : : +- CometColumnarExchange (9) + : : : +- CometExchange (9) : : : +- CometHashAggregate (8) : : : +- CometProject (7) : : : +- CometBroadcastHashJoin (6) @@ -81,9 +81,9 @@ Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] Keys [2]: [d_week_seq#5, ss_store_sk#1] Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -(9) CometColumnarExchange +(9) CometExchange Input [9]: [d_week_seq#5, ss_store_sk#1, sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] -Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometHashAggregate Input [9]: [d_week_seq#5, ss_store_sk#1, sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt index bb7a85817..140a76439 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q59/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id,s_store_name] CometHashAggregate [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] - CometColumnarExchange [d_week_seq,ss_store_sk] #1 + CometExchange [d_week_seq,ss_store_sk] #1 CometHashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum,d_day_name,ss_sales_price] CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt index 0341c20d7..7a8044667 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (37) +- CometFilter (36) +- CometHashAggregate (35) - +- CometColumnarExchange (34) + +- CometExchange (34) +- CometHashAggregate (33) +- CometProject (32) +- CometBroadcastHashJoin (31) @@ -33,7 +33,7 @@ +- CometBroadcastExchange (27) +- CometFilter (26) +- CometHashAggregate (25) - +- CometColumnarExchange (24) + +- CometExchange (24) +- CometHashAggregate (23) +- CometFilter (22) +- CometScan parquet spark_catalog.default.item (21) @@ -154,9 +154,9 @@ Input [2]: [i_current_price#16, i_category#17] Keys [1]: [i_category#17] Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] -(24) CometColumnarExchange +(24) CometExchange Input [3]: [i_category#17, sum#18, count#19] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (25) CometHashAggregate Input [3]: [i_category#17, sum#18, count#19] @@ -198,9 +198,9 @@ Input [1]: [ca_state#2] Keys [1]: [ca_state#2] Functions [1]: [partial_count(1)] -(34) CometColumnarExchange +(34) CometExchange Input [2]: [ca_state#2, count#21] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (35) CometHashAggregate Input [2]: [ca_state#2, count#21] @@ -253,7 +253,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 40 Hosting Expression = Subquery scalar-subquery#11, [id=#12] * ColumnarToRow (50) +- CometHashAggregate (49) - +- CometColumnarExchange (48) + +- CometExchange (48) +- CometHashAggregate (47) +- CometProject (46) +- CometFilter (45) @@ -280,9 +280,9 @@ Input [1]: [d_month_seq#24] Keys [1]: [d_month_seq#24] Functions: [] -(48) CometColumnarExchange +(48) CometExchange Input [1]: [d_month_seq#24] -Arguments: hashpartitioning(d_month_seq#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(d_month_seq#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (49) CometHashAggregate Input [1]: [d_month_seq#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt index 0e983b96a..05566cbca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q6/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [state,cnt] CometFilter [state,cnt] CometHashAggregate [state,cnt,ca_state,count,count(1)] - CometColumnarExchange [ca_state] #1 + CometExchange [ca_state] #1 CometHashAggregate [ca_state,count] CometProject [ca_state] CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] @@ -34,7 +34,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [d_month_seq] - CometColumnarExchange [d_month_seq] #5 + CometExchange [d_month_seq] #5 CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] @@ -53,7 +53,7 @@ WholeStageCodegen (1) CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] - CometColumnarExchange [i_category] #9 + CometExchange [i_category] #9 CometHashAggregate [i_category,sum,count,i_current_price] CometFilter [i_current_price,i_category] CometScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt index 0aeb4e2b4..3b5101925 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/explain.txt @@ -2,11 +2,11 @@ * ColumnarToRow (61) +- CometTakeOrderedAndProject (60) +- CometHashAggregate (59) - +- CometColumnarExchange (58) + +- CometExchange (58) +- CometHashAggregate (57) +- CometUnion (56) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -33,7 +33,7 @@ : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.item (17) :- CometHashAggregate (41) - : +- CometColumnarExchange (40) + : +- CometExchange (40) : +- CometHashAggregate (39) : +- CometProject (38) : +- CometBroadcastHashJoin (37) @@ -47,7 +47,7 @@ : : +- ReusedExchange (33) : +- ReusedExchange (36) +- CometHashAggregate (55) - +- CometColumnarExchange (54) + +- CometExchange (54) +- CometHashAggregate (53) +- CometProject (52) +- CometBroadcastHashJoin (51) @@ -183,9 +183,9 @@ Input [2]: [ss_ext_sales_price#3, i_item_id#12] Keys [1]: [i_item_id#12] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(26) CometColumnarExchange +(26) CometExchange Input [2]: [i_item_id#12, sum#15] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [2]: [i_item_id#12, sum#15] @@ -245,9 +245,9 @@ Input [2]: [cs_ext_sales_price#18, i_item_id#24] Keys [1]: [i_item_id#24] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] -(40) CometColumnarExchange +(40) CometExchange Input [2]: [i_item_id#24, sum#25] -Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (41) CometHashAggregate Input [2]: [i_item_id#24, sum#25] @@ -307,9 +307,9 @@ Input [2]: [ws_ext_sales_price#28, i_item_id#34] Keys [1]: [i_item_id#34] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] -(54) CometColumnarExchange +(54) CometExchange Input [2]: [i_item_id#34, sum#35] -Arguments: hashpartitioning(i_item_id#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (55) CometHashAggregate Input [2]: [i_item_id#34, sum#35] @@ -326,9 +326,9 @@ Input [2]: [i_item_id#12, total_sales#36] Keys [1]: [i_item_id#12] Functions [1]: [partial_sum(total_sales#36)] -(58) CometColumnarExchange +(58) CometExchange Input [3]: [i_item_id#12, sum#39, isEmpty#40] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (59) CometHashAggregate Input [3]: [i_item_id#12, sum#39, isEmpty#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt index 871f477b7..4dd8b3b5b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q60/simplified.txt @@ -3,11 +3,11 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,total_sales] CometHashAggregate [i_item_id,total_sales,sum,isEmpty,sum(total_sales)] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,isEmpty,total_sales] CometUnion [i_item_id,total_sales] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_item_id] #2 + CometExchange [i_item_id] #2 CometHashAggregate [i_item_id,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] @@ -42,7 +42,7 @@ WholeStageCodegen (1) CometFilter [i_item_id,i_category] CometScan parquet spark_catalog.default.item [i_item_id,i_category] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_item_id] #8 + CometExchange [i_item_id] #8 CometHashAggregate [i_item_id,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] @@ -57,7 +57,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk] #5 ReusedExchange [i_item_sk,i_item_id] #6 CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_item_id] #9 + CometExchange [i_item_id] #9 CometHashAggregate [i_item_id,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt index 3bbacfbb9..8fcd94392 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/explain.txt @@ -3,7 +3,7 @@ +- * BroadcastNestedLoopJoin Inner BuildRight (64) :- * ColumnarToRow (41) : +- CometHashAggregate (40) - : +- CometColumnarExchange (39) + : +- CometExchange (39) : +- CometHashAggregate (38) : +- CometProject (37) : +- CometBroadcastHashJoin (36) @@ -45,7 +45,7 @@ +- BroadcastExchange (63) +- * ColumnarToRow (62) +- CometHashAggregate (61) - +- CometColumnarExchange (60) + +- CometExchange (60) +- CometHashAggregate (59) +- CometProject (58) +- CometBroadcastHashJoin (57) @@ -247,9 +247,9 @@ Input [1]: [ss_ext_sales_price#5] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(39) CometColumnarExchange +(39) CometExchange Input [1]: [sum#23] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate Input [1]: [sum#23] @@ -336,9 +336,9 @@ Input [1]: [ss_ext_sales_price#28] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#28))] -(60) CometColumnarExchange +(60) CometExchange Input [1]: [sum#37] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (61) CometHashAggregate Input [1]: [sum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/simplified.txt index b95b0d616..548614416 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q61/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [promotions,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,ss_ext_sales_price] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] @@ -57,7 +57,7 @@ WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [total,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange #10 + CometExchange #10 CometHashAggregate [sum,ss_ext_sales_price] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62/explain.txt index f58411101..82005f930 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (28) +- CometTakeOrderedAndProject (27) +- CometHashAggregate (26) - +- CometColumnarExchange (25) + +- CometExchange (25) +- CometHashAggregate (24) +- CometProject (23) +- CometBroadcastHashJoin (22) @@ -145,9 +145,9 @@ Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _group Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(25) CometColumnarExchange +(25) CometExchange Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometHashAggregate Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62/simplified.txt index c7bcf72f0..6a9b1ced3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q62/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] CometHashAggregate [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum,sum,sum,sum,sum,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] - CometColumnarExchange [_groupingexpression,sm_type,web_name] #1 + CometExchange [_groupingexpression,sm_type,web_name] #1 CometHashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum,ws_ship_date_sk,ws_sold_date_sk] CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/explain.txt index 5fd263660..4dfd50a34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/explain.txt @@ -5,9 +5,9 @@ TakeOrderedAndProject (29) +- Window (26) +- * ColumnarToRow (25) +- CometSort (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) - +- CometColumnarExchange (21) + +- CometExchange (21) +- CometHashAggregate (20) +- CometProject (19) +- CometBroadcastHashJoin (18) @@ -127,18 +127,18 @@ Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#17] Keys [2]: [i_manager_id#5, d_moy#17] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -(21) CometColumnarExchange +(21) CometExchange Input [3]: [i_manager_id#5, d_moy#17, sum#19] -Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate Input [3]: [i_manager_id#5, d_moy#17, sum#19] Keys [2]: [i_manager_id#5, d_moy#17] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] -Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/simplified.txt index 1b538c9ab..8ea16ea74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q63/simplified.txt @@ -8,9 +8,9 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] ColumnarToRow InputAdapter CometSort [i_manager_id,sum_sales,_w0] - CometColumnarExchange [i_manager_id] #1 + CometExchange [i_manager_id] #1 CometHashAggregate [i_manager_id,sum_sales,_w0,d_moy,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_manager_id,d_moy] #2 + CometExchange [i_manager_id,d_moy] #2 CometHashAggregate [i_manager_id,d_moy,sum,ss_sales_price] CometProject [i_manager_id,ss_sales_price,d_moy] CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/explain.txt index b4e05a864..35bc18855 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/explain.txt @@ -7,7 +7,7 @@ +- * SortMergeJoin Inner (171) :- * ColumnarToRow (101) : +- CometSort (100) - : +- CometColumnarExchange (99) + : +- CometExchange (99) : +- CometHashAggregate (98) : +- CometHashAggregate (97) : +- CometProject (96) @@ -43,7 +43,7 @@ : : : : : : : : : : : : : : : :- CometProject (30) : : : : : : : : : : : : : : : : +- CometSortMergeJoin (29) : : : : : : : : : : : : : : : : :- CometSort (10) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (9) + : : : : : : : : : : : : : : : : : +- CometExchange (9) : : : : : : : : : : : : : : : : : +- CometProject (8) : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) @@ -56,17 +56,17 @@ : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) : : : : : : : : : : : : : : : : +- CometHashAggregate (25) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (24) + : : : : : : : : : : : : : : : : +- CometExchange (24) : : : : : : : : : : : : : : : : +- CometHashAggregate (23) : : : : : : : : : : : : : : : : +- CometProject (22) : : : : : : : : : : : : : : : : +- CometSortMergeJoin (21) : : : : : : : : : : : : : : : : :- CometSort (15) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (14) + : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (19) + : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (16) @@ -108,7 +108,7 @@ : +- CometScan parquet spark_catalog.default.item (91) +- * ColumnarToRow (170) +- CometSort (169) - +- CometColumnarExchange (168) + +- CometExchange (168) +- CometHashAggregate (167) +- CometHashAggregate (166) +- CometProject (165) @@ -144,7 +144,7 @@ : : : : : : : : : : : : : : :- CometProject (118) : : : : : : : : : : : : : : : +- CometSortMergeJoin (117) : : : : : : : : : : : : : : : :- CometSort (111) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (110) + : : : : : : : : : : : : : : : : +- CometExchange (110) : : : : : : : : : : : : : : : : +- CometProject (109) : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (108) : : : : : : : : : : : : : : : : :- CometBroadcastExchange (104) @@ -217,9 +217,9 @@ Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number# Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(9) CometColumnarExchange +(9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -240,9 +240,9 @@ Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -(14) CometColumnarExchange +(14) CometExchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (15) CometSort Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] @@ -263,9 +263,9 @@ Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -(19) CometColumnarExchange +(19) CometExchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] @@ -285,9 +285,9 @@ Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reverse Keys [1]: [cs_item_sk#17] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] -(24) CometColumnarExchange +(24) CometExchange Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometHashAggregate Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] @@ -629,9 +629,9 @@ Input [19]: [i_product_name#71, i_item_sk#68, s_store_name#35, s_zip#36, ca_stre Keys [15]: [i_product_name#71, i_item_sk#68, s_store_name#35, s_zip#36, ca_street_number#57, ca_street_name#58, ca_city#59, ca_zip#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, d_year#33, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -(99) CometColumnarExchange +(99) CometExchange Input [17]: [product_name#76, item_sk#77, store_name#78, store_zip#79, b_street_number#80, b_streen_name#81, b_city#82, b_zip#83, c_street_number#84, c_street_name#85, c_city#86, c_zip#87, syear#88, cnt#89, s1#90, s2#91, s3#92] -Arguments: hashpartitioning(item_sk#77, store_name#78, store_zip#79, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(item_sk#77, store_name#78, store_zip#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (100) CometSort Input [17]: [product_name#76, item_sk#77, store_name#78, store_zip#79, b_street_number#80, b_streen_name#81, b_city#82, b_zip#83, c_street_number#84, c_street_name#85, c_city#86, c_zip#87, syear#88, cnt#89, s1#90, s2#91, s3#92] @@ -680,9 +680,9 @@ Arguments: [ss_item_sk#93, ss_ticket_number#100], [sr_item_sk#106, sr_ticket_num Input [14]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_ticket_number#100, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104, sr_item_sk#106, sr_ticket_number#107] Arguments: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104], [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] -(110) CometColumnarExchange +(110) CometExchange Input [11]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] -Arguments: hashpartitioning(ss_item_sk#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ss_item_sk#93, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (111) CometSort Input [11]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] @@ -919,9 +919,9 @@ Input [19]: [i_product_name#154, i_item_sk#153, s_store_name#120, s_zip#121, ca_ Keys [15]: [i_product_name#154, i_item_sk#153, s_store_name#120, s_zip#121, ca_street_number#142, ca_street_name#143, ca_city#144, ca_zip#145, ca_street_number#147, ca_street_name#148, ca_city#149, ca_zip#150, d_year#118, d_year#129, d_year#131] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#101)), sum(UnscaledValue(ss_list_price#102)), sum(UnscaledValue(ss_coupon_amt#103))] -(168) CometColumnarExchange +(168) CometExchange Input [8]: [item_sk#158, store_name#159, store_zip#160, syear#161, cnt#162, s1#163, s2#164, s3#165] -Arguments: hashpartitioning(item_sk#158, store_name#159, store_zip#160, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(item_sk#158, store_name#159, store_zip#160, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (169) CometSort Input [8]: [item_sk#158, store_name#159, store_zip#160, syear#161, cnt#162, s1#163, s2#164, s3#165] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/simplified.txt index 84be0e3c3..665cf4e99 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q64/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (4) ColumnarToRow InputAdapter CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #2 + CometExchange [item_sk,store_name,store_zip] #2 CometHashAggregate [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -48,7 +48,7 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #3 + CometExchange [ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 @@ -68,17 +68,17 @@ WholeStageCodegen (4) CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - CometColumnarExchange [cs_item_sk] #6 + CometExchange [cs_item_sk] #6 CometHashAggregate [cs_item_sk,sum,sum,isEmpty,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometColumnarExchange [cs_item_sk,cs_order_number] #7 + CometExchange [cs_item_sk,cs_order_number] #7 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometColumnarExchange [cr_item_sk,cr_order_number] #8 + CometExchange [cr_item_sk,cr_order_number] #8 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] @@ -123,7 +123,7 @@ WholeStageCodegen (4) ColumnarToRow InputAdapter CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #19 + CometExchange [item_sk,store_name,store_zip] #19 CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -159,7 +159,7 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #20 + CometExchange [ss_item_sk] #20 CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #21 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/explain.txt index dd1796dd4..179c42f98 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/explain.txt @@ -12,7 +12,7 @@ : : +- CometBroadcastExchange (15) : : +- CometFilter (14) : : +- CometHashAggregate (13) - : : +- CometColumnarExchange (12) + : : +- CometExchange (12) : : +- CometHashAggregate (11) : : +- CometProject (10) : : +- CometBroadcastHashJoin (9) @@ -28,10 +28,10 @@ +- CometBroadcastExchange (35) +- CometFilter (34) +- CometHashAggregate (33) - +- CometColumnarExchange (32) + +- CometExchange (32) +- CometHashAggregate (31) +- CometHashAggregate (30) - +- CometColumnarExchange (29) + +- CometExchange (29) +- CometHashAggregate (28) +- CometProject (27) +- CometBroadcastHashJoin (26) @@ -96,9 +96,9 @@ Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] -(12) CometColumnarExchange +(12) CometExchange Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (13) CometHashAggregate Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] @@ -175,9 +175,9 @@ Input [3]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] Keys [2]: [ss_store_sk#18, ss_item_sk#17] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#19))] -(29) CometColumnarExchange +(29) CometExchange Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#23] -Arguments: hashpartitioning(ss_store_sk#18, ss_item_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_store_sk#18, ss_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (30) CometHashAggregate Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#23] @@ -189,9 +189,9 @@ Input [2]: [ss_store_sk#18, revenue#24] Keys [1]: [ss_store_sk#18] Functions [1]: [partial_avg(revenue#24)] -(32) CometColumnarExchange +(32) CometExchange Input [3]: [ss_store_sk#18, sum#25, count#26] -Arguments: hashpartitioning(ss_store_sk#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_store_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate Input [3]: [ss_store_sk#18, sum#25, count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt index e41f9161d..be08ee23a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q65/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 CometFilter [ss_store_sk,ss_item_sk,revenue] CometHashAggregate [ss_store_sk,ss_item_sk,revenue,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [ss_store_sk,ss_item_sk] #2 + CometExchange [ss_store_sk,ss_item_sk] #2 CometHashAggregate [ss_store_sk,ss_item_sk,sum,ss_sales_price] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] @@ -37,10 +37,10 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_store_sk,ave] #6 CometFilter [ss_store_sk,ave] CometHashAggregate [ss_store_sk,ave,sum,count,avg(revenue)] - CometColumnarExchange [ss_store_sk] #7 + CometExchange [ss_store_sk] #7 CometHashAggregate [ss_store_sk,sum,count,revenue] CometHashAggregate [ss_store_sk,revenue,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [ss_store_sk,ss_item_sk] #8 + CometExchange [ss_store_sk,ss_item_sk] #8 CometHashAggregate [ss_store_sk,ss_item_sk,sum,ss_sales_price] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt index d0c1814d1..c8cdc8094 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/explain.txt @@ -2,11 +2,11 @@ * ColumnarToRow (50) +- CometTakeOrderedAndProject (49) +- CometHashAggregate (48) - +- CometColumnarExchange (47) + +- CometExchange (47) +- CometHashAggregate (46) +- CometUnion (45) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -33,7 +33,7 @@ : +- CometFilter (20) : +- CometScan parquet spark_catalog.default.ship_mode (19) +- CometHashAggregate (44) - +- CometColumnarExchange (43) + +- CometExchange (43) +- CometHashAggregate (42) +- CometProject (41) +- CometBroadcastHashJoin (40) @@ -172,9 +172,9 @@ Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_nam Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17] Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -(26) CometColumnarExchange +(26) CometExchange Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] @@ -246,9 +246,9 @@ Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_wareho Keys [7]: [w_warehouse_name#80, w_warehouse_sq_ft#81, w_city#82, w_county#83, w_state#84, w_country#85, d_year#87] Functions [24]: [partial_sum(CASE WHEN (d_moy#88 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] -(43) CometColumnarExchange +(43) CometExchange Input [55]: [w_warehouse_name#80, w_warehouse_sq_ft#81, w_city#82, w_county#83, w_state#84, w_country#85, d_year#87, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Arguments: hashpartitioning(w_warehouse_name#80, w_warehouse_sq_ft#81, w_city#82, w_county#83, w_state#84, w_country#85, d_year#87, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(w_warehouse_name#80, w_warehouse_sq_ft#81, w_city#82, w_county#83, w_state#84, w_country#85, d_year#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (44) CometHashAggregate Input [55]: [w_warehouse_name#80, w_warehouse_sq_ft#81, w_city#82, w_county#83, w_state#84, w_country#85, d_year#87, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] @@ -264,9 +264,9 @@ Input [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#139, year#140] Functions [36]: [partial_sum(jan_sales#141), partial_sum(feb_sales#142), partial_sum(mar_sales#143), partial_sum(apr_sales#144), partial_sum(may_sales#145), partial_sum(jun_sales#146), partial_sum(jul_sales#147), partial_sum(aug_sales#148), partial_sum(sep_sales#149), partial_sum(oct_sales#150), partial_sum(nov_sales#151), partial_sum(dec_sales#152), partial_sum((jan_sales#141 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((feb_sales#142 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((mar_sales#143 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((apr_sales#144 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((may_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jun_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jul_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((aug_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((sep_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((oct_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((nov_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((dec_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum(jan_net#153), partial_sum(feb_net#154), partial_sum(mar_net#155), partial_sum(apr_net#156), partial_sum(may_net#157), partial_sum(jun_net#158), partial_sum(jul_net#159), partial_sum(aug_net#160), partial_sum(sep_net#161), partial_sum(oct_net#162), partial_sum(nov_net#163), partial_sum(dec_net#164)] -(47) CometColumnarExchange +(47) CometExchange Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#139, year#140, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#139, year#140, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#139, year#140, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (48) CometHashAggregate Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#139, year#140, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/simplified.txt index ce4354bc5..a2b5963e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q66/simplified.txt @@ -3,11 +3,11 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] CometUnion [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometFilter [sm_ship_mode_sk,sm_carrier] CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #8 + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #8 CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/explain.txt index a9602820d..a67cb1c71 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/explain.txt @@ -1,37 +1,35 @@ == Physical Plan == -TakeOrderedAndProject (33) -+- * Filter (32) - +- Window (31) - +- WindowGroupLimit (30) - +- * ColumnarToRow (29) - +- CometSort (28) - +- CometColumnarExchange (27) - +- RowToColumnar (26) - +- WindowGroupLimit (25) - +- * ColumnarToRow (24) - +- CometSort (23) - +- CometHashAggregate (22) - +- CometColumnarExchange (21) - +- CometHashAggregate (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.store (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometScan parquet spark_catalog.default.item (14) +TakeOrderedAndProject (31) ++- * Filter (30) + +- Window (29) + +- WindowGroupLimit (28) + +- * Sort (27) + +- Exchange (26) + +- WindowGroupLimit (25) + +- * ColumnarToRow (24) + +- CometSort (23) + +- CometHashAggregate (22) + +- CometExchange (21) + +- CometHashAggregate (20) + +- CometExpand (19) + +- CometProject (18) + +- CometBroadcastHashJoin (17) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (16) + +- CometFilter (15) + +- CometScan parquet spark_catalog.default.item (14) (1) Scan parquet spark_catalog.default.store_sales @@ -131,9 +129,9 @@ Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#19, i_class#20, i_brand Keys [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(21) CometColumnarExchange +(21) CometExchange Input [11]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate Input [11]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, sum#28, isEmpty#29] @@ -151,65 +149,59 @@ Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30] Arguments: [i_category#19], [sumsales#30 DESC NULLS LAST], rank(sumsales#30), 100, Partial -(26) RowToColumnar +(26) Exchange Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30] +Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(27) CometColumnarExchange +(27) Sort [codegen id : 2] Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30] -Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: [i_category#19 ASC NULLS FIRST, sumsales#30 DESC NULLS LAST], false, 0 -(28) CometSort -Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30] -Arguments: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30], [i_category#19 ASC NULLS FIRST, sumsales#30 DESC NULLS LAST] - -(29) ColumnarToRow [codegen id : 2] -Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30] - -(30) WindowGroupLimit +(28) WindowGroupLimit Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30] Arguments: [i_category#19], [sumsales#30 DESC NULLS LAST], rank(sumsales#30), 100, Final -(31) Window +(29) Window Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30] Arguments: [rank(sumsales#30) windowspecdefinition(i_category#19, sumsales#30 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#31], [i_category#19], [sumsales#30 DESC NULLS LAST] -(32) Filter [codegen id : 3] +(30) Filter [codegen id : 3] Input [10]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30, rk#31] Condition : (rk#31 <= 100) -(33) TakeOrderedAndProject +(31) TakeOrderedAndProject Input [10]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30, rk#31] Arguments: 100, [i_category#19 ASC NULLS FIRST, i_class#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, i_product_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_qoy#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, sumsales#30 ASC NULLS FIRST, rk#31 ASC NULLS FIRST], [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30, rk#31] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * ColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan parquet spark_catalog.default.date_dim (34) +BroadcastExchange (36) ++- * ColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan parquet spark_catalog.default.date_dim (32) -(34) Scan parquet spark_catalog.default.date_dim +(32) Scan parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(35) CometFilter +(33) CometFilter Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1200)) AND (d_month_seq#8 <= 1211)) AND isnotnull(d_date_sk#7)) -(36) CometProject +(34) CometProject Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -(37) ColumnarToRow [codegen id : 1] +(35) ColumnarToRow [codegen id : 1] Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -(38) BroadcastExchange +(36) BroadcastExchange Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/simplified.txt index 24aa5d08d..1f3b8c203 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q67/simplified.txt @@ -5,43 +5,41 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ Window [sumsales,i_category] WindowGroupLimit [i_category,sumsales] WholeStageCodegen (2) - ColumnarToRow + Sort [i_category,sumsales] InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometColumnarExchange [i_category] #1 - RowToColumnar - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,spark_grouping_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty,ss_sales_price,ss_quantity] - CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] - CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #5 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + Exchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,spark_grouping_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty,ss_sales_price,ss_quantity] + CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] + CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/explain.txt index c07b9bcb9..12a223631 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/explain.txt @@ -6,7 +6,7 @@ :- CometProject (33) : +- CometBroadcastHashJoin (32) : :- CometHashAggregate (28) - : : +- CometColumnarExchange (27) + : : +- CometExchange (27) : : +- CometHashAggregate (26) : : +- CometProject (25) : : +- CometBroadcastHashJoin (24) @@ -164,9 +164,9 @@ Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_pri Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] -(27) CometColumnarExchange +(27) CometExchange Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/simplified.txt index 426f7c489..568fd734b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q68/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometProject [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometHashAggregate [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,ss_addr_sk,ca_city,sum,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax))] - CometColumnarExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/explain.txt index 4991d569f..b931de292 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/explain.txt @@ -1,50 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (27) - : : +- * BroadcastHashJoin LeftAnti BuildRight (26) - : : :- * BroadcastHashJoin LeftAnti BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometProject (30) - : +- CometFilter (29) - : +- CometScan parquet spark_catalog.default.customer_address (28) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- Exchange (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (27) + : : +- * BroadcastHashJoin LeftAnti BuildRight (26) + : : :- * BroadcastHashJoin LeftAnti BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * ColumnarToRow (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan parquet spark_catalog.default.customer_address (28) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometFilter (36) + +- CometScan parquet spark_catalog.default.customer_demographics (35) (1) Scan parquet spark_catalog.default.customer @@ -240,56 +238,50 @@ Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#26] Results [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] -(42) RowToColumnar +(42) Exchange Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] +Arguments: hashpartitioning(cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) CometColumnarExchange -Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] -Arguments: hashpartitioning(cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 6] -Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] - -(45) HashAggregate [codegen id : 6] +(43) HashAggregate [codegen id : 6] Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] Keys [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#28] Results [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, count(1)#28 AS cnt1#29, cd_purchase_estimate#24, count(1)#28 AS cnt2#30, cd_credit_rating#25, count(1)#28 AS cnt3#31] -(46) TakeOrderedAndProject +(44) TakeOrderedAndProject Input [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#29, cd_purchase_estimate#24, cnt2#30, cd_credit_rating#25, cnt3#31] Arguments: 100, [cd_gender#21 ASC NULLS FIRST, cd_marital_status#22 ASC NULLS FIRST, cd_education_status#23 ASC NULLS FIRST, cd_purchase_estimate#24 ASC NULLS FIRST, cd_credit_rating#25 ASC NULLS FIRST], [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#29, cd_purchase_estimate#24, cnt2#30, cd_credit_rating#25, cnt3#31] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (51) -+- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.date_dim (47) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan parquet spark_catalog.default.date_dim (45) -(47) Scan parquet spark_catalog.default.date_dim +(45) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] ReadSchema: struct -(48) CometFilter +(46) CometFilter Input [3]: [d_date_sk#7, d_year#8, d_moy#9] Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2001)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 6)) AND isnotnull(d_date_sk#7)) -(49) CometProject +(47) CometProject Input [3]: [d_date_sk#7, d_year#8, d_moy#9] Arguments: [d_date_sk#7], [d_date_sk#7] -(50) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#7] -(51) BroadcastExchange +(49) BroadcastExchange Input [1]: [d_date_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt index 9e1e3edb6..c9cc4959b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q69/simplified.txt @@ -1,72 +1,70 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + InputAdapter + Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt index c8d74de3d..1f5e1338e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (30) +- CometTakeOrderedAndProject (29) +- CometHashAggregate (28) - +- CometColumnarExchange (27) + +- CometExchange (27) +- CometHashAggregate (26) +- CometProject (25) +- CometBroadcastHashJoin (24) @@ -156,9 +156,9 @@ Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -(27) CometColumnarExchange +(27) CometExchange Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] -Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/simplified.txt index 0725862ca..044813ad4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q7/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] CometHashAggregate [i_item_id,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/explain.txt index c144adc12..17e7163d2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/explain.txt @@ -1,53 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (49) -+- * Project (48) - +- Window (47) - +- * ColumnarToRow (46) - +- CometSort (45) - +- CometColumnarExchange (44) - +- RowToColumnar (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometColumnarExchange (40) - +- RowToColumnar (39) - +- * HashAggregate (38) - +- * Expand (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.store_sales (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.date_dim (3) - +- BroadcastExchange (34) - +- * BroadcastHashJoin LeftSemi BuildRight (33) - :- * ColumnarToRow (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.store (10) - +- BroadcastExchange (32) - +- * Project (31) - +- * Filter (30) - +- Window (29) - +- WindowGroupLimit (28) - +- * ColumnarToRow (27) - +- CometSort (26) - +- CometHashAggregate (25) - +- CometColumnarExchange (24) - +- CometHashAggregate (23) - +- CometProject (22) - +- CometBroadcastHashJoin (21) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.store_sales (13) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.store (15) - +- ReusedExchange (20) +TakeOrderedAndProject (45) ++- * Project (44) + +- Window (43) + +- * Sort (42) + +- Exchange (41) + +- * HashAggregate (40) + +- Exchange (39) + +- * HashAggregate (38) + +- * Expand (37) + +- * Project (36) + +- * BroadcastHashJoin Inner BuildRight (35) + :- * ColumnarToRow (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.date_dim (3) + +- BroadcastExchange (34) + +- * BroadcastHashJoin LeftSemi BuildRight (33) + :- * ColumnarToRow (12) + : +- CometFilter (11) + : +- CometScan parquet spark_catalog.default.store (10) + +- BroadcastExchange (32) + +- * Project (31) + +- * Filter (30) + +- Window (29) + +- WindowGroupLimit (28) + +- * ColumnarToRow (27) + +- CometSort (26) + +- CometHashAggregate (25) + +- CometExchange (24) + +- CometHashAggregate (23) + +- CometProject (22) + +- CometBroadcastHashJoin (21) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.store_sales (13) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometScan parquet spark_catalog.default.store (15) + +- ReusedExchange (20) (1) Scan parquet spark_catalog.default.store_sales @@ -160,9 +156,9 @@ Input [2]: [ss_net_profit#11, s_state#15] Keys [1]: [s_state#15] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] -(24) CometColumnarExchange +(24) CometExchange Input [2]: [s_state#15, sum#17] -Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (25) CometHashAggregate Input [2]: [s_state#15, sum#17] @@ -227,78 +223,66 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum#23] Results [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -(39) RowToColumnar +(39) Exchange Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(40) CometColumnarExchange -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(41) ColumnarToRow [codegen id : 5] -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] - -(42) HashAggregate [codegen id : 5] +(40) HashAggregate [codegen id : 5] Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS total_sum#26, s_state#20, s_county#21, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS lochierarchy#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS _w0#28, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) = 0) THEN s_state#20 END AS _w2#30] -(43) RowToColumnar -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] - -(44) CometColumnarExchange -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(45) CometSort +(41) Exchange Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30], [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST] +Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(46) ColumnarToRow [codegen id : 6] +(42) Sort [codegen id : 6] Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST], false, 0 -(47) Window +(43) Window Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] Arguments: [rank(_w0#28) windowspecdefinition(_w1#29, _w2#30, _w0#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#31], [_w1#29, _w2#30], [_w0#28 DESC NULLS LAST] -(48) Project [codegen id : 7] +(44) Project [codegen id : 7] Output [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] Input [8]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30, rank_within_parent#31] -(49) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN s_state#20 END ASC NULLS FIRST, rank_within_parent#31 ASC NULLS FIRST], [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (54) -+- * ColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.date_dim (50) +BroadcastExchange (50) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(50) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(51) CometFilter +(47) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) -(52) CometProject +(48) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(53) ColumnarToRow [codegen id : 1] +(49) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(54) BroadcastExchange +(50) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt index 95a617c5a..7da5aad4f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q70/simplified.txt @@ -4,71 +4,67 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (6) - ColumnarToRow + Sort [_w1,_w2,_w0] InputAdapter - CometSort [total_sum,s_state,s_county,lochierarchy,_w0,_w1,_w2] - CometColumnarExchange [_w1,_w2] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_state,s_county,spark_grouping_id] #2 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] + Exchange [_w1,_w2] #1 + WholeStageCodegen (5) + HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + InputAdapter + Exchange [s_state,s_county,spark_grouping_id] #2 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] + Expand [ss_net_profit,s_state,s_county] + Project [ss_net_profit,s_state,s_county] + BroadcastHashJoin [ss_store_sk,s_store_sk] + ColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (3) + BroadcastHashJoin [s_state,s_state] ColumnarToRow InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 + CometFilter [s_store_sk,s_county,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] WholeStageCodegen (1) ColumnarToRow InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [s_state,_w0] - CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] - CometColumnarExchange [s_state] #7 - CometHashAggregate [s_state,sum,ss_net_profit] - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #8 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #4 + CometSort [s_state,_w0] + CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #7 + CometHashAggregate [s_state,sum,ss_net_profit] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [s_store_sk,s_state] #8 + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt index 53691edc5..65ffab7e8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/explain.txt @@ -3,7 +3,7 @@ +- CometSort (36) +- CometColumnarExchange (35) +- CometHashAggregate (34) - +- CometColumnarExchange (33) + +- CometExchange (33) +- CometHashAggregate (32) +- CometProject (31) +- CometBroadcastHashJoin (30) @@ -192,9 +192,9 @@ Input [5]: [i_brand_id#2, i_brand#3, ext_price#13, t_hour#35, t_minute#36] Keys [4]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36] Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] -(33) CometColumnarExchange +(33) CometExchange Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#38] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (34) CometHashAggregate Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/simplified.txt index cd0700c28..88ca8b67d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q71/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometSort [brand_id,brand,t_hour,t_minute,ext_price] CometColumnarExchange [ext_price,brand_id] #1 CometHashAggregate [brand_id,brand,t_hour,t_minute,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ext_price))] - CometColumnarExchange [i_brand,i_brand_id,t_hour,t_minute] #2 + CometExchange [i_brand,i_brand_id,t_hour,t_minute] #2 CometHashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum,ext_price] CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/explain.txt index 2f6da7d32..f53b2f84e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/explain.txt @@ -1,72 +1,71 @@ == Physical Plan == -* ColumnarToRow (68) -+- CometTakeOrderedAndProject (67) - +- CometHashAggregate (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometProject (63) - +- CometSortMergeJoin (62) - :- CometSort (56) - : +- CometColumnarExchange (55) - : +- RowToColumnar (54) - : +- * Project (53) - : +- * BroadcastHashJoin LeftOuter BuildRight (52) - : :- * Project (47) - : : +- * BroadcastHashJoin Inner BuildRight (46) - : : :- * ColumnarToRow (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometProject (35) - : : : : +- CometBroadcastHashJoin (34) - : : : : :- CometProject (29) - : : : : : +- CometBroadcastHashJoin (28) - : : : : : :- CometProject (23) - : : : : : : +- CometBroadcastHashJoin (22) - : : : : : : :- CometProject (17) - : : : : : : : +- CometBroadcastHashJoin (16) - : : : : : : : :- CometProject (12) - : : : : : : : : +- CometBroadcastHashJoin (11) - : : : : : : : : :- CometProject (7) - : : : : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- CometBroadcastExchange (5) - : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) - : : : : : : : : +- CometBroadcastExchange (10) - : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) - : : : : : : : +- CometBroadcastExchange (15) - : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan parquet spark_catalog.default.item (13) - : : : : : : +- CometBroadcastExchange (21) - : : : : : : +- CometProject (20) - : : : : : : +- CometFilter (19) - : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) - : : : : : +- CometBroadcastExchange (27) - : : : : : +- CometProject (26) - : : : : : +- CometFilter (25) - : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) - : : : : +- CometBroadcastExchange (33) - : : : : +- CometProject (32) - : : : : +- CometFilter (31) - : : : : +- CometScan parquet spark_catalog.default.date_dim (30) - : : : +- CometBroadcastExchange (38) - : : : +- CometFilter (37) - : : : +- CometScan parquet spark_catalog.default.date_dim (36) - : : +- BroadcastExchange (45) - : : +- * ColumnarToRow (44) - : : +- CometFilter (43) - : : +- CometScan parquet spark_catalog.default.date_dim (42) - : +- BroadcastExchange (51) - : +- * ColumnarToRow (50) - : +- CometFilter (49) - : +- CometScan parquet spark_catalog.default.promotion (48) - +- CometSort (61) - +- CometColumnarExchange (60) - +- CometProject (59) - +- CometFilter (58) - +- CometScan parquet spark_catalog.default.catalog_returns (57) +TakeOrderedAndProject (67) ++- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * SortMergeJoin LeftOuter (62) + :- * Sort (55) + : +- Exchange (54) + : +- * Project (53) + : +- * BroadcastHashJoin LeftOuter BuildRight (52) + : :- * Project (47) + : : +- * BroadcastHashJoin Inner BuildRight (46) + : : :- * ColumnarToRow (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometScan parquet spark_catalog.default.date_dim (36) + : : +- BroadcastExchange (45) + : : +- * ColumnarToRow (44) + : : +- CometFilter (43) + : : +- CometScan parquet spark_catalog.default.date_dim (42) + : +- BroadcastExchange (51) + : +- * ColumnarToRow (50) + : +- CometFilter (49) + : +- CometScan parquet spark_catalog.default.promotion (48) + +- * ColumnarToRow (61) + +- CometSort (60) + +- CometExchange (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.catalog_returns (56) (1) Scan parquet spark_catalog.default.catalog_sales @@ -321,99 +320,101 @@ Join condition: None Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -(54) RowToColumnar +(54) Exchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(55) CometColumnarExchange +(55) Sort [codegen id : 4] Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 -(56) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] - -(57) Scan parquet spark_catalog.default.catalog_returns +(56) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(58) CometFilter +(57) CometFilter Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) -(59) CometProject +(58) CometProject Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] -(60) CometColumnarExchange +(59) CometExchange Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(61) CometSort +(60) CometSort Input [2]: [cr_item_sk#31, cr_order_number#32] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] -(62) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +(61) ColumnarToRow [codegen id : 5] +Input [2]: [cr_item_sk#31, cr_order_number#32] -(63) CometProject +(62) SortMergeJoin [codegen id : 6] +Left keys [2]: [cs_item_sk#4, cs_order_number#6] +Right keys [2]: [cr_item_sk#31, cr_order_number#32] +Join type: LeftOuter +Join condition: None + +(63) Project [codegen id : 6] +Output [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(64) CometHashAggregate +(64) HashAggregate [codegen id : 6] Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#34] +Results [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] -(65) CometColumnarExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +(65) Exchange +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] +Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(66) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] +(66) HashAggregate [codegen id : 7] +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#36] +Results [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count(1)#36 AS no_promo#37, count(1)#36 AS promo#38, count(1)#36 AS total_cnt#39] -(67) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -(68) ColumnarToRow [codegen id : 4] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +(67) TakeOrderedAndProject +Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] +Arguments: 100, [total_cnt#39 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (73) -+- * ColumnarToRow (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan parquet spark_catalog.default.date_dim (69) +BroadcastExchange (72) ++- * ColumnarToRow (71) + +- CometProject (70) + +- CometFilter (69) + +- CometScan parquet spark_catalog.default.date_dim (68) -(69) Scan parquet spark_catalog.default.date_dim +(68) Scan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(70) CometFilter +(69) CometFilter Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 1999)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) -(71) CometProject +(70) CometProject Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] -(72) ColumnarToRow [codegen id : 1] +(71) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -(73) BroadcastExchange +(72) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt index 1022e1a90..1efc2e5d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q72/simplified.txt @@ -1,86 +1,92 @@ -WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] - CometColumnarExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] - CometProject [w_warehouse_name,i_item_desc,d_week_seq] - CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometColumnarExchange [cs_item_sk,cs_order_number] #2 - RowToColumnar - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - ColumnarToRow - InputAdapter - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 - CometFilter [d_date_sk,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] +TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] + WholeStageCodegen (7) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] + InputAdapter + Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + WholeStageCodegen (6) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] + Project [w_warehouse_name,i_item_desc,d_week_seq] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter + WholeStageCodegen (4) + Sort [cs_item_sk,cs_order_number] + InputAdapter + Exchange [cs_item_sk,cs_order_number] #2 + WholeStageCodegen (3) + Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_promo_sk,p_promo_sk] + Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] + ColumnarToRow + InputAdapter + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #7 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #8 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometFilter [d_date_sk,d_week_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #11 - WholeStageCodegen (1) + BroadcastExchange #12 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] - CometSort [cr_item_sk,cr_order_number] - CometColumnarExchange [cr_item_sk,cr_order_number] #13 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometFilter [p_promo_sk] + CometScan parquet spark_catalog.default.promotion [p_promo_sk] + InputAdapter + WholeStageCodegen (5) + ColumnarToRow + InputAdapter + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #13 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt index 6ea426978..8b4e36c96 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/explain.txt @@ -6,7 +6,7 @@ +- CometBroadcastHashJoin (28) :- CometFilter (24) : +- CometHashAggregate (23) - : +- CometColumnarExchange (22) + : +- CometExchange (22) : +- CometHashAggregate (21) : +- CometProject (20) : +- CometBroadcastHashJoin (19) @@ -134,9 +134,9 @@ Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt index 5fb8a197a..441736532 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q73/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometColumnarExchange [ss_ticket_number,ss_customer_sk] #2 + CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt index ecc0fd1ca..b23b0b48c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/explain.txt @@ -8,7 +8,7 @@ : :- CometBroadcastHashJoin (33) : : :- CometFilter (16) : : : +- CometHashAggregate (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometHashAggregate (13) : : : +- CometProject (12) : : : +- CometBroadcastHashJoin (11) @@ -24,7 +24,7 @@ : : : +- CometScan parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometProject (28) : : +- CometBroadcastHashJoin (27) @@ -41,7 +41,7 @@ : +- CometBroadcastExchange (48) : +- CometFilter (47) : +- CometHashAggregate (46) - : +- CometColumnarExchange (45) + : +- CometExchange (45) : +- CometHashAggregate (44) : +- CometProject (43) : +- CometBroadcastHashJoin (42) @@ -55,7 +55,7 @@ : +- ReusedExchange (41) +- CometBroadcastExchange (64) +- CometHashAggregate (63) - +- CometColumnarExchange (62) + +- CometExchange (62) +- CometHashAggregate (61) +- CometProject (60) +- CometBroadcastHashJoin (59) @@ -134,9 +134,9 @@ Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_yea Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -(14) CometColumnarExchange +(14) CometExchange Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#11] @@ -212,9 +212,9 @@ Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, ss_net_paid#19, d Keys [4]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#19))] -(30) CometColumnarExchange +(30) CometExchange Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, sum#24] -Arguments: hashpartitioning(c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, sum#24] @@ -283,9 +283,9 @@ Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ws_net_paid#34, d Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#34))] -(45) CometColumnarExchange +(45) CometExchange Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, sum#39] -Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometHashAggregate Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, sum#39] @@ -362,9 +362,9 @@ Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, ws_net_paid#47, d Keys [4]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#47))] -(62) CometColumnarExchange +(62) CometExchange Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, sum#52] -Arguments: hashpartitioning(c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (63) CometHashAggregate Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, sum#52] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt index 0b288df94..631a82f1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q74/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -32,7 +32,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -56,7 +56,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -71,7 +71,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt index ca9a5cd59..76b1cac0c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/explain.txt @@ -4,19 +4,19 @@ TakeOrderedAndProject (125) +- * SortMergeJoin Inner (123) :- * ColumnarToRow (67) : +- CometSort (66) - : +- CometColumnarExchange (65) + : +- CometExchange (65) : +- CometFilter (64) : +- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometHashAggregate (60) - : +- CometColumnarExchange (59) + : +- CometExchange (59) : +- CometHashAggregate (58) : +- CometUnion (57) : :- CometProject (22) : : +- CometSortMergeJoin (21) : : :- CometSort (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometProject (13) : : : +- CometBroadcastHashJoin (12) : : : :- CometProject (8) @@ -31,14 +31,14 @@ TakeOrderedAndProject (125) : : : +- CometFilter (10) : : : +- CometScan parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) : : +- CometScan parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) - : : : +- CometColumnarExchange (31) + : : : +- CometExchange (31) : : : +- CometProject (30) : : : +- CometBroadcastHashJoin (29) : : : :- CometProject (27) @@ -48,14 +48,14 @@ TakeOrderedAndProject (125) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) - : : +- CometColumnarExchange (36) + : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) : : +- CometScan parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) - : : +- CometColumnarExchange (48) + : : +- CometExchange (48) : : +- CometProject (47) : : +- CometBroadcastHashJoin (46) : : :- CometProject (44) @@ -65,25 +65,25 @@ TakeOrderedAndProject (125) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) - : +- CometColumnarExchange (53) + : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) : +- CometScan parquet spark_catalog.default.web_returns (50) +- * ColumnarToRow (122) +- CometSort (121) - +- CometColumnarExchange (120) + +- CometExchange (120) +- CometFilter (119) +- CometHashAggregate (118) - +- CometColumnarExchange (117) + +- CometExchange (117) +- CometHashAggregate (116) +- CometHashAggregate (115) - +- CometColumnarExchange (114) + +- CometExchange (114) +- CometHashAggregate (113) +- CometUnion (112) :- CometProject (83) : +- CometSortMergeJoin (82) : :- CometSort (79) - : : +- CometColumnarExchange (78) + : : +- CometExchange (78) : : +- CometProject (77) : : +- CometBroadcastHashJoin (76) : : :- CometProject (72) @@ -99,7 +99,7 @@ TakeOrderedAndProject (125) :- CometProject (97) : +- CometSortMergeJoin (96) : :- CometSort (93) - : : +- CometColumnarExchange (92) + : : +- CometExchange (92) : : +- CometProject (91) : : +- CometBroadcastHashJoin (90) : : :- CometProject (88) @@ -113,7 +113,7 @@ TakeOrderedAndProject (125) +- CometProject (111) +- CometSortMergeJoin (110) :- CometSort (107) - : +- CometColumnarExchange (106) + : +- CometExchange (106) : +- CometProject (105) : +- CometBroadcastHashJoin (104) : :- CometProject (102) @@ -190,9 +190,9 @@ Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -(14) CometColumnarExchange +(14) CometExchange Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] @@ -213,9 +213,9 @@ Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -(19) CometColumnarExchange +(19) CometExchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -266,9 +266,9 @@ Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -(31) CometColumnarExchange +(31) CometExchange Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] @@ -289,9 +289,9 @@ Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -(36) CometColumnarExchange +(36) CometExchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -342,9 +342,9 @@ Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -(48) CometColumnarExchange +(48) CometExchange Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] @@ -365,9 +365,9 @@ Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -(53) CometColumnarExchange +(53) CometExchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -392,9 +392,9 @@ Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_ Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] Functions: [] -(59) CometColumnarExchange +(59) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] @@ -406,9 +406,9 @@ Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_ Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -(62) CometColumnarExchange +(62) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (63) CometHashAggregate Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] @@ -419,9 +419,9 @@ Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] Condition : isnotnull(sales_cnt#64) -(65) CometColumnarExchange +(65) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (66) CometSort Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] @@ -478,9 +478,9 @@ Arguments: [cs_sold_date_sk#70], [d_date_sk#77], Inner, BuildRight Input [11]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -(78) CometColumnarExchange +(78) CometExchange Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (79) CometSort Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] @@ -538,9 +538,9 @@ Arguments: [ss_sold_date_sk#87], [d_date_sk#94], Inner, BuildRight Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_date_sk#94, d_year#95] Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -(92) CometColumnarExchange +(92) CometExchange Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (93) CometSort Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] @@ -598,9 +598,9 @@ Arguments: [ws_sold_date_sk#104], [d_date_sk#111], Inner, BuildRight Input [11]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -(106) CometColumnarExchange +(106) CometExchange Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (107) CometSort Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] @@ -632,9 +632,9 @@ Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufac Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] Functions: [] -(114) CometColumnarExchange +(114) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (115) CometHashAggregate Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] @@ -646,9 +646,9 @@ Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufac Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -(117) CometColumnarExchange +(117) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] +Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (118) CometHashAggregate Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] @@ -659,9 +659,9 @@ Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] Condition : isnotnull(sales_cnt#118) -(120) CometColumnarExchange +(120) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] +Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (121) CometSort Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/simplified.txt index a942e8071..4ae82c2cf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q75/simplified.txt @@ -7,19 +7,19 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #4 + CometExchange [cs_order_number,cs_item_sk] #4 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -41,14 +41,14 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometColumnarExchange [cr_order_number,cr_item_sk] #8 + CometExchange [cr_order_number,cr_item_sk] #8 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #9 + CometExchange [ss_ticket_number,ss_item_sk] #9 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -59,14 +59,14 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 ReusedExchange [d_date_sk,d_year] #7 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #10 + CometExchange [sr_ticket_number,sr_item_sk] #10 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #11 + CometExchange [ws_order_number,ws_item_sk] #11 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -77,7 +77,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 ReusedExchange [d_date_sk,d_year] #7 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometColumnarExchange [wr_order_number,wr_item_sk] #12 + CometExchange [wr_order_number,wr_item_sk] #12 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] @@ -86,19 +86,19 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #16 + CometExchange [cs_order_number,cs_item_sk] #16 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -121,7 +121,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #19 + CometExchange [ss_ticket_number,ss_item_sk] #19 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -136,7 +136,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #20 + CometExchange [ws_order_number,ws_item_sk] #20 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/explain.txt index d1e7938f7..b60b7b3a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (34) +- CometTakeOrderedAndProject (33) +- CometHashAggregate (32) - +- CometColumnarExchange (31) + +- CometExchange (31) +- CometHashAggregate (30) +- CometUnion (29) :- CometProject (12) @@ -177,9 +177,9 @@ Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_ Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12))] -(31) CometColumnarExchange +(31) CometExchange Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#37, sum#38] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#37, sum#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/simplified.txt index a3a571fa1..eb3e14d5e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q76/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] CometHashAggregate [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt,count,sum,count(1),sum(UnscaledValue(ext_sales_price))] - CometColumnarExchange [channel,col_name,d_year,d_qoy,i_category] #1 + CometExchange [channel,col_name,d_year,d_qoy,i_category] #1 CometHashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum,ext_sales_price] CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/explain.txt index 2c2016542..b160070d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/explain.txt @@ -1,90 +1,88 @@ == Physical Plan == -TakeOrderedAndProject (86) -+- * HashAggregate (85) - +- * ColumnarToRow (84) - +- CometColumnarExchange (83) - +- RowToColumnar (82) - +- * HashAggregate (81) - +- * Expand (80) - +- Union (79) - :- * ColumnarToRow (31) - : +- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometHashAggregate (16) - : : +- CometColumnarExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) - : +- CometBroadcastExchange (28) - : +- CometHashAggregate (27) - : +- CometColumnarExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (21) - : : +- CometBroadcastHashJoin (20) - : : :- CometFilter (18) - : : : +- CometScan parquet spark_catalog.default.store_returns (17) - : : +- ReusedExchange (19) - : +- ReusedExchange (22) - :- * Project (50) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) - : :- BroadcastExchange (40) - : : +- * ColumnarToRow (39) - : : +- CometHashAggregate (38) - : : +- CometColumnarExchange (37) - : : +- CometHashAggregate (36) - : : +- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometScan parquet spark_catalog.default.catalog_sales (32) - : : +- ReusedExchange (33) - : +- * ColumnarToRow (48) - : +- CometHashAggregate (47) - : +- CometColumnarExchange (46) - : +- CometHashAggregate (45) - : +- CometProject (44) - : +- CometBroadcastHashJoin (43) - : :- CometScan parquet spark_catalog.default.catalog_returns (41) - : +- ReusedExchange (42) - +- * ColumnarToRow (78) - +- CometProject (77) - +- CometBroadcastHashJoin (76) - :- CometHashAggregate (63) - : +- CometColumnarExchange (62) - : +- CometHashAggregate (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometFilter (52) - : : : +- CometScan parquet spark_catalog.default.web_sales (51) - : : +- ReusedExchange (53) - : +- CometBroadcastExchange (58) - : +- CometFilter (57) - : +- CometScan parquet spark_catalog.default.web_page (56) - +- CometBroadcastExchange (75) - +- CometHashAggregate (74) - +- CometColumnarExchange (73) - +- CometHashAggregate (72) - +- CometProject (71) - +- CometBroadcastHashJoin (70) - :- CometProject (68) - : +- CometBroadcastHashJoin (67) - : :- CometFilter (65) - : : +- CometScan parquet spark_catalog.default.web_returns (64) - : +- ReusedExchange (66) - +- ReusedExchange (69) +TakeOrderedAndProject (84) ++- * HashAggregate (83) + +- Exchange (82) + +- * HashAggregate (81) + +- * Expand (80) + +- Union (79) + :- * ColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometHashAggregate (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (28) + : +- CometHashAggregate (27) + : +- CometExchange (26) + : +- CometHashAggregate (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (21) + : : +- CometBroadcastHashJoin (20) + : : :- CometFilter (18) + : : : +- CometScan parquet spark_catalog.default.store_returns (17) + : : +- ReusedExchange (19) + : +- ReusedExchange (22) + :- * Project (50) + : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : :- BroadcastExchange (40) + : : +- * ColumnarToRow (39) + : : +- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometScan parquet spark_catalog.default.catalog_sales (32) + : : +- ReusedExchange (33) + : +- * ColumnarToRow (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometScan parquet spark_catalog.default.catalog_returns (41) + : +- ReusedExchange (42) + +- * ColumnarToRow (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometFilter (52) + : : : +- CometScan parquet spark_catalog.default.web_sales (51) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometScan parquet spark_catalog.default.web_page (56) + +- CometBroadcastExchange (75) + +- CometHashAggregate (74) + +- CometExchange (73) + +- CometHashAggregate (72) + +- CometProject (71) + +- CometBroadcastHashJoin (70) + :- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometFilter (65) + : : +- CometScan parquet spark_catalog.default.web_returns (64) + : +- ReusedExchange (66) + +- ReusedExchange (69) (1) Scan parquet spark_catalog.default.store_sales @@ -156,9 +154,9 @@ Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] Keys [1]: [s_store_sk#8] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [s_store_sk#8, sum#9, sum#10] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [s_store_sk#8, sum#9, sum#10] @@ -206,9 +204,9 @@ Input [3]: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] Keys [1]: [s_store_sk#17] Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#12)), partial_sum(UnscaledValue(sr_net_loss#13))] -(26) CometColumnarExchange +(26) CometExchange Input [3]: [s_store_sk#17, sum#18, sum#19] -Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate Input [3]: [s_store_sk#17, sum#18, sum#19] @@ -255,9 +253,9 @@ Input [3]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] Keys [1]: [cs_call_center_sk#28] Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#29)), partial_sum(UnscaledValue(cs_net_profit#30))] -(37) CometColumnarExchange +(37) CometExchange Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometHashAggregate Input [3]: [cs_call_center_sk#28, sum#34, sum#35] @@ -295,9 +293,9 @@ Input [2]: [cr_return_amount#38, cr_net_loss#39] Keys: [] Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#38)), partial_sum(UnscaledValue(cr_net_loss#39))] -(46) CometColumnarExchange +(46) CometExchange Input [2]: [sum#43, sum#44] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (47) CometHashAggregate Input [2]: [sum#43, sum#44] @@ -368,9 +366,9 @@ Input [3]: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] Keys [1]: [wp_web_page_sk#56] Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#51)), partial_sum(UnscaledValue(ws_net_profit#52))] -(62) CometColumnarExchange +(62) CometExchange Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (63) CometHashAggregate Input [3]: [wp_web_page_sk#56, sum#57, sum#58] @@ -418,9 +416,9 @@ Input [3]: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] Keys [1]: [wp_web_page_sk#65] Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#60)), partial_sum(UnscaledValue(wr_net_loss#61))] -(73) CometColumnarExchange +(73) CometExchange Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (74) CometHashAggregate Input [3]: [wp_web_page_sk#65, sum#66, sum#67] @@ -456,56 +454,50 @@ Functions [3]: [partial_sum(sales#22), partial_sum(returns#24), partial_sum(prof Aggregate Attributes [6]: [sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] Results [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] -(82) RowToColumnar +(82) Exchange Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] +Arguments: hashpartitioning(channel#76, id#77, spark_grouping_id#78, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(83) CometColumnarExchange -Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] -Arguments: hashpartitioning(channel#76, id#77, spark_grouping_id#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(84) ColumnarToRow [codegen id : 6] -Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] - -(85) HashAggregate [codegen id : 6] +(83) HashAggregate [codegen id : 6] Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] Keys [3]: [channel#76, id#77, spark_grouping_id#78] Functions [3]: [sum(sales#22), sum(returns#24), sum(profit#25)] Aggregate Attributes [3]: [sum(sales#22)#91, sum(returns#24)#92, sum(profit#25)#93] Results [5]: [channel#76, id#77, sum(sales#22)#91 AS sales#94, sum(returns#24)#92 AS returns#95, sum(profit#25)#93 AS profit#96] -(86) TakeOrderedAndProject +(84) TakeOrderedAndProject Input [5]: [channel#76, id#77, sales#94, returns#95, profit#96] Arguments: 100, [channel#76 ASC NULLS FIRST, id#77 ASC NULLS FIRST], [channel#76, id#77, sales#94, returns#95, profit#96] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (91) -+- * ColumnarToRow (90) - +- CometProject (89) - +- CometFilter (88) - +- CometScan parquet spark_catalog.default.date_dim (87) +BroadcastExchange (89) ++- * ColumnarToRow (88) + +- CometProject (87) + +- CometFilter (86) + +- CometScan parquet spark_catalog.default.date_dim (85) -(87) Scan parquet spark_catalog.default.date_dim +(85) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_date#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] ReadSchema: struct -(88) CometFilter +(86) CometFilter Input [2]: [d_date_sk#6, d_date#7] Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6)) -(89) CometProject +(87) CometProject Input [2]: [d_date_sk#6, d_date#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(90) ColumnarToRow [codegen id : 1] +(88) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(91) BroadcastExchange +(89) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/simplified.txt index 605180016..e8226ab1f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q77/simplified.txt @@ -1,112 +1,110 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (6) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) + InputAdapter + Exchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (5) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [returns,profit,profit_loss,s_store_sk] [sales,returns,profit,channel,id] + CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] + CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_sk] #2 + CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 + CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] + CometExchange [s_store_sk] #7 + CometHashAggregate [s_store_sk,sum,sum,sr_return_amt,sr_net_loss] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 + WholeStageCodegen (3) + Project [sales,returns,profit,profit_loss,cs_call_center_sk] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] + CometExchange [cs_call_center_sk] #9 + CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 ColumnarToRow InputAdapter - CometProject [returns,profit,profit_loss,s_store_sk] [sales,returns,profit,channel,id] - CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] - CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] - CometColumnarExchange [s_store_sk] #2 - CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 - CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - CometColumnarExchange [s_store_sk] #7 - CometHashAggregate [s_store_sk,sum,sum,sr_return_amt,sr_net_loss] - CometProject [sr_return_amt,sr_net_loss,s_store_sk] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] - CometProject [sr_store_sk,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [s_store_sk] #5 - WholeStageCodegen (3) - Project [sales,returns,profit,profit_loss,cs_call_center_sk] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometColumnarExchange [cs_call_center_sk] #9 - CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ColumnarToRow - InputAdapter - CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometColumnarExchange #10 - CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 + CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] + CometExchange #10 + CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] + CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] + CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] + CometExchange [wp_web_page_sk] #11 + CometHashAggregate [wp_web_page_sk,sum,sum,ws_ext_sales_price,ws_net_profit] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] - CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] - CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - CometColumnarExchange [wp_web_page_sk] #11 - CometHashAggregate [wp_web_page_sk,sum,sum,ws_ext_sales_price,ws_net_profit] - CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] - CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [wp_web_page_sk] #12 - CometFilter [wp_web_page_sk] - CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 - CometHashAggregate [wp_web_page_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] - CometColumnarExchange [wp_web_page_sk] #14 - CometHashAggregate [wp_web_page_sk,sum,sum,wr_return_amt,wr_net_loss] - CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] - CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] - CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [wp_web_page_sk] #12 + CometBroadcastExchange [wp_web_page_sk] #12 + CometFilter [wp_web_page_sk] + CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 + CometHashAggregate [wp_web_page_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] + CometExchange [wp_web_page_sk] #14 + CometHashAggregate [wp_web_page_sk,sum,sum,wr_return_amt,wr_net_loss] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/explain.txt index 2ffeb0737..acd754b08 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q78/explain.txt @@ -7,7 +7,7 @@ TakeOrderedAndProject (67) : +- CometSortMergeJoin (42) : :- CometSort (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -15,11 +15,11 @@ TakeOrderedAndProject (67) : : : +- CometFilter (11) : : : +- CometSortMergeJoin (10) : : : :- CometSort (4) - : : : : +- CometColumnarExchange (3) + : : : : +- CometExchange (3) : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- CometSort (9) - : : : +- CometColumnarExchange (8) + : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) : : : +- CometScan parquet spark_catalog.default.store_returns (5) @@ -29,7 +29,7 @@ TakeOrderedAndProject (67) : +- CometSort (41) : +- CometFilter (40) : +- CometHashAggregate (39) - : +- CometColumnarExchange (38) + : +- CometExchange (38) : +- CometHashAggregate (37) : +- CometProject (36) : +- CometBroadcastHashJoin (35) @@ -37,11 +37,11 @@ TakeOrderedAndProject (67) : : +- CometFilter (32) : : +- CometSortMergeJoin (31) : : :- CometSort (25) - : : : +- CometColumnarExchange (24) + : : : +- CometExchange (24) : : : +- CometFilter (23) : : : +- CometScan parquet spark_catalog.default.web_sales (22) : : +- CometSort (30) - : : +- CometColumnarExchange (29) + : : +- CometExchange (29) : : +- CometProject (28) : : +- CometFilter (27) : : +- CometScan parquet spark_catalog.default.web_returns (26) @@ -49,7 +49,7 @@ TakeOrderedAndProject (67) +- CometSort (63) +- CometFilter (62) +- CometHashAggregate (61) - +- CometColumnarExchange (60) + +- CometExchange (60) +- CometHashAggregate (59) +- CometProject (58) +- CometBroadcastHashJoin (57) @@ -57,11 +57,11 @@ TakeOrderedAndProject (67) : +- CometFilter (54) : +- CometSortMergeJoin (53) : :- CometSort (47) - : : +- CometColumnarExchange (46) + : : +- CometExchange (46) : : +- CometFilter (45) : : +- CometScan parquet spark_catalog.default.catalog_sales (44) : +- CometSort (52) - : +- CometColumnarExchange (51) + : +- CometExchange (51) : +- CometProject (50) : +- CometFilter (49) : +- CometScan parquet spark_catalog.default.catalog_returns (48) @@ -80,9 +80,9 @@ ReadSchema: struct -(106) CometFilter +(104) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2000-08-23)) AND (d_date#15 <= 2000-09-22)) AND isnotnull(d_date_sk#14)) -(107) CometProject +(105) CometProject Input [2]: [d_date_sk#14, d_date#15] Arguments: [d_date_sk#14], [d_date_sk#14] -(108) ColumnarToRow [codegen id : 1] +(106) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(109) BroadcastExchange +(107) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt index dfcc35f6e..012c711bf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q80/simplified.txt @@ -1,124 +1,122 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (5) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_id] #2 - CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk,ss_ticket_number] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometColumnarExchange [sr_item_sk,sr_ticket_number] #5 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #7 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #8 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #9 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [cp_catalog_page_id] #10 - CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometColumnarExchange [cs_item_sk,cs_order_number] #11 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometColumnarExchange [cr_item_sk,cr_order_number] #12 - CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [web_site_id] #14 - CometHashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] - CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] - CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] - CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometColumnarExchange [ws_item_sk,ws_order_number] #15 - CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometColumnarExchange [wr_item_sk,wr_order_number] #16 - CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [web_site_sk,web_site_id] #17 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 + InputAdapter + Exchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #5 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #8 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #9 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #10 + CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #11 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [web_site_id] #14 + CometHashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #15 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #16 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt index 2223ccb22..e72c5b017 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/explain.txt @@ -9,7 +9,7 @@ : : +- CometBroadcastHashJoin (34) : : :- CometFilter (17) : : : +- CometHashAggregate (16) - : : : +- CometColumnarExchange (15) + : : : +- CometExchange (15) : : : +- CometHashAggregate (14) : : : +- CometProject (13) : : : +- CometBroadcastHashJoin (12) @@ -27,10 +27,10 @@ : : +- CometBroadcastExchange (33) : : +- CometFilter (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometHashAggregate (28) - : : +- CometColumnarExchange (27) + : : +- CometExchange (27) : : +- CometHashAggregate (26) : : +- CometProject (25) : : +- CometBroadcastHashJoin (24) @@ -117,9 +117,9 @@ Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] @@ -171,9 +171,9 @@ Input [3]: [cr_returning_customer_sk#14, cr_return_amt_inc_tax#16, ca_state#21] Keys [2]: [cr_returning_customer_sk#14, ca_state#21] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#16))] -(27) CometColumnarExchange +(27) CometExchange Input [3]: [cr_returning_customer_sk#14, ca_state#21, sum#22] -Arguments: hashpartitioning(cr_returning_customer_sk#14, ca_state#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cr_returning_customer_sk#14, ca_state#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [3]: [cr_returning_customer_sk#14, ca_state#21, sum#22] @@ -185,9 +185,9 @@ Input [2]: [ctr_state#23, ctr_total_return#24] Keys [1]: [ctr_state#23] Functions [1]: [partial_avg(ctr_total_return#24)] -(30) CometColumnarExchange +(30) CometExchange Input [3]: [ctr_state#23, sum#25, count#26] -Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ctr_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (31) CometHashAggregate Input [3]: [ctr_state#23, sum#25, count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/simplified.txt index feaeefa9a..b35b76210 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q81/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] CometHashAggregate [ctr_customer_sk,ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(cr_return_amt_inc_tax))] - CometColumnarExchange [cr_returning_customer_sk,ca_state] #1 + CometExchange [cr_returning_customer_sk,ca_state] #1 CometHashAggregate [cr_returning_customer_sk,ca_state,sum,cr_return_amt_inc_tax] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] @@ -36,10 +36,10 @@ WholeStageCodegen (1) CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #5 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_state,sum,count,avg(ctr_total_return)] - CometColumnarExchange [ctr_state] #6 + CometExchange [ctr_state] #6 CometHashAggregate [ctr_state,sum,count,ctr_total_return] CometHashAggregate [ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(cr_return_amt_inc_tax))] - CometColumnarExchange [cr_returning_customer_sk,ca_state] #7 + CometExchange [cr_returning_customer_sk,ca_state] #7 CometHashAggregate [cr_returning_customer_sk,ca_state,sum,cr_return_amt_inc_tax] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/explain.txt index 2519fbc97..774b6b112 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometBroadcastHashJoin (20) @@ -132,9 +132,9 @@ Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Functions: [] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt index 869073de1..42c3f40e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q82/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometColumnarExchange [i_item_id,i_item_desc,i_current_price] #1 + CometExchange [i_item_id,i_item_desc,i_current_price] #1 CometHashAggregate [i_item_id,i_item_desc,i_current_price] CometProject [i_item_id,i_item_desc,i_current_price] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,ss_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt index 972a58976..ffd8a5eb1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/explain.txt @@ -6,7 +6,7 @@ :- CometProject (32) : +- CometBroadcastHashJoin (31) : :- CometHashAggregate (18) - : : +- CometColumnarExchange (17) + : : +- CometExchange (17) : : +- CometHashAggregate (16) : : +- CometProject (15) : : +- CometBroadcastHashJoin (14) @@ -25,7 +25,7 @@ : : +- ReusedExchange (10) : +- CometBroadcastExchange (30) : +- CometHashAggregate (29) - : +- CometColumnarExchange (28) + : +- CometExchange (28) : +- CometHashAggregate (27) : +- CometProject (26) : +- CometBroadcastHashJoin (25) @@ -37,7 +37,7 @@ : +- ReusedExchange (24) +- CometBroadcastExchange (44) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometProject (40) +- CometBroadcastHashJoin (39) @@ -126,9 +126,9 @@ Input [2]: [sr_return_quantity#2, i_item_id#6] Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(sr_return_quantity#2)] -(17) CometColumnarExchange +(17) CometExchange Input [2]: [i_item_id#6, sum#10] -Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [2]: [i_item_id#6, sum#10] @@ -176,9 +176,9 @@ Input [2]: [cr_return_quantity#12, i_item_id#16] Keys [1]: [i_item_id#16] Functions [1]: [partial_sum(cr_return_quantity#12)] -(28) CometColumnarExchange +(28) CometExchange Input [2]: [i_item_id#16, sum#18] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate Input [2]: [i_item_id#16, sum#18] @@ -239,9 +239,9 @@ Input [2]: [wr_return_quantity#24, i_item_id#28] Keys [1]: [i_item_id#28] Functions [1]: [partial_sum(wr_return_quantity#24)] -(42) CometColumnarExchange +(42) CometExchange Input [2]: [i_item_id#28, sum#30] -Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (43) CometHashAggregate Input [2]: [i_item_id#28, sum#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt index 8365f11a4..77345d28a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q83/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometProject [item_id,sr_item_qty,cr_item_qty] CometBroadcastHashJoin [item_id,sr_item_qty,item_id,cr_item_qty] CometHashAggregate [item_id,sr_item_qty,i_item_id,sum,sum(sr_return_quantity)] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,sr_return_quantity] CometProject [sr_return_quantity,i_item_id] CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] @@ -43,7 +43,7 @@ WholeStageCodegen (1) ReusedExchange [d_date] #3 CometBroadcastExchange [item_id,cr_item_qty] #7 CometHashAggregate [item_id,cr_item_qty,i_item_id,sum,sum(cr_return_quantity)] - CometColumnarExchange [i_item_id] #8 + CometExchange [i_item_id] #8 CometHashAggregate [i_item_id,sum,cr_return_quantity] CometProject [cr_return_quantity,i_item_id] CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] @@ -56,7 +56,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk] #6 CometBroadcastExchange [item_id,wr_item_qty] #9 CometHashAggregate [item_id,wr_item_qty,i_item_id,sum,sum(wr_return_quantity)] - CometColumnarExchange [i_item_id] #10 + CometExchange [i_item_id] #10 CometHashAggregate [i_item_id,sum,wr_return_quantity] CometProject [wr_return_quantity,i_item_id] CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt index 968c7402e..07aa05d94 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (45) +- CometTakeOrderedAndProject (44) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometProject (40) +- CometBroadcastHashJoin (39) @@ -243,9 +243,9 @@ Input [4]: [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#31] Keys [1]: [r_reason_desc#31] Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#16)), partial_avg(UnscaledValue(wr_fee#15))] -(42) CometColumnarExchange +(42) CometExchange Input [7]: [r_reason_desc#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Arguments: hashpartitioning(r_reason_desc#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(r_reason_desc#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (43) CometHashAggregate Input [7]: [r_reason_desc#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt index 9467b5c77..4e4ece1bd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q85/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] CometHashAggregate [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,sum,count,sum,count,sum,count,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] - CometColumnarExchange [r_reason_desc] #1 + CometExchange [r_reason_desc] #1 CometHashAggregate [r_reason_desc,sum,count,sum,count,sum,count,ws_quantity,wr_refunded_cash,wr_fee] CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt index 3d0fe32c7..675e6e931 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (23) +- Window (21) +- * ColumnarToRow (20) +- CometSort (19) - +- CometColumnarExchange (18) + +- CometExchange (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometExpand (14) +- CometProject (13) @@ -97,18 +97,18 @@ Input [4]: [ws_net_paid#2, i_category#10, i_class#11, spark_grouping_id#12] Keys [3]: [i_category#10, i_class#11, spark_grouping_id#12] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [i_category#10, i_class#11, spark_grouping_id#12, sum#13] -Arguments: hashpartitioning(i_category#10, i_class#11, spark_grouping_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#10, i_class#11, spark_grouping_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [i_category#10, i_class#11, spark_grouping_id#12, sum#13] Keys [3]: [i_category#10, i_class#11, spark_grouping_id#12] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -(18) CometColumnarExchange +(18) CometExchange Input [7]: [total_sum#14, i_category#10, i_class#11, lochierarchy#15, _w0#16, _w1#17, _w2#18] -Arguments: hashpartitioning(_w1#17, _w2#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(_w1#17, _w2#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort Input [7]: [total_sum#14, i_category#10, i_class#11, lochierarchy#15, _w0#16, _w1#17, _w2#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt index bbe4a2d2f..86480d58c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q86/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl ColumnarToRow InputAdapter CometSort [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2] - CometColumnarExchange [_w1,_w2] #1 + CometExchange [_w1,_w2] #1 CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2,spark_grouping_id,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [i_category,i_class,spark_grouping_id] #2 + CometExchange [i_category,i_class,spark_grouping_id] #2 CometHashAggregate [i_category,i_class,spark_grouping_id,sum,ws_net_paid] CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] CometProject [ws_net_paid,i_category,i_class] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt index d7ea7098e..d023b5b31 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/explain.txt @@ -1,55 +1,53 @@ == Physical Plan == -* HashAggregate (51) -+- * ColumnarToRow (50) - +- CometColumnarExchange (49) - +- RowToColumnar (48) - +- * HashAggregate (47) - +- * Project (46) - +- * BroadcastHashJoin LeftAnti BuildRight (45) - :- * BroadcastHashJoin LeftAnti BuildRight (31) - : :- * ColumnarToRow (17) - : : +- CometHashAggregate (16) - : : +- CometColumnarExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer (9) - : +- BroadcastExchange (30) - : +- * ColumnarToRow (29) - : +- CometHashAggregate (28) - : +- CometColumnarExchange (27) - : +- CometHashAggregate (26) - : +- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometFilter (19) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) - : : +- ReusedExchange (20) - : +- ReusedExchange (23) - +- BroadcastExchange (44) - +- * ColumnarToRow (43) - +- CometHashAggregate (42) - +- CometColumnarExchange (41) - +- CometHashAggregate (40) - +- CometProject (39) - +- CometBroadcastHashJoin (38) - :- CometProject (36) - : +- CometBroadcastHashJoin (35) - : :- CometFilter (33) - : : +- CometScan parquet spark_catalog.default.web_sales (32) - : +- ReusedExchange (34) - +- ReusedExchange (37) +* HashAggregate (49) ++- Exchange (48) + +- * HashAggregate (47) + +- * Project (46) + +- * BroadcastHashJoin LeftAnti BuildRight (45) + :- * BroadcastHashJoin LeftAnti BuildRight (31) + : :- * ColumnarToRow (17) + : : +- CometHashAggregate (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.customer (9) + : +- BroadcastExchange (30) + : +- * ColumnarToRow (29) + : +- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- BroadcastExchange (44) + +- * ColumnarToRow (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometFilter (33) + : : +- CometScan parquet spark_catalog.default.web_sales (32) + : +- ReusedExchange (34) + +- ReusedExchange (37) (1) Scan parquet spark_catalog.default.store_sales @@ -121,9 +119,9 @@ Input [3]: [c_last_name#9, c_first_name#8, d_date#5] Keys [3]: [c_last_name#9, c_first_name#8, d_date#5] Functions: [] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [c_last_name#9, c_first_name#8, d_date#5] -Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [c_last_name#9, c_first_name#8, d_date#5] @@ -174,9 +172,9 @@ Input [3]: [c_last_name#17, c_first_name#16, d_date#14] Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(27) CometColumnarExchange +(27) CometExchange Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [3]: [c_last_name#17, c_first_name#16, d_date#14] @@ -237,9 +235,9 @@ Input [3]: [c_last_name#25, c_first_name#24, d_date#22] Keys [3]: [c_last_name#25, c_first_name#24, d_date#22] Functions: [] -(41) CometColumnarExchange +(41) CometExchange Input [3]: [c_last_name#25, c_first_name#24, d_date#22] -Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (42) CometHashAggregate Input [3]: [c_last_name#25, c_first_name#24, d_date#22] @@ -270,17 +268,11 @@ Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#26] Results [1]: [count#27] -(48) RowToColumnar +(48) Exchange Input [1]: [count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(49) CometColumnarExchange -Input [1]: [count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(50) ColumnarToRow [codegen id : 4] -Input [1]: [count#27] - -(51) HashAggregate [codegen id : 4] +(49) HashAggregate [codegen id : 4] Input [1]: [count#27] Keys: [] Functions [1]: [count(1)] @@ -290,32 +282,32 @@ Results [1]: [count(1)#28 AS count(1)#29] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 -BroadcastExchange (56) -+- * ColumnarToRow (55) - +- CometProject (54) - +- CometFilter (53) - +- CometScan parquet spark_catalog.default.date_dim (52) +BroadcastExchange (54) ++- * ColumnarToRow (53) + +- CometProject (52) + +- CometFilter (51) + +- CometScan parquet spark_catalog.default.date_dim (50) -(52) Scan parquet spark_catalog.default.date_dim +(50) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(53) CometFilter +(51) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) -(54) CometProject +(52) CometProject Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] -(55) ColumnarToRow [codegen id : 1] +(53) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#4, d_date#5] -(56) BroadcastExchange +(54) BroadcastExchange Input [2]: [d_date_sk#4, d_date#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt index 69f33a57a..6e3328ae1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q87/simplified.txt @@ -1,71 +1,69 @@ WholeStageCodegen (4) HashAggregate [count] [count(1),count(1),count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (3) + HashAggregate [count,count] + Project + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + ColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [d_date_sk,d_date] #4 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #7 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_date] #4 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #2 + CometExchange [c_last_name,c_first_name,d_date] #9 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ss_customer_sk,d_date] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #4 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #7 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [cs_bill_customer_sk,d_date] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] - CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #9 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ws_bill_customer_sk,d_date] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_date] #4 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt index cd9e4817c..1dface7ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/explain.txt @@ -8,7 +8,7 @@ : : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) : : : : : : :- * ColumnarToRow (25) : : : : : : : +- CometHashAggregate (24) -: : : : : : : +- CometColumnarExchange (23) +: : : : : : : +- CometExchange (23) : : : : : : : +- CometHashAggregate (22) : : : : : : : +- CometProject (21) : : : : : : : +- CometBroadcastHashJoin (20) @@ -34,7 +34,7 @@ : : : : : : +- BroadcastExchange (45) : : : : : : +- * ColumnarToRow (44) : : : : : : +- CometHashAggregate (43) -: : : : : : +- CometColumnarExchange (42) +: : : : : : +- CometExchange (42) : : : : : : +- CometHashAggregate (41) : : : : : : +- CometProject (40) : : : : : : +- CometBroadcastHashJoin (39) @@ -54,7 +54,7 @@ : : : : : +- BroadcastExchange (66) : : : : : +- * ColumnarToRow (65) : : : : : +- CometHashAggregate (64) -: : : : : +- CometColumnarExchange (63) +: : : : : +- CometExchange (63) : : : : : +- CometHashAggregate (62) : : : : : +- CometProject (61) : : : : : +- CometBroadcastHashJoin (60) @@ -74,7 +74,7 @@ : : : : +- BroadcastExchange (87) : : : : +- * ColumnarToRow (86) : : : : +- CometHashAggregate (85) -: : : : +- CometColumnarExchange (84) +: : : : +- CometExchange (84) : : : : +- CometHashAggregate (83) : : : : +- CometProject (82) : : : : +- CometBroadcastHashJoin (81) @@ -94,7 +94,7 @@ : : : +- BroadcastExchange (108) : : : +- * ColumnarToRow (107) : : : +- CometHashAggregate (106) -: : : +- CometColumnarExchange (105) +: : : +- CometExchange (105) : : : +- CometHashAggregate (104) : : : +- CometProject (103) : : : +- CometBroadcastHashJoin (102) @@ -114,7 +114,7 @@ : : +- BroadcastExchange (129) : : +- * ColumnarToRow (128) : : +- CometHashAggregate (127) -: : +- CometColumnarExchange (126) +: : +- CometExchange (126) : : +- CometHashAggregate (125) : : +- CometProject (124) : : +- CometBroadcastHashJoin (123) @@ -134,7 +134,7 @@ : +- BroadcastExchange (150) : +- * ColumnarToRow (149) : +- CometHashAggregate (148) -: +- CometColumnarExchange (147) +: +- CometExchange (147) : +- CometHashAggregate (146) : +- CometProject (145) : +- CometBroadcastHashJoin (144) @@ -154,7 +154,7 @@ +- BroadcastExchange (171) +- * ColumnarToRow (170) +- CometHashAggregate (169) - +- CometColumnarExchange (168) + +- CometExchange (168) +- CometHashAggregate (167) +- CometProject (166) +- CometBroadcastHashJoin (165) @@ -276,9 +276,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(23) CometColumnarExchange +(23) CometExchange Input [1]: [count#13] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [1]: [count#13] @@ -359,9 +359,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(42) CometColumnarExchange +(42) CometExchange Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (43) CometHashAggregate Input [1]: [count#24] @@ -450,9 +450,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(63) CometColumnarExchange +(63) CometExchange Input [1]: [count#35] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (64) CometHashAggregate Input [1]: [count#35] @@ -541,9 +541,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(84) CometColumnarExchange +(84) CometExchange Input [1]: [count#46] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (85) CometHashAggregate Input [1]: [count#46] @@ -632,9 +632,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(105) CometColumnarExchange +(105) CometExchange Input [1]: [count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (106) CometHashAggregate Input [1]: [count#57] @@ -723,9 +723,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(126) CometColumnarExchange +(126) CometExchange Input [1]: [count#68] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (127) CometHashAggregate Input [1]: [count#68] @@ -814,9 +814,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(147) CometColumnarExchange +(147) CometExchange Input [1]: [count#79] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (148) CometHashAggregate Input [1]: [count#79] @@ -905,9 +905,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(168) CometColumnarExchange +(168) CometExchange Input [1]: [count#90] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (169) CometHashAggregate Input [1]: [count#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt index b01685ed0..1bb61b6c9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q88/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h8_30_to_9,count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -38,7 +38,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h9_to_9_30,count,count(1)] - CometColumnarExchange #6 + CometExchange #6 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -61,7 +61,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h9_30_to_10,count,count(1)] - CometColumnarExchange #9 + CometExchange #9 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -84,7 +84,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h10_to_10_30,count,count(1)] - CometColumnarExchange #12 + CometExchange #12 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -107,7 +107,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h10_30_to_11,count,count(1)] - CometColumnarExchange #15 + CometExchange #15 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -130,7 +130,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h11_to_11_30,count,count(1)] - CometColumnarExchange #18 + CometExchange #18 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -153,7 +153,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h11_30_to_12,count,count(1)] - CometColumnarExchange #21 + CometExchange #21 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -176,7 +176,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h12_to_12_30,count,count(1)] - CometColumnarExchange #24 + CometExchange #24 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt index 6f973cee6..6789e5b18 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/explain.txt @@ -5,9 +5,9 @@ TakeOrderedAndProject (28) +- Window (25) +- * ColumnarToRow (24) +- CometSort (23) - +- CometColumnarExchange (22) + +- CometExchange (22) +- CometHashAggregate (21) - +- CometColumnarExchange (20) + +- CometExchange (20) +- CometHashAggregate (19) +- CometProject (18) +- CometBroadcastHashJoin (17) @@ -122,18 +122,18 @@ Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#12, s_st Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] -(20) CometColumnarExchange +(20) CometExchange Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum#16] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum#16] Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12] Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] -(22) CometColumnarExchange +(22) CometExchange Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (23) CometSort Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#17, _w0#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt index 26c510f5a..97b9563be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q89/simplified.txt @@ -8,9 +8,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla ColumnarToRow InputAdapter CometSort [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 CometHashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 + CometExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 CometHashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum,ss_sales_price] CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt index 375b036fc..30b6c1e44 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/explain.txt @@ -29,7 +29,7 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery * ColumnarToRow (12) +- CometProject (11) +- CometHashAggregate (10) - +- CometColumnarExchange (9) + +- CometExchange (9) +- CometHashAggregate (8) +- CometProject (7) +- CometFilter (6) @@ -56,9 +56,9 @@ Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#18)), partial_avg(UnscaledValue(ss_net_paid#19))] -(9) CometColumnarExchange +(9) CometExchange Input [5]: [count#21, sum#22, count#23, sum#24, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometHashAggregate Input [5]: [count#21, sum#22, count#23, sum#24, count#25] @@ -80,7 +80,7 @@ Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery * ColumnarToRow (20) +- CometProject (19) +- CometHashAggregate (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometFilter (14) @@ -107,9 +107,9 @@ Input [2]: [ss_ext_discount_amt#31, ss_net_paid#32] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#31)), partial_avg(UnscaledValue(ss_net_paid#32))] -(17) CometColumnarExchange +(17) CometExchange Input [5]: [count#34, sum#35, count#36, sum#37, count#38] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometHashAggregate Input [5]: [count#34, sum#35, count#36, sum#37, count#38] @@ -131,7 +131,7 @@ Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery * ColumnarToRow (28) +- CometProject (27) +- CometHashAggregate (26) - +- CometColumnarExchange (25) + +- CometExchange (25) +- CometHashAggregate (24) +- CometProject (23) +- CometFilter (22) @@ -158,9 +158,9 @@ Input [2]: [ss_ext_discount_amt#44, ss_net_paid#45] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#44)), partial_avg(UnscaledValue(ss_net_paid#45))] -(25) CometColumnarExchange +(25) CometExchange Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (26) CometHashAggregate Input [5]: [count#47, sum#48, count#49, sum#50, count#51] @@ -182,7 +182,7 @@ Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer * ColumnarToRow (36) +- CometProject (35) +- CometHashAggregate (34) - +- CometColumnarExchange (33) + +- CometExchange (33) +- CometHashAggregate (32) +- CometProject (31) +- CometFilter (30) @@ -209,9 +209,9 @@ Input [2]: [ss_ext_discount_amt#57, ss_net_paid#58] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#57)), partial_avg(UnscaledValue(ss_net_paid#58))] -(33) CometColumnarExchange +(33) CometExchange Input [5]: [count#60, sum#61, count#62, sum#63, count#64] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (34) CometHashAggregate Input [5]: [count#60, sum#61, count#62, sum#63, count#64] @@ -233,7 +233,7 @@ Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer * ColumnarToRow (44) +- CometProject (43) +- CometHashAggregate (42) - +- CometColumnarExchange (41) + +- CometExchange (41) +- CometHashAggregate (40) +- CometProject (39) +- CometFilter (38) @@ -260,9 +260,9 @@ Input [2]: [ss_ext_discount_amt#70, ss_net_paid#71] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#70)), partial_avg(UnscaledValue(ss_net_paid#71))] -(41) CometColumnarExchange +(41) CometExchange Input [5]: [count#73, sum#74, count#75, sum#76, count#77] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (42) CometHashAggregate Input [5]: [count#73, sum#74, count#75, sum#76, count#77] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt index e0d03e2ef..3d69c60d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q9/simplified.txt @@ -6,7 +6,7 @@ WholeStageCodegen (1) InputAdapter CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] @@ -19,7 +19,7 @@ WholeStageCodegen (1) InputAdapter CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometColumnarExchange #2 + CometExchange #2 CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] @@ -32,7 +32,7 @@ WholeStageCodegen (1) InputAdapter CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometColumnarExchange #3 + CometExchange #3 CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] @@ -45,7 +45,7 @@ WholeStageCodegen (1) InputAdapter CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometColumnarExchange #4 + CometExchange #4 CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] @@ -58,7 +58,7 @@ WholeStageCodegen (1) InputAdapter CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometColumnarExchange #5 + CometExchange #5 CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt index ab17ec1d0..b55193e7e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/explain.txt @@ -3,7 +3,7 @@ +- * BroadcastNestedLoopJoin Inner BuildRight (46) :- * ColumnarToRow (25) : +- CometHashAggregate (24) - : +- CometColumnarExchange (23) + : +- CometExchange (23) : +- CometHashAggregate (22) : +- CometProject (21) : +- CometBroadcastHashJoin (20) @@ -29,7 +29,7 @@ +- BroadcastExchange (45) +- * ColumnarToRow (44) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometProject (40) +- CometBroadcastHashJoin (39) @@ -151,9 +151,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(23) CometColumnarExchange +(23) CometExchange Input [1]: [count#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [1]: [count#11] @@ -234,9 +234,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(42) CometColumnarExchange +(42) CometExchange Input [1]: [count#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (43) CometHashAggregate Input [1]: [count#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt index 65860a550..95fd73d86 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q90/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [amc,count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] @@ -33,7 +33,7 @@ WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [pmc,count,count(1)] - CometColumnarExchange #6 + CometExchange #6 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt index 836e03f4a..aecb46518 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/explain.txt @@ -3,7 +3,7 @@ +- CometSort (40) +- CometColumnarExchange (39) +- CometHashAggregate (38) - +- CometColumnarExchange (37) + +- CometExchange (37) +- CometHashAggregate (36) +- CometProject (35) +- CometBroadcastHashJoin (34) @@ -215,9 +215,9 @@ Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_mari Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21] Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] -(37) CometColumnarExchange +(37) CometExchange Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, sum#24] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (38) CometHashAggregate Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, sum#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt index ff17b986a..0dc197b1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q91/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometSort [Call_Center,Call_Center_Name,Manager,Returns_Loss] CometColumnarExchange [Returns_Loss] #1 CometHashAggregate [Call_Center,Call_Center_Name,Manager,Returns_Loss,cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum,sum(UnscaledValue(cr_net_loss))] - CometColumnarExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 + CometExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 CometHashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum,cr_net_loss] CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt index 7d3423272..f3c063f6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (30) +- CometHashAggregate (29) - +- CometColumnarExchange (28) + +- CometExchange (28) +- CometHashAggregate (27) +- CometProject (26) +- CometBroadcastHashJoin (25) @@ -18,7 +18,7 @@ : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) - : +- CometColumnarExchange (18) + : +- CometExchange (18) : +- CometHashAggregate (17) : +- CometProject (16) : +- CometBroadcastHashJoin (15) @@ -116,9 +116,9 @@ Input [2]: [ws_item_sk#7, ws_ext_discount_amt#8] Keys [1]: [ws_item_sk#7] Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#8))] -(18) CometColumnarExchange +(18) CometExchange Input [3]: [ws_item_sk#7, sum#13, count#14] -Arguments: hashpartitioning(ws_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (19) CometHashAggregate Input [3]: [ws_item_sk#7, sum#13, count#14] @@ -159,9 +159,9 @@ Input [1]: [ws_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] -(28) CometColumnarExchange +(28) CometExchange Input [1]: [sum#17] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate Input [1]: [sum#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt index 065e38bb1..f49dd1ea3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q92/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [Excess Discount Amount ,sum,sum(UnscaledValue(ws_ext_discount_amt))] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,ws_ext_discount_amt] CometProject [ws_ext_discount_amt] CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] @@ -27,7 +27,7 @@ WholeStageCodegen (1) CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #4 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] CometHashAggregate [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,sum,count,avg(UnscaledValue(ws_ext_discount_amt))] - CometColumnarExchange [ws_item_sk] #5 + CometExchange [ws_item_sk] #5 CometHashAggregate [ws_item_sk,sum,count,ws_ext_discount_amt] CometProject [ws_item_sk,ws_ext_discount_amt] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt index ee3fb07aa..4c672e397 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q93/explain.txt @@ -2,18 +2,18 @@ * ColumnarToRow (22) +- CometTakeOrderedAndProject (21) +- CometHashAggregate (20) - +- CometColumnarExchange (19) + +- CometExchange (19) +- CometHashAggregate (18) +- CometProject (17) +- CometBroadcastHashJoin (16) :- CometProject (11) : +- CometSortMergeJoin (10) : :- CometSort (4) - : : +- CometColumnarExchange (3) + : : +- CometExchange (3) : : +- CometProject (2) : : +- CometScan parquet spark_catalog.default.store_sales (1) : +- CometSort (9) - : +- CometColumnarExchange (8) + : +- CometExchange (8) : +- CometProject (7) : +- CometFilter (6) : +- CometScan parquet spark_catalog.default.store_returns (5) @@ -33,9 +33,9 @@ ReadSchema: struct Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order_number#10] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [ws_warehouse_sk#9, ws_order_number#10] @@ -115,9 +113,9 @@ ReadSchema: struct Input [2]: [wr_order_number#12, wr_returned_date_sk#13] Arguments: [wr_order_number#12], [wr_order_number#12] -(16) CometColumnarExchange +(16) CometExchange Input [1]: [wr_order_number#12] -Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (17) CometSort Input [1]: [wr_order_number#12] @@ -249,17 +247,11 @@ Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] Results [3]: [sum#22, sum#23, count#25] -(44) RowToColumnar +(44) Exchange Input [3]: [sum#22, sum#23, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(45) CometColumnarExchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(46) ColumnarToRow [codegen id : 9] -Input [3]: [sum#22, sum#23, count#25] - -(47) HashAggregate [codegen id : 9] +(45) HashAggregate [codegen id : 9] Input [3]: [sum#22, sum#23, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt index 3f324f9f2..16295d42c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q94/simplified.txt @@ -1,70 +1,68 @@ WholeStageCodegen (9) HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (8) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (3) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometColumnarExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - CometColumnarExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [wr_order_number] - CometColumnarExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (8) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_web_site_sk,web_site_sk] + Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] + Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_date_sk,d_date_sk] + SortMergeJoin [ws_order_number,wr_order_number] + InputAdapter + WholeStageCodegen (3) + Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) + BroadcastExchange #5 + WholeStageCodegen (5) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) + BroadcastExchange #6 + WholeStageCodegen (6) ColumnarToRow InputAdapter - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt index f58d42056..348ee49e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/explain.txt @@ -1,67 +1,65 @@ == Physical Plan == -* HashAggregate (63) -+- * ColumnarToRow (62) - +- CometColumnarExchange (61) - +- RowToColumnar (60) - +- * HashAggregate (59) - +- * HashAggregate (58) - +- * HashAggregate (57) - +- * Project (56) - +- * BroadcastHashJoin Inner BuildRight (55) - :- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (42) - : : +- * BroadcastHashJoin Inner BuildRight (41) - : : :- * SortMergeJoin LeftSemi (35) - : : : :- * SortMergeJoin LeftSemi (18) - : : : : :- * ColumnarToRow (6) - : : : : : +- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- * Project (17) - : : : : +- * SortMergeJoin Inner (16) - : : : : :- * ColumnarToRow (12) - : : : : : +- CometSort (11) - : : : : : +- CometColumnarExchange (10) - : : : : : +- CometProject (9) - : : : : : +- CometFilter (8) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (7) - : : : : +- * ColumnarToRow (15) - : : : : +- CometSort (14) - : : : : +- ReusedExchange (13) - : : : +- * Project (34) - : : : +- * SortMergeJoin Inner (33) - : : : :- * ColumnarToRow (24) - : : : : +- CometSort (23) - : : : : +- CometColumnarExchange (22) - : : : : +- CometProject (21) - : : : : +- CometFilter (20) - : : : : +- CometScan parquet spark_catalog.default.web_returns (19) - : : : +- * Project (32) - : : : +- * SortMergeJoin Inner (31) - : : : :- * ColumnarToRow (27) - : : : : +- CometSort (26) - : : : : +- ReusedExchange (25) - : : : +- * ColumnarToRow (30) - : : : +- CometSort (29) - : : : +- ReusedExchange (28) - : : +- BroadcastExchange (40) - : : +- * ColumnarToRow (39) - : : +- CometProject (38) - : : +- CometFilter (37) - : : +- CometScan parquet spark_catalog.default.date_dim (36) - : +- BroadcastExchange (47) - : +- * ColumnarToRow (46) - : +- CometProject (45) - : +- CometFilter (44) - : +- CometScan parquet spark_catalog.default.customer_address (43) - +- BroadcastExchange (54) - +- * ColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.web_site (50) +* HashAggregate (61) ++- Exchange (60) + +- * HashAggregate (59) + +- * HashAggregate (58) + +- * HashAggregate (57) + +- * Project (56) + +- * BroadcastHashJoin Inner BuildRight (55) + :- * Project (49) + : +- * BroadcastHashJoin Inner BuildRight (48) + : :- * Project (42) + : : +- * BroadcastHashJoin Inner BuildRight (41) + : : :- * SortMergeJoin LeftSemi (35) + : : : :- * SortMergeJoin LeftSemi (18) + : : : : :- * ColumnarToRow (6) + : : : : : +- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : +- * Project (17) + : : : : +- * SortMergeJoin Inner (16) + : : : : :- * ColumnarToRow (12) + : : : : : +- CometSort (11) + : : : : : +- CometExchange (10) + : : : : : +- CometProject (9) + : : : : : +- CometFilter (8) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (7) + : : : : +- * ColumnarToRow (15) + : : : : +- CometSort (14) + : : : : +- ReusedExchange (13) + : : : +- * Project (34) + : : : +- * SortMergeJoin Inner (33) + : : : :- * ColumnarToRow (24) + : : : : +- CometSort (23) + : : : : +- CometExchange (22) + : : : : +- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometScan parquet spark_catalog.default.web_returns (19) + : : : +- * Project (32) + : : : +- * SortMergeJoin Inner (31) + : : : :- * ColumnarToRow (27) + : : : : +- CometSort (26) + : : : : +- ReusedExchange (25) + : : : +- * ColumnarToRow (30) + : : : +- CometSort (29) + : : : +- ReusedExchange (28) + : : +- BroadcastExchange (40) + : : +- * ColumnarToRow (39) + : : +- CometProject (38) + : : +- CometFilter (37) + : : +- CometScan parquet spark_catalog.default.date_dim (36) + : +- BroadcastExchange (47) + : +- * ColumnarToRow (46) + : +- CometProject (45) + : +- CometFilter (44) + : +- CometScan parquet spark_catalog.default.customer_address (43) + +- BroadcastExchange (54) + +- * ColumnarToRow (53) + +- CometProject (52) + +- CometFilter (51) + +- CometScan parquet spark_catalog.default.web_site (50) (1) Scan parquet spark_catalog.default.web_sales @@ -79,9 +77,9 @@ Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(4) CometColumnarExchange +(4) CometExchange Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] @@ -105,9 +103,9 @@ Condition : (isnotnull(ws_order_number#9) AND isnotnull(ws_warehouse_sk#8)) Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_number#9] -(10) CometColumnarExchange +(10) CometExchange Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (11) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] @@ -157,9 +155,9 @@ Condition : isnotnull(wr_order_number#13) Input [2]: [wr_order_number#13, wr_returned_date_sk#14] Arguments: [wr_order_number#13], [wr_order_number#13] -(22) CometColumnarExchange +(22) CometExchange Input [1]: [wr_order_number#13] -Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (23) CometSort Input [1]: [wr_order_number#13] @@ -331,17 +329,11 @@ Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#25, sum(UnscaledValue(ws_net_profit#6))#26, count(ws_order_number#4)#29] Results [3]: [sum#27, sum#28, count#30] -(60) RowToColumnar +(60) Exchange Input [3]: [sum#27, sum#28, count#30] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(61) CometColumnarExchange -Input [3]: [sum#27, sum#28, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(62) ColumnarToRow [codegen id : 15] -Input [3]: [sum#27, sum#28, count#30] - -(63) HashAggregate [codegen id : 15] +(61) HashAggregate [codegen id : 15] Input [3]: [sum#27, sum#28, count#30] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt index 2c736e6e5..5cfa7827a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q95/simplified.txt @@ -1,101 +1,99 @@ WholeStageCodegen (15) HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (14) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (5) - SortMergeJoin [ws_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometColumnarExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - CometColumnarExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - WholeStageCodegen (10) - Project [wr_order_number] - SortMergeJoin [wr_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometSort [wr_order_number] - CometColumnarExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometFilter [wr_order_number,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + InputAdapter + Exchange #1 + WholeStageCodegen (14) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_web_site_sk,web_site_sk] + Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] + Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_date_sk,d_date_sk] + SortMergeJoin [ws_order_number,wr_order_number] + InputAdapter + WholeStageCodegen (5) + SortMergeJoin [ws_order_number,ws_order_number] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow InputAdapter - WholeStageCodegen (9) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (4) + Project [ws_order_number] + SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + InputAdapter + WholeStageCodegen (2) + ColumnarToRow InputAdapter - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (3) + ColumnarToRow InputAdapter - WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + InputAdapter + WholeStageCodegen (10) + Project [wr_order_number] + SortMergeJoin [wr_order_number,ws_order_number] + InputAdapter + WholeStageCodegen (6) + ColumnarToRow + InputAdapter + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometFilter [wr_order_number,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + InputAdapter + WholeStageCodegen (9) + Project [ws_order_number] + SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + InputAdapter + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + InputAdapter + WholeStageCodegen (8) + ColumnarToRow + InputAdapter + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 InputAdapter - BroadcastExchange #6 - WholeStageCodegen (12) + BroadcastExchange #5 + WholeStageCodegen (11) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (13) + BroadcastExchange #6 + WholeStageCodegen (12) ColumnarToRow InputAdapter - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (13) + ColumnarToRow + InputAdapter + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt index 6f9d1a3f2..c4b2cf973 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometBroadcastHashJoin (20) @@ -129,9 +129,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(23) CometColumnarExchange +(23) CometExchange Input [1]: [count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [1]: [count#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt index b8b3e211d..d9a87aa3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q96/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [count(1),count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt index e119edec6..2eec0079d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == * ColumnarToRow (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometSortMergeJoin (20) :- CometSort (11) : +- CometHashAggregate (10) - : +- CometColumnarExchange (9) + : +- CometExchange (9) : +- CometHashAggregate (8) : +- CometProject (7) : +- CometBroadcastHashJoin (6) @@ -18,7 +18,7 @@ : +- CometScan parquet spark_catalog.default.date_dim (2) +- CometSort (19) +- CometHashAggregate (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometBroadcastHashJoin (14) @@ -66,9 +66,9 @@ Input [2]: [ss_item_sk#1, ss_customer_sk#2] Keys [2]: [ss_customer_sk#2, ss_item_sk#1] Functions: [] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometHashAggregate Input [2]: [ss_customer_sk#2, ss_item_sk#1] @@ -103,9 +103,9 @@ Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] Keys [2]: [cs_bill_customer_sk#9, cs_item_sk#10] Functions: [] -(17) CometColumnarExchange +(17) CometExchange Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Arguments: hashpartitioning(cs_bill_customer_sk#9, cs_item_sk#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_bill_customer_sk#9, cs_item_sk#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometHashAggregate Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] @@ -130,9 +130,9 @@ Input [2]: [customer_sk#7, customer_sk#14] Keys: [] Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(customer_sk#14)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END)] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [sum#16, sum#17, sum#18] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (24) CometHashAggregate Input [3]: [sum#16, sum#17, sum#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt index da4461d84..0036a4bd8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q97/simplified.txt @@ -2,13 +2,13 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [store_only,catalog_only,store_and_catalog,sum,sum,sum,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,sum,sum,customer_sk,customer_sk] CometProject [customer_sk,customer_sk] CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] - CometColumnarExchange [ss_customer_sk,ss_item_sk] #2 + CometExchange [ss_customer_sk,ss_item_sk] #2 CometHashAggregate [ss_customer_sk,ss_item_sk] CometProject [ss_item_sk,ss_customer_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -27,7 +27,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] - CometColumnarExchange [cs_bill_customer_sk,cs_item_sk] #5 + CometExchange [cs_bill_customer_sk,cs_item_sk] #5 CometHashAggregate [cs_bill_customer_sk,cs_item_sk] CometProject [cs_bill_customer_sk,cs_item_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt index 0239cc78c..3c017673f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/explain.txt @@ -8,9 +8,9 @@ +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -96,18 +96,18 @@ Input [6]: [ss_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt index 663bc2e3e..ac0911dee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q98/simplified.txt @@ -13,9 +13,9 @@ WholeStageCodegen (3) ColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometColumnarExchange [i_class] #2 + CometExchange [i_class] #2 CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt index ceb28b9e8..7935bb4c6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (28) +- CometTakeOrderedAndProject (27) +- CometHashAggregate (26) - +- CometColumnarExchange (25) + +- CometExchange (25) +- CometHashAggregate (24) +- CometProject (23) +- CometBroadcastHashJoin (22) @@ -145,9 +145,9 @@ Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupi Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(25) CometColumnarExchange +(25) CometExchange Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometHashAggregate Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/simplified.txt index f5ca9c40a..51599575d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark3_5/q99/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] CometHashAggregate [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum,sum,sum,sum,sum,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] - CometColumnarExchange [_groupingexpression,sm_type,cc_name] #1 + CometExchange [_groupingexpression,sm_type,cc_name] #1 CometHashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum,cs_ship_date_sk,cs_sold_date_sk] CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt index d1a177b29..f3cd64a14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/explain.txt @@ -1,53 +1,47 @@ == Physical Plan == -TakeOrderedAndProject (49) -+- * Project (48) - +- * BroadcastHashJoin Inner BuildRight (47) - :- * Project (42) - : +- * BroadcastHashJoin Inner BuildRight (41) - : :- * Project (35) - : : +- * BroadcastHashJoin Inner BuildRight (34) - : : :- * Filter (15) - : : : +- * HashAggregate (14) - : : : +- * ColumnarToRow (13) - : : : +- CometColumnarExchange (12) - : : : +- RowToColumnar (11) - : : : +- * HashAggregate (10) - : : : +- * ColumnarToRow (9) - : : : +- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_returns (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- BroadcastExchange (33) - : : +- * Filter (32) - : : +- * HashAggregate (31) - : : +- * ColumnarToRow (30) - : : +- CometColumnarExchange (29) - : : +- RowToColumnar (28) - : : +- * HashAggregate (27) - : : +- * HashAggregate (26) - : : +- * ColumnarToRow (25) - : : +- CometColumnarExchange (24) - : : +- RowToColumnar (23) - : : +- * HashAggregate (22) - : : +- * ColumnarToRow (21) - : : +- CometProject (20) - : : +- CometBroadcastHashJoin (19) - : : :- CometFilter (17) - : : : +- CometScan parquet spark_catalog.default.store_returns (16) - : : +- ReusedExchange (18) - : +- BroadcastExchange (40) - : +- * ColumnarToRow (39) - : +- CometProject (38) - : +- CometFilter (37) - : +- CometScan parquet spark_catalog.default.store (36) - +- BroadcastExchange (46) - +- * ColumnarToRow (45) - +- CometFilter (44) - +- CometScan parquet spark_catalog.default.customer (43) +TakeOrderedAndProject (43) ++- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * Project (29) + : : +- * BroadcastHashJoin Inner BuildRight (28) + : : :- * Filter (13) + : : : +- * HashAggregate (12) + : : : +- Exchange (11) + : : : +- * HashAggregate (10) + : : : +- * ColumnarToRow (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (27) + : : +- * Filter (26) + : : +- * HashAggregate (25) + : : +- Exchange (24) + : : +- * HashAggregate (23) + : : +- * HashAggregate (22) + : : +- Exchange (21) + : : +- * HashAggregate (20) + : : +- * ColumnarToRow (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometFilter (15) + : : : +- CometScan parquet spark_catalog.default.store_returns (14) + : : +- ReusedExchange (16) + : +- BroadcastExchange (34) + : +- * ColumnarToRow (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometScan parquet spark_catalog.default.store (30) + +- BroadcastExchange (40) + +- * ColumnarToRow (39) + +- CometFilter (38) + +- CometScan parquet spark_catalog.default.customer (37) (1) Scan parquet spark_catalog.default.store_returns @@ -100,28 +94,22 @@ Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] Aggregate Attributes [1]: [sum#8] Results [3]: [sr_customer_sk#1, sr_store_sk#2, sum#9] -(11) RowToColumnar +(11) Exchange Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#9] +Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(12) CometColumnarExchange -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#9] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(13) ColumnarToRow [codegen id : 7] -Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#9] - -(14) HashAggregate [codegen id : 7] +(12) HashAggregate [codegen id : 7] Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#9] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [sum(UnscaledValue(sr_return_amt#3))] Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#3))#10] Results [3]: [sr_customer_sk#1 AS ctr_customer_sk#11, sr_store_sk#2 AS ctr_store_sk#12, MakeDecimal(sum(UnscaledValue(sr_return_amt#3))#10,17,2) AS ctr_total_return#13] -(15) Filter [codegen id : 7] +(13) Filter [codegen id : 7] Input [3]: [ctr_customer_sk#11, ctr_store_sk#12, ctr_total_return#13] Condition : isnotnull(ctr_total_return#13) -(16) Scan parquet spark_catalog.default.store_returns +(14) Scan parquet spark_catalog.default.store_returns Output [4]: [sr_customer_sk#14, sr_store_sk#15, sr_return_amt#16, sr_returned_date_sk#17] Batched: true Location: InMemoryFileIndex [] @@ -129,187 +117,175 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#17), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(17) CometFilter +(15) CometFilter Input [4]: [sr_customer_sk#14, sr_store_sk#15, sr_return_amt#16, sr_returned_date_sk#17] Condition : isnotnull(sr_store_sk#15) -(18) ReusedExchange [Reuses operator id: 6] +(16) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#19] -(19) CometBroadcastHashJoin +(17) CometBroadcastHashJoin Left output [4]: [sr_customer_sk#14, sr_store_sk#15, sr_return_amt#16, sr_returned_date_sk#17] Right output [1]: [d_date_sk#19] Arguments: [sr_returned_date_sk#17], [d_date_sk#19], Inner, BuildRight -(20) CometProject +(18) CometProject Input [5]: [sr_customer_sk#14, sr_store_sk#15, sr_return_amt#16, sr_returned_date_sk#17, d_date_sk#19] Arguments: [sr_customer_sk#14, sr_store_sk#15, sr_return_amt#16], [sr_customer_sk#14, sr_store_sk#15, sr_return_amt#16] -(21) ColumnarToRow [codegen id : 2] +(19) ColumnarToRow [codegen id : 2] Input [3]: [sr_customer_sk#14, sr_store_sk#15, sr_return_amt#16] -(22) HashAggregate [codegen id : 2] +(20) HashAggregate [codegen id : 2] Input [3]: [sr_customer_sk#14, sr_store_sk#15, sr_return_amt#16] Keys [2]: [sr_customer_sk#14, sr_store_sk#15] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#16))] Aggregate Attributes [1]: [sum#20] Results [3]: [sr_customer_sk#14, sr_store_sk#15, sum#21] -(23) RowToColumnar +(21) Exchange Input [3]: [sr_customer_sk#14, sr_store_sk#15, sum#21] +Arguments: hashpartitioning(sr_customer_sk#14, sr_store_sk#15, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(24) CometColumnarExchange -Input [3]: [sr_customer_sk#14, sr_store_sk#15, sum#21] -Arguments: hashpartitioning(sr_customer_sk#14, sr_store_sk#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(25) ColumnarToRow [codegen id : 3] -Input [3]: [sr_customer_sk#14, sr_store_sk#15, sum#21] - -(26) HashAggregate [codegen id : 3] +(22) HashAggregate [codegen id : 3] Input [3]: [sr_customer_sk#14, sr_store_sk#15, sum#21] Keys [2]: [sr_customer_sk#14, sr_store_sk#15] Functions [1]: [sum(UnscaledValue(sr_return_amt#16))] Aggregate Attributes [1]: [sum(UnscaledValue(sr_return_amt#16))#10] Results [2]: [sr_store_sk#15 AS ctr_store_sk#22, MakeDecimal(sum(UnscaledValue(sr_return_amt#16))#10,17,2) AS ctr_total_return#23] -(27) HashAggregate [codegen id : 3] +(23) HashAggregate [codegen id : 3] Input [2]: [ctr_store_sk#22, ctr_total_return#23] Keys [1]: [ctr_store_sk#22] Functions [1]: [partial_avg(ctr_total_return#23)] Aggregate Attributes [2]: [sum#24, count#25] Results [3]: [ctr_store_sk#22, sum#26, count#27] -(28) RowToColumnar -Input [3]: [ctr_store_sk#22, sum#26, count#27] - -(29) CometColumnarExchange -Input [3]: [ctr_store_sk#22, sum#26, count#27] -Arguments: hashpartitioning(ctr_store_sk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(30) ColumnarToRow [codegen id : 4] +(24) Exchange Input [3]: [ctr_store_sk#22, sum#26, count#27] +Arguments: hashpartitioning(ctr_store_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(31) HashAggregate [codegen id : 4] +(25) HashAggregate [codegen id : 4] Input [3]: [ctr_store_sk#22, sum#26, count#27] Keys [1]: [ctr_store_sk#22] Functions [1]: [avg(ctr_total_return#23)] Aggregate Attributes [1]: [avg(ctr_total_return#23)#28] Results [2]: [(avg(ctr_total_return#23)#28 * 1.2) AS (avg(ctr_total_return) * 1.2)#29, ctr_store_sk#22] -(32) Filter [codegen id : 4] +(26) Filter [codegen id : 4] Input [2]: [(avg(ctr_total_return) * 1.2)#29, ctr_store_sk#22] Condition : isnotnull((avg(ctr_total_return) * 1.2)#29) -(33) BroadcastExchange +(27) BroadcastExchange Input [2]: [(avg(ctr_total_return) * 1.2)#29, ctr_store_sk#22] Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=4] -(34) BroadcastHashJoin [codegen id : 7] +(28) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ctr_store_sk#12] Right keys [1]: [ctr_store_sk#22] Join type: Inner Join condition: (cast(ctr_total_return#13 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#29) -(35) Project [codegen id : 7] +(29) Project [codegen id : 7] Output [2]: [ctr_customer_sk#11, ctr_store_sk#12] Input [5]: [ctr_customer_sk#11, ctr_store_sk#12, ctr_total_return#13, (avg(ctr_total_return) * 1.2)#29, ctr_store_sk#22] -(36) Scan parquet spark_catalog.default.store +(30) Scan parquet spark_catalog.default.store Output [2]: [s_store_sk#30, s_state#31] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] ReadSchema: struct -(37) CometFilter +(31) CometFilter Input [2]: [s_store_sk#30, s_state#31] Condition : ((isnotnull(s_state#31) AND (s_state#31 = TN)) AND isnotnull(s_store_sk#30)) -(38) CometProject +(32) CometProject Input [2]: [s_store_sk#30, s_state#31] Arguments: [s_store_sk#30], [s_store_sk#30] -(39) ColumnarToRow [codegen id : 5] +(33) ColumnarToRow [codegen id : 5] Input [1]: [s_store_sk#30] -(40) BroadcastExchange +(34) BroadcastExchange Input [1]: [s_store_sk#30] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -(41) BroadcastHashJoin [codegen id : 7] +(35) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ctr_store_sk#12] Right keys [1]: [s_store_sk#30] Join type: Inner Join condition: None -(42) Project [codegen id : 7] +(36) Project [codegen id : 7] Output [1]: [ctr_customer_sk#11] Input [3]: [ctr_customer_sk#11, ctr_store_sk#12, s_store_sk#30] -(43) Scan parquet spark_catalog.default.customer +(37) Scan parquet spark_catalog.default.customer Output [2]: [c_customer_sk#32, c_customer_id#33] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(44) CometFilter +(38) CometFilter Input [2]: [c_customer_sk#32, c_customer_id#33] Condition : isnotnull(c_customer_sk#32) -(45) ColumnarToRow [codegen id : 6] +(39) ColumnarToRow [codegen id : 6] Input [2]: [c_customer_sk#32, c_customer_id#33] -(46) BroadcastExchange +(40) BroadcastExchange Input [2]: [c_customer_sk#32, c_customer_id#33] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] -(47) BroadcastHashJoin [codegen id : 7] +(41) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ctr_customer_sk#11] Right keys [1]: [c_customer_sk#32] Join type: Inner Join condition: None -(48) Project [codegen id : 7] +(42) Project [codegen id : 7] Output [1]: [c_customer_id#33] Input [3]: [ctr_customer_sk#11, c_customer_sk#32, c_customer_id#33] -(49) TakeOrderedAndProject +(43) TakeOrderedAndProject Input [1]: [c_customer_id#33] Arguments: 100, [c_customer_id#33 ASC NULLS FIRST], [c_customer_id#33] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (54) -+- * ColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.date_dim (50) +BroadcastExchange (48) ++- * ColumnarToRow (47) + +- CometProject (46) + +- CometFilter (45) + +- CometScan parquet spark_catalog.default.date_dim (44) -(50) Scan parquet spark_catalog.default.date_dim +(44) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(51) CometFilter +(45) CometFilter Input [2]: [d_date_sk#6, d_year#7] Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) -(52) CometProject +(46) CometProject Input [2]: [d_date_sk#6, d_year#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(53) ColumnarToRow [codegen id : 1] +(47) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(54) BroadcastExchange +(48) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] -Subquery:2 Hosting operator id = 16 Hosting Expression = sr_returned_date_sk#17 IN dynamicpruning#5 +Subquery:2 Hosting operator id = 14 Hosting Expression = sr_returned_date_sk#17 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt index b0c85f80b..43bad5d70 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q1/simplified.txt @@ -8,56 +8,50 @@ TakeOrderedAndProject [c_customer_id] BroadcastHashJoin [ctr_store_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2)] Filter [ctr_total_return] HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_customer_sk,ctr_store_sk,ctr_total_return,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [sr_customer_sk,sr_store_sk] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] - CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] - CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + Exchange [sr_customer_sk,sr_store_sk] #1 + WholeStageCodegen (1) + HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] + CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] + CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #4 WholeStageCodegen (4) Filter [(avg(ctr_total_return) * 1.2)] HashAggregate [ctr_store_sk,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ctr_store_sk] #5 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count] - HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [sr_customer_sk,sr_store_sk] #6 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] - CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] - CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 + InputAdapter + Exchange [ctr_store_sk] #5 + WholeStageCodegen (3) + HashAggregate [ctr_store_sk,ctr_total_return] [sum,count,sum,count] + HashAggregate [sr_customer_sk,sr_store_sk,sum] [sum(UnscaledValue(sr_return_amt)),ctr_store_sk,ctr_total_return,sum] + InputAdapter + Exchange [sr_customer_sk,sr_store_sk] #6 + WholeStageCodegen (2) + HashAggregate [sr_customer_sk,sr_store_sk,sr_return_amt] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] + CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] + CometFilter [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #3 InputAdapter BroadcastExchange #7 WholeStageCodegen (5) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt index 6f8c539fb..4a29b7260 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/explain.txt @@ -1,51 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- * ColumnarToRow (45) - +- CometColumnarExchange (44) - +- RowToColumnar (43) - +- * HashAggregate (42) - +- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (33) - : +- * ColumnarToRow (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (39) - +- * ColumnarToRow (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.customer_demographics (36) +TakeOrderedAndProject (45) ++- * HashAggregate (44) + +- Exchange (43) + +- * HashAggregate (42) + +- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * ColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (39) + +- * ColumnarToRow (38) + +- CometFilter (37) + +- CometScan parquet spark_catalog.default.customer_demographics (36) (1) Scan parquet spark_catalog.default.customer @@ -245,56 +243,50 @@ Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#31] Results [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] -(43) RowToColumnar +(43) Exchange Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(44) CometColumnarExchange -Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] -Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(45) ColumnarToRow [codegen id : 6] -Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] - -(46) HashAggregate [codegen id : 6] +(44) HashAggregate [codegen id : 6] Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] Keys [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#33] Results [14]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, count(1)#33 AS cnt1#34, cd_purchase_estimate#26, count(1)#33 AS cnt2#35, cd_credit_rating#27, count(1)#33 AS cnt3#36, cd_dep_count#28, count(1)#33 AS cnt4#37, cd_dep_employed_count#29, count(1)#33 AS cnt5#38, cd_dep_college_count#30, count(1)#33 AS cnt6#39] -(47) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [14]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#34, cd_purchase_estimate#26, cnt2#35, cd_credit_rating#27, cnt3#36, cd_dep_count#28, cnt4#37, cd_dep_employed_count#29, cnt5#38, cd_dep_college_count#30, cnt6#39] Arguments: 100, [cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_education_status#25 ASC NULLS FIRST, cd_purchase_estimate#26 ASC NULLS FIRST, cd_credit_rating#27 ASC NULLS FIRST, cd_dep_count#28 ASC NULLS FIRST, cd_dep_employed_count#29 ASC NULLS FIRST, cd_dep_college_count#30 ASC NULLS FIRST], [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#34, cd_purchase_estimate#26, cnt2#35, cd_credit_rating#27, cnt3#36, cd_dep_count#28, cnt4#37, cd_dep_employed_count#29, cnt5#38, cd_dep_college_count#30, cnt6#39] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * ColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan parquet spark_catalog.default.date_dim (48) +BroadcastExchange (50) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(48) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(49) CometFilter +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Condition : (((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2002)) AND (d_moy#11 >= 1)) AND (d_moy#11 <= 4)) AND isnotnull(d_date_sk#9)) -(50) CometProject +(48) CometProject Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Arguments: [d_date_sk#9], [d_date_sk#9] -(51) ColumnarToRow [codegen id : 1] +(49) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(52) BroadcastExchange +(50) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt index a3ab6bbbb..efd4b187d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q10/simplified.txt @@ -1,73 +1,71 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + InputAdapter + Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_county] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt index 2707bf8e3..3f5dd24f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/explain.txt @@ -1,84 +1,76 @@ == Physical Plan == -TakeOrderedAndProject (80) -+- * Project (79) - +- * BroadcastHashJoin Inner BuildRight (78) - :- * Project (60) - : +- * BroadcastHashJoin Inner BuildRight (59) - : :- * Project (40) - : : +- * BroadcastHashJoin Inner BuildRight (39) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- * ColumnarToRow (17) - : : : +- CometColumnarExchange (16) - : : : +- RowToColumnar (15) - : : : +- * HashAggregate (14) - : : : +- * ColumnarToRow (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- BroadcastExchange (38) - : : +- * HashAggregate (37) - : : +- * ColumnarToRow (36) - : : +- CometColumnarExchange (35) - : : +- RowToColumnar (34) - : : +- * HashAggregate (33) - : : +- * ColumnarToRow (32) - : : +- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (26) - : : : +- CometBroadcastHashJoin (25) - : : : :- CometFilter (21) - : : : : +- CometScan parquet spark_catalog.default.customer (20) - : : : +- CometBroadcastExchange (24) - : : : +- CometFilter (23) - : : : +- CometScan parquet spark_catalog.default.store_sales (22) - : : +- CometBroadcastExchange (29) - : : +- CometFilter (28) - : : +- CometScan parquet spark_catalog.default.date_dim (27) - : +- BroadcastExchange (58) - : +- * Filter (57) - : +- * HashAggregate (56) - : +- * ColumnarToRow (55) - : +- CometColumnarExchange (54) - : +- RowToColumnar (53) - : +- * HashAggregate (52) - : +- * ColumnarToRow (51) - : +- CometProject (50) - : +- CometBroadcastHashJoin (49) - : :- CometProject (47) - : : +- CometBroadcastHashJoin (46) - : : :- CometFilter (42) - : : : +- CometScan parquet spark_catalog.default.customer (41) - : : +- CometBroadcastExchange (45) - : : +- CometFilter (44) - : : +- CometScan parquet spark_catalog.default.web_sales (43) - : +- ReusedExchange (48) - +- BroadcastExchange (77) - +- * HashAggregate (76) - +- * ColumnarToRow (75) - +- CometColumnarExchange (74) - +- RowToColumnar (73) - +- * HashAggregate (72) - +- * ColumnarToRow (71) - +- CometProject (70) - +- CometBroadcastHashJoin (69) - :- CometProject (67) - : +- CometBroadcastHashJoin (66) - : :- CometFilter (62) - : : +- CometScan parquet spark_catalog.default.customer (61) - : +- CometBroadcastExchange (65) - : +- CometFilter (64) - : +- CometScan parquet spark_catalog.default.web_sales (63) - +- ReusedExchange (68) +TakeOrderedAndProject (72) ++- * Project (71) + +- * BroadcastHashJoin Inner BuildRight (70) + :- * Project (54) + : +- * BroadcastHashJoin Inner BuildRight (53) + : :- * Project (36) + : : +- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (17) + : : : +- * HashAggregate (16) + : : : +- Exchange (15) + : : : +- * HashAggregate (14) + : : : +- * ColumnarToRow (13) + : : : +- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- Exchange (32) + : : +- * HashAggregate (31) + : : +- * ColumnarToRow (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometFilter (19) + : : : : +- CometScan parquet spark_catalog.default.customer (18) + : : : +- CometBroadcastExchange (22) + : : : +- CometFilter (21) + : : : +- CometScan parquet spark_catalog.default.store_sales (20) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (52) + : +- * Filter (51) + : +- * HashAggregate (50) + : +- Exchange (49) + : +- * HashAggregate (48) + : +- * ColumnarToRow (47) + : +- CometProject (46) + : +- CometBroadcastHashJoin (45) + : :- CometProject (43) + : : +- CometBroadcastHashJoin (42) + : : :- CometFilter (38) + : : : +- CometScan parquet spark_catalog.default.customer (37) + : : +- CometBroadcastExchange (41) + : : +- CometFilter (40) + : : +- CometScan parquet spark_catalog.default.web_sales (39) + : +- ReusedExchange (44) + +- BroadcastExchange (69) + +- * HashAggregate (68) + +- Exchange (67) + +- * HashAggregate (66) + +- * ColumnarToRow (65) + +- CometProject (64) + +- CometBroadcastHashJoin (63) + :- CometProject (61) + : +- CometBroadcastHashJoin (60) + : :- CometFilter (56) + : : +- CometScan parquet spark_catalog.default.customer (55) + : +- CometBroadcastExchange (59) + : +- CometFilter (58) + : +- CometScan parquet spark_catalog.default.web_sales (57) + +- ReusedExchange (62) (1) Scan parquet spark_catalog.default.customer @@ -151,39 +143,33 @@ Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discoun Aggregate Attributes [1]: [sum#16] Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#17] -(15) RowToColumnar +(15) Exchange Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#17] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(16) CometColumnarExchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#17] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(17) ColumnarToRow [codegen id : 8] -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#17] - -(18) HashAggregate [codegen id : 8] +(16) HashAggregate [codegen id : 8] Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#17] Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#18] Results [2]: [c_customer_id#2 AS customer_id#19, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#18,18,2) AS year_total#20] -(19) Filter [codegen id : 8] +(17) Filter [codegen id : 8] Input [2]: [customer_id#19, year_total#20] Condition : (isnotnull(year_total#20) AND (year_total#20 > 0.00)) -(20) Scan parquet spark_catalog.default.customer +(18) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(21) CometFilter +(19) CometFilter Input [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_customer_id#22)) -(22) Scan parquet spark_catalog.default.store_sales +(20) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] @@ -191,100 +177,94 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#32), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(23) CometFilter +(21) CometFilter Input [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Condition : isnotnull(ss_customer_sk#29) -(24) CometBroadcastExchange +(22) CometBroadcastExchange Input [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Arguments: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] -(25) CometBroadcastHashJoin +(23) CometBroadcastHashJoin Left output [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] Right output [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Arguments: [c_customer_sk#21], [ss_customer_sk#29], Inner, BuildRight -(26) CometProject +(24) CometProject Input [12]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32], [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] -(27) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#34, d_year#35] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [2]: [d_date_sk#34, d_year#35] Condition : ((isnotnull(d_year#35) AND (d_year#35 = 2002)) AND isnotnull(d_date_sk#34)) -(29) CometBroadcastExchange +(27) CometBroadcastExchange Input [2]: [d_date_sk#34, d_year#35] Arguments: [d_date_sk#34, d_year#35] -(30) CometBroadcastHashJoin +(28) CometBroadcastHashJoin Left output [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Right output [2]: [d_date_sk#34, d_year#35] Arguments: [ss_sold_date_sk#32], [d_date_sk#34], Inner, BuildRight -(31) CometProject +(29) CometProject Input [12]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32, d_date_sk#34, d_year#35] Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, d_year#35], [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, d_year#35] -(32) ColumnarToRow [codegen id : 2] +(30) ColumnarToRow [codegen id : 2] Input [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, d_year#35] -(33) HashAggregate [codegen id : 2] +(31) HashAggregate [codegen id : 2] Input [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, d_year#35] Keys [8]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#31 - ss_ext_discount_amt#30)))] Aggregate Attributes [1]: [sum#36] Results [9]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, sum#37] -(34) RowToColumnar -Input [9]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, sum#37] - -(35) CometColumnarExchange -Input [9]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, sum#37] -Arguments: hashpartitioning(c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(36) ColumnarToRow [codegen id : 3] +(32) Exchange Input [9]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, sum#37] +Arguments: hashpartitioning(c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(37) HashAggregate [codegen id : 3] +(33) HashAggregate [codegen id : 3] Input [9]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, sum#37] Keys [8]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#31 - ss_ext_discount_amt#30)))] Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#31 - ss_ext_discount_amt#30)))#18] Results [3]: [c_customer_id#22 AS customer_id#38, c_preferred_cust_flag#25 AS customer_preferred_cust_flag#39, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#31 - ss_ext_discount_amt#30)))#18,18,2) AS year_total#40] -(38) BroadcastExchange +(34) BroadcastExchange Input [3]: [customer_id#38, customer_preferred_cust_flag#39, year_total#40] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(39) BroadcastHashJoin [codegen id : 8] +(35) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#19] Right keys [1]: [customer_id#38] Join type: Inner Join condition: None -(40) Project [codegen id : 8] +(36) Project [codegen id : 8] Output [4]: [customer_id#19, year_total#20, customer_preferred_cust_flag#39, year_total#40] Input [5]: [customer_id#19, year_total#20, customer_id#38, customer_preferred_cust_flag#39, year_total#40] -(41) Scan parquet spark_catalog.default.customer +(37) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(42) CometFilter +(38) CometFilter Input [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] Condition : (isnotnull(c_customer_sk#41) AND isnotnull(c_customer_id#42)) -(43) Scan parquet spark_catalog.default.web_sales +(39) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] Batched: true Location: InMemoryFileIndex [] @@ -292,92 +272,86 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#52), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(44) CometFilter +(40) CometFilter Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] Condition : isnotnull(ws_bill_customer_sk#49) -(45) CometBroadcastExchange +(41) CometBroadcastExchange Input [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] Arguments: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -(46) CometBroadcastHashJoin +(42) CometBroadcastHashJoin Left output [8]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48] Right output [4]: [ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] Arguments: [c_customer_sk#41], [ws_bill_customer_sk#49], Inner, BuildRight -(47) CometProject +(43) CometProject Input [12]: [c_customer_sk#41, c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_bill_customer_sk#49, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] Arguments: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52], [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] -(48) ReusedExchange [Reuses operator id: 10] +(44) ReusedExchange [Reuses operator id: 10] Output [2]: [d_date_sk#54, d_year#55] -(49) CometBroadcastHashJoin +(45) CometBroadcastHashJoin Left output [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52] Right output [2]: [d_date_sk#54, d_year#55] Arguments: [ws_sold_date_sk#52], [d_date_sk#54], Inner, BuildRight -(50) CometProject +(46) CometProject Input [12]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, ws_sold_date_sk#52, d_date_sk#54, d_year#55] Arguments: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, d_year#55], [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, d_year#55] -(51) ColumnarToRow [codegen id : 4] +(47) ColumnarToRow [codegen id : 4] Input [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, d_year#55] -(52) HashAggregate [codegen id : 4] +(48) HashAggregate [codegen id : 4] Input [10]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, ws_ext_discount_amt#50, ws_ext_list_price#51, d_year#55] Keys [8]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#55] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))] Aggregate Attributes [1]: [sum#56] Results [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#55, sum#57] -(53) RowToColumnar -Input [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#55, sum#57] - -(54) CometColumnarExchange -Input [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#55, sum#57] -Arguments: hashpartitioning(c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(55) ColumnarToRow [codegen id : 5] +(49) Exchange Input [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#55, sum#57] +Arguments: hashpartitioning(c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#55, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(56) HashAggregate [codegen id : 5] +(50) HashAggregate [codegen id : 5] Input [9]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#55, sum#57] Keys [8]: [c_customer_id#42, c_first_name#43, c_last_name#44, c_preferred_cust_flag#45, c_birth_country#46, c_login#47, c_email_address#48, d_year#55] Functions [1]: [sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))] Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))#58] Results [2]: [c_customer_id#42 AS customer_id#59, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#51 - ws_ext_discount_amt#50)))#58,18,2) AS year_total#60] -(57) Filter [codegen id : 5] +(51) Filter [codegen id : 5] Input [2]: [customer_id#59, year_total#60] Condition : (isnotnull(year_total#60) AND (year_total#60 > 0.00)) -(58) BroadcastExchange +(52) BroadcastExchange Input [2]: [customer_id#59, year_total#60] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(59) BroadcastHashJoin [codegen id : 8] +(53) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#19] Right keys [1]: [customer_id#59] Join type: Inner Join condition: None -(60) Project [codegen id : 8] +(54) Project [codegen id : 8] Output [5]: [customer_id#19, year_total#20, customer_preferred_cust_flag#39, year_total#40, year_total#60] Input [6]: [customer_id#19, year_total#20, customer_preferred_cust_flag#39, year_total#40, customer_id#59, year_total#60] -(61) Scan parquet spark_catalog.default.customer +(55) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#61, c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(62) CometFilter +(56) CometFilter Input [8]: [c_customer_sk#61, c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68] Condition : (isnotnull(c_customer_sk#61) AND isnotnull(c_customer_id#62)) -(63) Scan parquet spark_catalog.default.web_sales +(57) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] Batched: true Location: InMemoryFileIndex [] @@ -385,134 +359,128 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#72), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(64) CometFilter +(58) CometFilter Input [4]: [ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] Condition : isnotnull(ws_bill_customer_sk#69) -(65) CometBroadcastExchange +(59) CometBroadcastExchange Input [4]: [ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] Arguments: [ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] -(66) CometBroadcastHashJoin +(60) CometBroadcastHashJoin Left output [8]: [c_customer_sk#61, c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68] Right output [4]: [ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] Arguments: [c_customer_sk#61], [ws_bill_customer_sk#69], Inner, BuildRight -(67) CometProject +(61) CometProject Input [12]: [c_customer_sk#61, c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, ws_bill_customer_sk#69, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] Arguments: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72], [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] -(68) ReusedExchange [Reuses operator id: 29] +(62) ReusedExchange [Reuses operator id: 27] Output [2]: [d_date_sk#74, d_year#75] -(69) CometBroadcastHashJoin +(63) CometBroadcastHashJoin Left output [10]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72] Right output [2]: [d_date_sk#74, d_year#75] Arguments: [ws_sold_date_sk#72], [d_date_sk#74], Inner, BuildRight -(70) CometProject +(64) CometProject Input [12]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, ws_ext_discount_amt#70, ws_ext_list_price#71, ws_sold_date_sk#72, d_date_sk#74, d_year#75] Arguments: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, ws_ext_discount_amt#70, ws_ext_list_price#71, d_year#75], [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, ws_ext_discount_amt#70, ws_ext_list_price#71, d_year#75] -(71) ColumnarToRow [codegen id : 6] +(65) ColumnarToRow [codegen id : 6] Input [10]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, ws_ext_discount_amt#70, ws_ext_list_price#71, d_year#75] -(72) HashAggregate [codegen id : 6] +(66) HashAggregate [codegen id : 6] Input [10]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, ws_ext_discount_amt#70, ws_ext_list_price#71, d_year#75] Keys [8]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, d_year#75] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#71 - ws_ext_discount_amt#70)))] Aggregate Attributes [1]: [sum#76] Results [9]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, d_year#75, sum#77] -(73) RowToColumnar -Input [9]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, d_year#75, sum#77] - -(74) CometColumnarExchange -Input [9]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, d_year#75, sum#77] -Arguments: hashpartitioning(c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, d_year#75, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(75) ColumnarToRow [codegen id : 7] +(67) Exchange Input [9]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, d_year#75, sum#77] +Arguments: hashpartitioning(c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, d_year#75, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(76) HashAggregate [codegen id : 7] +(68) HashAggregate [codegen id : 7] Input [9]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, d_year#75, sum#77] Keys [8]: [c_customer_id#62, c_first_name#63, c_last_name#64, c_preferred_cust_flag#65, c_birth_country#66, c_login#67, c_email_address#68, d_year#75] Functions [1]: [sum(UnscaledValue((ws_ext_list_price#71 - ws_ext_discount_amt#70)))] Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#71 - ws_ext_discount_amt#70)))#58] Results [2]: [c_customer_id#62 AS customer_id#78, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#71 - ws_ext_discount_amt#70)))#58,18,2) AS year_total#79] -(77) BroadcastExchange +(69) BroadcastExchange Input [2]: [customer_id#78, year_total#79] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(78) BroadcastHashJoin [codegen id : 8] +(70) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#19] Right keys [1]: [customer_id#78] Join type: Inner Join condition: (CASE WHEN (year_total#60 > 0.00) THEN (year_total#79 / year_total#60) END > CASE WHEN (year_total#20 > 0.00) THEN (year_total#40 / year_total#20) END) -(79) Project [codegen id : 8] +(71) Project [codegen id : 8] Output [1]: [customer_preferred_cust_flag#39] Input [7]: [customer_id#19, year_total#20, customer_preferred_cust_flag#39, year_total#40, year_total#60, customer_id#78, year_total#79] -(80) TakeOrderedAndProject +(72) TakeOrderedAndProject Input [1]: [customer_preferred_cust_flag#39] Arguments: 100, [customer_preferred_cust_flag#39 ASC NULLS FIRST], [customer_preferred_cust_flag#39] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (84) -+- * ColumnarToRow (83) - +- CometFilter (82) - +- CometScan parquet spark_catalog.default.date_dim (81) +BroadcastExchange (76) ++- * ColumnarToRow (75) + +- CometFilter (74) + +- CometScan parquet spark_catalog.default.date_dim (73) -(81) Scan parquet spark_catalog.default.date_dim +(73) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_year#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(82) CometFilter +(74) CometFilter Input [2]: [d_date_sk#14, d_year#15] Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2001)) AND isnotnull(d_date_sk#14)) -(83) ColumnarToRow [codegen id : 1] +(75) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#14, d_year#15] -(84) BroadcastExchange +(76) BroadcastExchange Input [2]: [d_date_sk#14, d_year#15] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#32 IN dynamicpruning#33 -BroadcastExchange (88) -+- * ColumnarToRow (87) - +- CometFilter (86) - +- CometScan parquet spark_catalog.default.date_dim (85) +Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#32 IN dynamicpruning#33 +BroadcastExchange (80) ++- * ColumnarToRow (79) + +- CometFilter (78) + +- CometScan parquet spark_catalog.default.date_dim (77) -(85) Scan parquet spark_catalog.default.date_dim +(77) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#34, d_year#35] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(86) CometFilter +(78) CometFilter Input [2]: [d_date_sk#34, d_year#35] Condition : ((isnotnull(d_year#35) AND (d_year#35 = 2002)) AND isnotnull(d_date_sk#34)) -(87) ColumnarToRow [codegen id : 1] +(79) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#34, d_year#35] -(88) BroadcastExchange +(80) BroadcastExchange Input [2]: [d_date_sk#34, d_year#35] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] -Subquery:3 Hosting operator id = 43 Hosting Expression = ws_sold_date_sk#52 IN dynamicpruning#13 +Subquery:3 Hosting operator id = 39 Hosting Expression = ws_sold_date_sk#52 IN dynamicpruning#13 -Subquery:4 Hosting operator id = 63 Hosting Expression = ws_sold_date_sk#72 IN dynamicpruning#33 +Subquery:4 Hosting operator id = 57 Hosting Expression = ws_sold_date_sk#72 IN dynamicpruning#33 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt index 456639607..240d88bb7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q11/simplified.txt @@ -8,11 +8,38 @@ TakeOrderedAndProject [customer_preferred_cust_flag] BroadcastHashJoin [customer_id,customer_id] Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - RowToColumnar - WholeStageCodegen (1) + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_preferred_cust_flag,year_total,sum] + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + WholeStageCodegen (2) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] ColumnarToRow InputAdapter @@ -22,94 +49,59 @@ TakeOrderedAndProject [customer_preferred_cust_flag] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #8 WholeStageCodegen (1) ColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_preferred_cust_flag,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #10 WholeStageCodegen (5) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + WholeStageCodegen (4) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_year] #4 InputAdapter BroadcastExchange #13 WholeStageCodegen (7) HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + WholeStageCodegen (6) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt index a97ace27a..d7d56bf6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/explain.txt @@ -1,30 +1,26 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * Project (25) - +- Window (24) - +- * ColumnarToRow (23) - +- CometSort (22) - +- CometColumnarExchange (21) - +- RowToColumnar (20) - +- * HashAggregate (19) - +- * ColumnarToRow (18) - +- CometColumnarExchange (17) - +- RowToColumnar (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.web_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) +TakeOrderedAndProject (22) ++- * Project (21) + +- Window (20) + +- * Sort (19) + +- Exchange (18) + +- * HashAggregate (17) + +- Exchange (16) + +- * HashAggregate (15) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometBroadcastHashJoin (12) + :- CometProject (7) + : +- CometBroadcastHashJoin (6) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (11) + +- CometProject (10) + +- CometFilter (9) + +- CometScan parquet spark_catalog.default.date_dim (8) (1) Scan parquet spark_catalog.default.web_sales @@ -101,78 +97,66 @@ Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] Aggregate Attributes [1]: [sum#13] Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -(16) RowToColumnar +(16) Exchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 2] -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] - -(19) HashAggregate [codegen id : 2] +(17) HashAggregate [codegen id : 2] Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#15] Results [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#15,17,2) AS itemrevenue#16, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#15,17,2) AS _w0#17, i_item_id#6] -(20) RowToColumnar -Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] - -(21) CometColumnarExchange -Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(22) CometSort +(18) Exchange Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] -Arguments: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6], [i_class#9 ASC NULLS FIRST] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(23) ColumnarToRow [codegen id : 3] +(19) Sort [codegen id : 3] Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] +Arguments: [i_class#9 ASC NULLS FIRST], false, 0 -(24) Window +(20) Window Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] Arguments: [sum(_w0#17) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#9] -(25) Project [codegen id : 4] +(21) Project [codegen id : 4] Output [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#6] Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6, _we0#18] -(26) TakeOrderedAndProject +(22) TakeOrderedAndProject Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19, i_item_id#6] Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * ColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.date_dim (27) +BroadcastExchange (27) ++- * ColumnarToRow (26) + +- CometProject (25) + +- CometFilter (24) + +- CometScan parquet spark_catalog.default.date_dim (23) -(27) Scan parquet spark_catalog.default.date_dim +(23) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(24) CometFilter Input [2]: [d_date_sk#11, d_date#12] Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 1999-02-22)) AND (d_date#12 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(29) CometProject +(25) CometProject Input [2]: [d_date_sk#11, d_date#12] Arguments: [d_date_sk#11], [d_date_sk#11] -(30) ColumnarToRow [codegen id : 1] +(26) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#11] -(31) BroadcastExchange +(27) BroadcastExchange Input [1]: [d_date_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt index 9ca64cd0e..1640fa3b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q12/simplified.txt @@ -4,39 +4,35 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (3) - ColumnarToRow + Sort [i_class] InputAdapter - CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometColumnarExchange [i_class] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + Exchange [i_class] #1 + WholeStageCodegen (2) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] + InputAdapter + Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt index 7cb01ba03..e137f931f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/explain.txt @@ -1,39 +1,37 @@ == Physical Plan == -* HashAggregate (35) -+- * ColumnarToRow (34) - +- CometColumnarExchange (33) - +- RowToColumnar (32) - +- * HashAggregate (31) - +- * ColumnarToRow (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometProject (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.customer_address (8) - : : +- CometBroadcastExchange (17) - : : +- CometProject (16) - : : +- CometFilter (15) - : : +- CometScan parquet spark_catalog.default.date_dim (14) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometScan parquet spark_catalog.default.customer_demographics (20) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.household_demographics (25) +* HashAggregate (33) ++- Exchange (32) + +- * HashAggregate (31) + +- * ColumnarToRow (30) + +- CometProject (29) + +- CometBroadcastHashJoin (28) + :- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.store (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (17) + : : +- CometProject (16) + : : +- CometFilter (15) + : : +- CometScan parquet spark_catalog.default.date_dim (14) + : +- CometBroadcastExchange (22) + : +- CometFilter (21) + : +- CometScan parquet spark_catalog.default.customer_demographics (20) + +- CometBroadcastExchange (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.household_demographics (25) (1) Scan parquet spark_catalog.default.store_sales @@ -186,17 +184,11 @@ Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sal Aggregate Attributes [7]: [sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29] Results [7]: [sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36] -(32) RowToColumnar +(32) Exchange Input [7]: [sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] -(33) CometColumnarExchange -Input [7]: [sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(34) ColumnarToRow [codegen id : 2] -Input [7]: [sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36] - -(35) HashAggregate [codegen id : 2] +(33) HashAggregate [codegen id : 2] Input [7]: [sum#30, count#31, sum#32, count#33, sum#34, count#35, sum#36] Keys: [] Functions [4]: [avg(ss_quantity#5), avg(UnscaledValue(ss_ext_sales_price#7)), avg(UnscaledValue(ss_ext_wholesale_cost#8)), sum(UnscaledValue(ss_ext_wholesale_cost#8))] @@ -206,32 +198,32 @@ Results [4]: [avg(ss_quantity#5)#37 AS avg(ss_quantity)#41, cast((avg(UnscaledVa ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#10 IN dynamicpruning#11 -BroadcastExchange (40) -+- * ColumnarToRow (39) - +- CometProject (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.date_dim (36) +BroadcastExchange (38) ++- * ColumnarToRow (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan parquet spark_catalog.default.date_dim (34) -(36) Scan parquet spark_catalog.default.date_dim +(34) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_year#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(37) CometFilter +(35) CometFilter Input [2]: [d_date_sk#16, d_year#17] Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(38) CometProject +(36) CometProject Input [2]: [d_date_sk#16, d_year#17] Arguments: [d_date_sk#16], [d_date_sk#16] -(39) ColumnarToRow [codegen id : 1] +(37) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#16] -(40) BroadcastExchange +(38) BroadcastExchange Input [1]: [d_date_sk#16] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt index bbabebeea..cc52bb323 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q13/simplified.txt @@ -1,47 +1,45 @@ WholeStageCodegen (2) HashAggregate [sum,count,sum,count,sum,count,sum] [avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost)),avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum,sum,count,sum,count,sum,count,sum] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] - CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] - CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk,d_date_sk] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] - CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] - CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk] #3 - CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [ca_address_sk,ca_state] #4 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk,hd_dep_count] #7 - CometFilter [hd_demo_sk,hd_dep_count] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] + InputAdapter + Exchange #1 + WholeStageCodegen (1) + HashAggregate [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] [sum,count,sum,count,sum,count,sum,sum,count,sum,count,sum,count,sum] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] + CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] + CometProject [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk,d_date_sk] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] + CometProject [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk,s_store_sk] + CometFilter [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk] #3 + CometFilter [s_store_sk] + CometScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometProject [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [hd_demo_sk,hd_dep_count] #7 + CometFilter [hd_demo_sk,hd_dep_count] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt index 93d1685a5..e8d67e286 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/explain.txt @@ -1,116 +1,108 @@ == Physical Plan == -TakeOrderedAndProject (112) -+- * HashAggregate (111) - +- * ColumnarToRow (110) - +- CometColumnarExchange (109) - +- RowToColumnar (108) - +- * HashAggregate (107) - +- * Expand (106) - +- Union (105) - :- * Project (68) - : +- * Filter (67) - : +- * HashAggregate (66) - : +- * ColumnarToRow (65) - : +- CometColumnarExchange (64) - : +- RowToColumnar (63) - : +- * HashAggregate (62) - : +- * ColumnarToRow (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (54) - : : +- CometBroadcastHashJoin (53) - : : :- CometBroadcastHashJoin (47) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (46) - : : : +- CometProject (45) - : : : +- CometBroadcastHashJoin (44) - : : : :- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.item (3) - : : : +- CometBroadcastExchange (43) - : : : +- CometBroadcastHashJoin (42) - : : : :- CometHashAggregate (32) - : : : : +- CometColumnarExchange (31) - : : : : +- CometHashAggregate (30) - : : : : +- CometProject (29) - : : : : +- CometBroadcastHashJoin (28) - : : : : :- CometProject (26) - : : : : : +- CometBroadcastHashJoin (25) - : : : : : :- CometFilter (6) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) - : : : : : +- CometBroadcastExchange (24) - : : : : : +- CometBroadcastHashJoin (23) - : : : : : :- CometFilter (8) - : : : : : : +- CometScan parquet spark_catalog.default.item (7) - : : : : : +- CometBroadcastExchange (22) - : : : : : +- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometProject (15) - : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) - : : : : : : +- CometBroadcastExchange (13) - : : : : : : +- CometFilter (12) - : : : : : : +- CometScan parquet spark_catalog.default.item (11) - : : : : : +- CometBroadcastExchange (19) - : : : : : +- CometProject (18) - : : : : : +- CometFilter (17) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) - : : : : +- ReusedExchange (27) - : : : +- CometBroadcastExchange (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometProject (37) - : : : : +- CometBroadcastHashJoin (36) - : : : : :- CometFilter (34) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) - : : : : +- ReusedExchange (35) - : : : +- ReusedExchange (38) - : : +- CometBroadcastExchange (52) - : : +- CometBroadcastHashJoin (51) - : : :- CometFilter (49) - : : : +- CometScan parquet spark_catalog.default.item (48) - : : +- ReusedExchange (50) - : +- CometBroadcastExchange (58) - : +- CometProject (57) - : +- CometFilter (56) - : +- CometScan parquet spark_catalog.default.date_dim (55) - :- * Project (86) - : +- * Filter (85) - : +- * HashAggregate (84) - : +- * ColumnarToRow (83) - : +- CometColumnarExchange (82) - : +- RowToColumnar (81) - : +- * HashAggregate (80) - : +- * ColumnarToRow (79) - : +- CometProject (78) - : +- CometBroadcastHashJoin (77) - : :- CometProject (75) - : : +- CometBroadcastHashJoin (74) - : : :- CometBroadcastHashJoin (72) - : : : :- CometFilter (70) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (69) - : : : +- ReusedExchange (71) - : : +- ReusedExchange (73) - : +- ReusedExchange (76) - +- * Project (104) - +- * Filter (103) - +- * HashAggregate (102) - +- * ColumnarToRow (101) - +- CometColumnarExchange (100) - +- RowToColumnar (99) - +- * HashAggregate (98) - +- * ColumnarToRow (97) - +- CometProject (96) - +- CometBroadcastHashJoin (95) - :- CometProject (93) - : +- CometBroadcastHashJoin (92) - : :- CometBroadcastHashJoin (90) - : : :- CometFilter (88) - : : : +- CometScan parquet spark_catalog.default.web_sales (87) - : : +- ReusedExchange (89) - : +- ReusedExchange (91) - +- ReusedExchange (94) +TakeOrderedAndProject (104) ++- * HashAggregate (103) + +- Exchange (102) + +- * HashAggregate (101) + +- * Expand (100) + +- Union (99) + :- * Project (66) + : +- * Filter (65) + : +- * HashAggregate (64) + : +- Exchange (63) + : +- * HashAggregate (62) + : +- * ColumnarToRow (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometScan parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometScan parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometScan parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan parquet spark_catalog.default.date_dim (55) + :- * Project (82) + : +- * Filter (81) + : +- * HashAggregate (80) + : +- Exchange (79) + : +- * HashAggregate (78) + : +- * ColumnarToRow (77) + : +- CometProject (76) + : +- CometBroadcastHashJoin (75) + : :- CometProject (73) + : : +- CometBroadcastHashJoin (72) + : : :- CometBroadcastHashJoin (70) + : : : :- CometFilter (68) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (67) + : : : +- ReusedExchange (69) + : : +- ReusedExchange (71) + : +- ReusedExchange (74) + +- * Project (98) + +- * Filter (97) + +- * HashAggregate (96) + +- Exchange (95) + +- * HashAggregate (94) + +- * ColumnarToRow (93) + +- CometProject (92) + +- CometBroadcastHashJoin (91) + :- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometBroadcastHashJoin (86) + : : :- CometFilter (84) + : : : +- CometScan parquet spark_catalog.default.web_sales (83) + : : +- ReusedExchange (85) + : +- ReusedExchange (87) + +- ReusedExchange (90) (1) Scan parquet spark_catalog.default.store_sales @@ -262,9 +254,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -408,32 +400,26 @@ Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_pric Aggregate Attributes [3]: [sum#46, isEmpty#47, count#48] Results [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#49, isEmpty#50, count#51] -(63) RowToColumnar +(63) Exchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#49, isEmpty#50, count#51] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(64) CometColumnarExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#49, isEmpty#50, count#51] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(65) ColumnarToRow [codegen id : 2] -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#49, isEmpty#50, count#51] - -(66) HashAggregate [codegen id : 2] +(64) HashAggregate [codegen id : 2] Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#49, isEmpty#50, count#51] Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#52, count(1)#53] Results [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#52 AS sales#54, count(1)#53 AS number_sales#55] -(67) Filter [codegen id : 2] +(65) Filter [codegen id : 2] Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#54, number_sales#55] Condition : (isnotnull(sales#54) AND (cast(sales#54 as decimal(32,6)) > cast(Subquery scalar-subquery#56, [id=#57] as decimal(32,6)))) -(68) Project [codegen id : 2] +(66) Project [codegen id : 2] Output [6]: [sales#54, number_sales#55, store AS channel#58, i_brand_id#40 AS i_brand_id#59, i_class_id#41 AS i_class_id#60, i_category_id#42 AS i_category_id#61] Input [5]: [i_brand_id#40, i_class_id#41, i_category_id#42, sales#54, number_sales#55] -(69) Scan parquet spark_catalog.default.catalog_sales +(67) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#62, cs_quantity#63, cs_list_price#64, cs_sold_date_sk#65] Batched: true Location: InMemoryFileIndex [] @@ -441,78 +427,72 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#65), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(70) CometFilter +(68) CometFilter Input [4]: [cs_item_sk#62, cs_quantity#63, cs_list_price#64, cs_sold_date_sk#65] Condition : isnotnull(cs_item_sk#62) -(71) ReusedExchange [Reuses operator id: 46] +(69) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#67] -(72) CometBroadcastHashJoin +(70) CometBroadcastHashJoin Left output [4]: [cs_item_sk#62, cs_quantity#63, cs_list_price#64, cs_sold_date_sk#65] Right output [1]: [ss_item_sk#67] Arguments: [cs_item_sk#62], [ss_item_sk#67], LeftSemi, BuildRight -(73) ReusedExchange [Reuses operator id: 52] +(71) ReusedExchange [Reuses operator id: 52] Output [4]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71] -(74) CometBroadcastHashJoin +(72) CometBroadcastHashJoin Left output [4]: [cs_item_sk#62, cs_quantity#63, cs_list_price#64, cs_sold_date_sk#65] Right output [4]: [i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71] Arguments: [cs_item_sk#62], [i_item_sk#68], Inner, BuildRight -(75) CometProject +(73) CometProject Input [8]: [cs_item_sk#62, cs_quantity#63, cs_list_price#64, cs_sold_date_sk#65, i_item_sk#68, i_brand_id#69, i_class_id#70, i_category_id#71] Arguments: [cs_quantity#63, cs_list_price#64, cs_sold_date_sk#65, i_brand_id#69, i_class_id#70, i_category_id#71], [cs_quantity#63, cs_list_price#64, cs_sold_date_sk#65, i_brand_id#69, i_class_id#70, i_category_id#71] -(76) ReusedExchange [Reuses operator id: 58] +(74) ReusedExchange [Reuses operator id: 58] Output [1]: [d_date_sk#72] -(77) CometBroadcastHashJoin +(75) CometBroadcastHashJoin Left output [6]: [cs_quantity#63, cs_list_price#64, cs_sold_date_sk#65, i_brand_id#69, i_class_id#70, i_category_id#71] Right output [1]: [d_date_sk#72] Arguments: [cs_sold_date_sk#65], [d_date_sk#72], Inner, BuildRight -(78) CometProject +(76) CometProject Input [7]: [cs_quantity#63, cs_list_price#64, cs_sold_date_sk#65, i_brand_id#69, i_class_id#70, i_category_id#71, d_date_sk#72] Arguments: [cs_quantity#63, cs_list_price#64, i_brand_id#69, i_class_id#70, i_category_id#71], [cs_quantity#63, cs_list_price#64, i_brand_id#69, i_class_id#70, i_category_id#71] -(79) ColumnarToRow [codegen id : 3] +(77) ColumnarToRow [codegen id : 3] Input [5]: [cs_quantity#63, cs_list_price#64, i_brand_id#69, i_class_id#70, i_category_id#71] -(80) HashAggregate [codegen id : 3] +(78) HashAggregate [codegen id : 3] Input [5]: [cs_quantity#63, cs_list_price#64, i_brand_id#69, i_class_id#70, i_category_id#71] Keys [3]: [i_brand_id#69, i_class_id#70, i_category_id#71] Functions [2]: [partial_sum((cast(cs_quantity#63 as decimal(10,0)) * cs_list_price#64)), partial_count(1)] Aggregate Attributes [3]: [sum#73, isEmpty#74, count#75] Results [6]: [i_brand_id#69, i_class_id#70, i_category_id#71, sum#76, isEmpty#77, count#78] -(81) RowToColumnar -Input [6]: [i_brand_id#69, i_class_id#70, i_category_id#71, sum#76, isEmpty#77, count#78] - -(82) CometColumnarExchange +(79) Exchange Input [6]: [i_brand_id#69, i_class_id#70, i_category_id#71, sum#76, isEmpty#77, count#78] -Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_brand_id#69, i_class_id#70, i_category_id#71, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(83) ColumnarToRow [codegen id : 4] -Input [6]: [i_brand_id#69, i_class_id#70, i_category_id#71, sum#76, isEmpty#77, count#78] - -(84) HashAggregate [codegen id : 4] +(80) HashAggregate [codegen id : 4] Input [6]: [i_brand_id#69, i_class_id#70, i_category_id#71, sum#76, isEmpty#77, count#78] Keys [3]: [i_brand_id#69, i_class_id#70, i_category_id#71] Functions [2]: [sum((cast(cs_quantity#63 as decimal(10,0)) * cs_list_price#64)), count(1)] Aggregate Attributes [2]: [sum((cast(cs_quantity#63 as decimal(10,0)) * cs_list_price#64))#79, count(1)#80] Results [5]: [i_brand_id#69, i_class_id#70, i_category_id#71, sum((cast(cs_quantity#63 as decimal(10,0)) * cs_list_price#64))#79 AS sales#81, count(1)#80 AS number_sales#82] -(85) Filter [codegen id : 4] +(81) Filter [codegen id : 4] Input [5]: [i_brand_id#69, i_class_id#70, i_category_id#71, sales#81, number_sales#82] Condition : (isnotnull(sales#81) AND (cast(sales#81 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#56, [id=#57] as decimal(32,6)))) -(86) Project [codegen id : 4] +(82) Project [codegen id : 4] Output [6]: [sales#81, number_sales#82, catalog AS channel#83, i_brand_id#69, i_class_id#70, i_category_id#71] Input [5]: [i_brand_id#69, i_class_id#70, i_category_id#71, sales#81, number_sales#82] -(87) Scan parquet spark_catalog.default.web_sales +(83) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#84, ws_quantity#85, ws_list_price#86, ws_sold_date_sk#87] Batched: true Location: InMemoryFileIndex [] @@ -520,287 +500,267 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#87), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(88) CometFilter +(84) CometFilter Input [4]: [ws_item_sk#84, ws_quantity#85, ws_list_price#86, ws_sold_date_sk#87] Condition : isnotnull(ws_item_sk#84) -(89) ReusedExchange [Reuses operator id: 46] +(85) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#89] -(90) CometBroadcastHashJoin +(86) CometBroadcastHashJoin Left output [4]: [ws_item_sk#84, ws_quantity#85, ws_list_price#86, ws_sold_date_sk#87] Right output [1]: [ss_item_sk#89] Arguments: [ws_item_sk#84], [ss_item_sk#89], LeftSemi, BuildRight -(91) ReusedExchange [Reuses operator id: 52] +(87) ReusedExchange [Reuses operator id: 52] Output [4]: [i_item_sk#90, i_brand_id#91, i_class_id#92, i_category_id#93] -(92) CometBroadcastHashJoin +(88) CometBroadcastHashJoin Left output [4]: [ws_item_sk#84, ws_quantity#85, ws_list_price#86, ws_sold_date_sk#87] Right output [4]: [i_item_sk#90, i_brand_id#91, i_class_id#92, i_category_id#93] Arguments: [ws_item_sk#84], [i_item_sk#90], Inner, BuildRight -(93) CometProject +(89) CometProject Input [8]: [ws_item_sk#84, ws_quantity#85, ws_list_price#86, ws_sold_date_sk#87, i_item_sk#90, i_brand_id#91, i_class_id#92, i_category_id#93] Arguments: [ws_quantity#85, ws_list_price#86, ws_sold_date_sk#87, i_brand_id#91, i_class_id#92, i_category_id#93], [ws_quantity#85, ws_list_price#86, ws_sold_date_sk#87, i_brand_id#91, i_class_id#92, i_category_id#93] -(94) ReusedExchange [Reuses operator id: 58] +(90) ReusedExchange [Reuses operator id: 58] Output [1]: [d_date_sk#94] -(95) CometBroadcastHashJoin +(91) CometBroadcastHashJoin Left output [6]: [ws_quantity#85, ws_list_price#86, ws_sold_date_sk#87, i_brand_id#91, i_class_id#92, i_category_id#93] Right output [1]: [d_date_sk#94] Arguments: [ws_sold_date_sk#87], [d_date_sk#94], Inner, BuildRight -(96) CometProject +(92) CometProject Input [7]: [ws_quantity#85, ws_list_price#86, ws_sold_date_sk#87, i_brand_id#91, i_class_id#92, i_category_id#93, d_date_sk#94] Arguments: [ws_quantity#85, ws_list_price#86, i_brand_id#91, i_class_id#92, i_category_id#93], [ws_quantity#85, ws_list_price#86, i_brand_id#91, i_class_id#92, i_category_id#93] -(97) ColumnarToRow [codegen id : 5] +(93) ColumnarToRow [codegen id : 5] Input [5]: [ws_quantity#85, ws_list_price#86, i_brand_id#91, i_class_id#92, i_category_id#93] -(98) HashAggregate [codegen id : 5] +(94) HashAggregate [codegen id : 5] Input [5]: [ws_quantity#85, ws_list_price#86, i_brand_id#91, i_class_id#92, i_category_id#93] Keys [3]: [i_brand_id#91, i_class_id#92, i_category_id#93] Functions [2]: [partial_sum((cast(ws_quantity#85 as decimal(10,0)) * ws_list_price#86)), partial_count(1)] Aggregate Attributes [3]: [sum#95, isEmpty#96, count#97] Results [6]: [i_brand_id#91, i_class_id#92, i_category_id#93, sum#98, isEmpty#99, count#100] -(99) RowToColumnar -Input [6]: [i_brand_id#91, i_class_id#92, i_category_id#93, sum#98, isEmpty#99, count#100] - -(100) CometColumnarExchange +(95) Exchange Input [6]: [i_brand_id#91, i_class_id#92, i_category_id#93, sum#98, isEmpty#99, count#100] -Arguments: hashpartitioning(i_brand_id#91, i_class_id#92, i_category_id#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_brand_id#91, i_class_id#92, i_category_id#93, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(101) ColumnarToRow [codegen id : 6] -Input [6]: [i_brand_id#91, i_class_id#92, i_category_id#93, sum#98, isEmpty#99, count#100] - -(102) HashAggregate [codegen id : 6] +(96) HashAggregate [codegen id : 6] Input [6]: [i_brand_id#91, i_class_id#92, i_category_id#93, sum#98, isEmpty#99, count#100] Keys [3]: [i_brand_id#91, i_class_id#92, i_category_id#93] Functions [2]: [sum((cast(ws_quantity#85 as decimal(10,0)) * ws_list_price#86)), count(1)] Aggregate Attributes [2]: [sum((cast(ws_quantity#85 as decimal(10,0)) * ws_list_price#86))#101, count(1)#102] Results [5]: [i_brand_id#91, i_class_id#92, i_category_id#93, sum((cast(ws_quantity#85 as decimal(10,0)) * ws_list_price#86))#101 AS sales#103, count(1)#102 AS number_sales#104] -(103) Filter [codegen id : 6] +(97) Filter [codegen id : 6] Input [5]: [i_brand_id#91, i_class_id#92, i_category_id#93, sales#103, number_sales#104] Condition : (isnotnull(sales#103) AND (cast(sales#103 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#56, [id=#57] as decimal(32,6)))) -(104) Project [codegen id : 6] +(98) Project [codegen id : 6] Output [6]: [sales#103, number_sales#104, web AS channel#105, i_brand_id#91, i_class_id#92, i_category_id#93] Input [5]: [i_brand_id#91, i_class_id#92, i_category_id#93, sales#103, number_sales#104] -(105) Union +(99) Union -(106) Expand [codegen id : 7] +(100) Expand [codegen id : 7] Input [6]: [sales#54, number_sales#55, channel#58, i_brand_id#59, i_class_id#60, i_category_id#61] Arguments: [[sales#54, number_sales#55, channel#58, i_brand_id#59, i_class_id#60, i_category_id#61, 0], [sales#54, number_sales#55, channel#58, i_brand_id#59, i_class_id#60, null, 1], [sales#54, number_sales#55, channel#58, i_brand_id#59, null, null, 3], [sales#54, number_sales#55, channel#58, null, null, null, 7], [sales#54, number_sales#55, null, null, null, null, 15]], [sales#54, number_sales#55, channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110] -(107) HashAggregate [codegen id : 7] +(101) HashAggregate [codegen id : 7] Input [7]: [sales#54, number_sales#55, channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110] Keys [5]: [channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110] Functions [2]: [partial_sum(sales#54), partial_sum(number_sales#55)] Aggregate Attributes [3]: [sum#111, isEmpty#112, sum#113] Results [8]: [channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110, sum#114, isEmpty#115, sum#116] -(108) RowToColumnar -Input [8]: [channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110, sum#114, isEmpty#115, sum#116] - -(109) CometColumnarExchange +(102) Exchange Input [8]: [channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110, sum#114, isEmpty#115, sum#116] -Arguments: hashpartitioning(channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(110) ColumnarToRow [codegen id : 8] -Input [8]: [channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110, sum#114, isEmpty#115, sum#116] - -(111) HashAggregate [codegen id : 8] +(103) HashAggregate [codegen id : 8] Input [8]: [channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110, sum#114, isEmpty#115, sum#116] Keys [5]: [channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, spark_grouping_id#110] Functions [2]: [sum(sales#54), sum(number_sales#55)] Aggregate Attributes [2]: [sum(sales#54)#117, sum(number_sales#55)#118] Results [6]: [channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, sum(sales#54)#117 AS sum(sales)#119, sum(number_sales#55)#118 AS sum(number_sales)#120] -(112) TakeOrderedAndProject +(104) TakeOrderedAndProject Input [6]: [channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, sum(sales)#119, sum(number_sales)#120] Arguments: 100, [channel#106 ASC NULLS FIRST, i_brand_id#107 ASC NULLS FIRST, i_class_id#108 ASC NULLS FIRST, i_category_id#109 ASC NULLS FIRST], [channel#106, i_brand_id#107, i_class_id#108, i_category_id#109, sum(sales)#119, sum(number_sales)#120] ===== Subqueries ===== -Subquery:1 Hosting operator id = 67 Hosting Expression = Subquery scalar-subquery#56, [id=#57] -* HashAggregate (131) -+- * ColumnarToRow (130) - +- CometColumnarExchange (129) - +- RowToColumnar (128) - +- * HashAggregate (127) - +- * ColumnarToRow (126) - +- CometUnion (125) - :- CometProject (116) - : +- CometBroadcastHashJoin (115) - : :- CometScan parquet spark_catalog.default.store_sales (113) - : +- ReusedExchange (114) - :- CometProject (120) - : +- CometBroadcastHashJoin (119) - : :- CometScan parquet spark_catalog.default.catalog_sales (117) - : +- ReusedExchange (118) - +- CometProject (124) - +- CometBroadcastHashJoin (123) - :- CometScan parquet spark_catalog.default.web_sales (121) - +- ReusedExchange (122) - - -(113) Scan parquet spark_catalog.default.store_sales +Subquery:1 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#56, [id=#57] +* HashAggregate (121) ++- Exchange (120) + +- * HashAggregate (119) + +- * ColumnarToRow (118) + +- CometUnion (117) + :- CometProject (108) + : +- CometBroadcastHashJoin (107) + : :- CometScan parquet spark_catalog.default.store_sales (105) + : +- ReusedExchange (106) + :- CometProject (112) + : +- CometBroadcastHashJoin (111) + : :- CometScan parquet spark_catalog.default.catalog_sales (109) + : +- ReusedExchange (110) + +- CometProject (116) + +- CometBroadcastHashJoin (115) + :- CometScan parquet spark_catalog.default.web_sales (113) + +- ReusedExchange (114) + + +(105) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#121, ss_list_price#122, ss_sold_date_sk#123] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#123), dynamicpruningexpression(ss_sold_date_sk#123 IN dynamicpruning#124)] ReadSchema: struct -(114) ReusedExchange [Reuses operator id: 19] +(106) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#125] -(115) CometBroadcastHashJoin +(107) CometBroadcastHashJoin Left output [3]: [ss_quantity#121, ss_list_price#122, ss_sold_date_sk#123] Right output [1]: [d_date_sk#125] Arguments: [ss_sold_date_sk#123], [d_date_sk#125], Inner, BuildRight -(116) CometProject +(108) CometProject Input [4]: [ss_quantity#121, ss_list_price#122, ss_sold_date_sk#123, d_date_sk#125] Arguments: [quantity#126, list_price#127], [ss_quantity#121 AS quantity#126, ss_list_price#122 AS list_price#127] -(117) Scan parquet spark_catalog.default.catalog_sales +(109) Scan parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#128, cs_list_price#129, cs_sold_date_sk#130] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#130), dynamicpruningexpression(cs_sold_date_sk#130 IN dynamicpruning#131)] ReadSchema: struct -(118) ReusedExchange [Reuses operator id: 19] +(110) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#132] -(119) CometBroadcastHashJoin +(111) CometBroadcastHashJoin Left output [3]: [cs_quantity#128, cs_list_price#129, cs_sold_date_sk#130] Right output [1]: [d_date_sk#132] Arguments: [cs_sold_date_sk#130], [d_date_sk#132], Inner, BuildRight -(120) CometProject +(112) CometProject Input [4]: [cs_quantity#128, cs_list_price#129, cs_sold_date_sk#130, d_date_sk#132] Arguments: [quantity#133, list_price#134], [cs_quantity#128 AS quantity#133, cs_list_price#129 AS list_price#134] -(121) Scan parquet spark_catalog.default.web_sales +(113) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#135, ws_list_price#136, ws_sold_date_sk#137] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#137), dynamicpruningexpression(ws_sold_date_sk#137 IN dynamicpruning#138)] ReadSchema: struct -(122) ReusedExchange [Reuses operator id: 19] +(114) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#139] -(123) CometBroadcastHashJoin +(115) CometBroadcastHashJoin Left output [3]: [ws_quantity#135, ws_list_price#136, ws_sold_date_sk#137] Right output [1]: [d_date_sk#139] Arguments: [ws_sold_date_sk#137], [d_date_sk#139], Inner, BuildRight -(124) CometProject +(116) CometProject Input [4]: [ws_quantity#135, ws_list_price#136, ws_sold_date_sk#137, d_date_sk#139] Arguments: [quantity#140, list_price#141], [ws_quantity#135 AS quantity#140, ws_list_price#136 AS list_price#141] -(125) CometUnion +(117) CometUnion Child 0 Input [2]: [quantity#126, list_price#127] Child 1 Input [2]: [quantity#133, list_price#134] Child 2 Input [2]: [quantity#140, list_price#141] -(126) ColumnarToRow [codegen id : 1] +(118) ColumnarToRow [codegen id : 1] Input [2]: [quantity#126, list_price#127] -(127) HashAggregate [codegen id : 1] +(119) HashAggregate [codegen id : 1] Input [2]: [quantity#126, list_price#127] Keys: [] Functions [1]: [partial_avg((cast(quantity#126 as decimal(10,0)) * list_price#127))] Aggregate Attributes [2]: [sum#142, count#143] Results [2]: [sum#144, count#145] -(128) RowToColumnar -Input [2]: [sum#144, count#145] - -(129) CometColumnarExchange +(120) Exchange Input [2]: [sum#144, count#145] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(130) ColumnarToRow [codegen id : 2] -Input [2]: [sum#144, count#145] - -(131) HashAggregate [codegen id : 2] +(121) HashAggregate [codegen id : 2] Input [2]: [sum#144, count#145] Keys: [] Functions [1]: [avg((cast(quantity#126 as decimal(10,0)) * list_price#127))] Aggregate Attributes [1]: [avg((cast(quantity#126 as decimal(10,0)) * list_price#127))#146] Results [1]: [avg((cast(quantity#126 as decimal(10,0)) * list_price#127))#146 AS average_sales#147] -Subquery:2 Hosting operator id = 113 Hosting Expression = ss_sold_date_sk#123 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 105 Hosting Expression = ss_sold_date_sk#123 IN dynamicpruning#12 -Subquery:3 Hosting operator id = 117 Hosting Expression = cs_sold_date_sk#130 IN dynamicpruning#12 +Subquery:3 Hosting operator id = 109 Hosting Expression = cs_sold_date_sk#130 IN dynamicpruning#12 -Subquery:4 Hosting operator id = 121 Hosting Expression = ws_sold_date_sk#137 IN dynamicpruning#12 +Subquery:4 Hosting operator id = 113 Hosting Expression = ws_sold_date_sk#137 IN dynamicpruning#12 Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (136) -+- * ColumnarToRow (135) - +- CometProject (134) - +- CometFilter (133) - +- CometScan parquet spark_catalog.default.date_dim (132) +BroadcastExchange (126) ++- * ColumnarToRow (125) + +- CometProject (124) + +- CometFilter (123) + +- CometScan parquet spark_catalog.default.date_dim (122) -(132) Scan parquet spark_catalog.default.date_dim +(122) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(133) CometFilter +(123) CometFilter Input [3]: [d_date_sk#43, d_year#44, d_moy#45] Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2001)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) -(134) CometProject +(124) CometProject Input [3]: [d_date_sk#43, d_year#44, d_moy#45] Arguments: [d_date_sk#43], [d_date_sk#43] -(135) ColumnarToRow [codegen id : 1] +(125) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#43] -(136) BroadcastExchange +(126) BroadcastExchange Input [1]: [d_date_sk#43] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] Subquery:6 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (141) -+- * ColumnarToRow (140) - +- CometProject (139) - +- CometFilter (138) - +- CometScan parquet spark_catalog.default.date_dim (137) +BroadcastExchange (131) ++- * ColumnarToRow (130) + +- CometProject (129) + +- CometFilter (128) + +- CometScan parquet spark_catalog.default.date_dim (127) -(137) Scan parquet spark_catalog.default.date_dim +(127) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#148] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(138) CometFilter +(128) CometFilter Input [2]: [d_date_sk#26, d_year#148] Condition : (((isnotnull(d_year#148) AND (d_year#148 >= 1999)) AND (d_year#148 <= 2001)) AND isnotnull(d_date_sk#26)) -(139) CometProject +(129) CometProject Input [2]: [d_date_sk#26, d_year#148] Arguments: [d_date_sk#26], [d_date_sk#26] -(140) ColumnarToRow [codegen id : 1] +(130) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#26] -(141) BroadcastExchange +(131) BroadcastExchange Input [1]: [d_date_sk#26] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] @@ -808,12 +768,12 @@ Subquery:7 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dy Subquery:8 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 -Subquery:9 Hosting operator id = 85 Hosting Expression = ReusedSubquery Subquery scalar-subquery#56, [id=#57] +Subquery:9 Hosting operator id = 81 Hosting Expression = ReusedSubquery Subquery scalar-subquery#56, [id=#57] -Subquery:10 Hosting operator id = 69 Hosting Expression = cs_sold_date_sk#65 IN dynamicpruning#5 +Subquery:10 Hosting operator id = 67 Hosting Expression = cs_sold_date_sk#65 IN dynamicpruning#5 -Subquery:11 Hosting operator id = 103 Hosting Expression = ReusedSubquery Subquery scalar-subquery#56, [id=#57] +Subquery:11 Hosting operator id = 97 Hosting Expression = ReusedSubquery Subquery scalar-subquery#56, [id=#57] -Subquery:12 Hosting operator id = 87 Hosting Expression = ws_sold_date_sk#87 IN dynamicpruning#5 +Subquery:12 Hosting operator id = 83 Hosting Expression = ws_sold_date_sk#87 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt index 9b8f32e55..e0e466c18 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14a/simplified.txt @@ -1,177 +1,167 @@ TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] WholeStageCodegen (8) HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum(sales),sum(number_sales),sum,isEmpty,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - Expand [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] - InputAdapter - Union - WholeStageCodegen (2) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - Subquery #3 - WholeStageCodegen (2) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] + InputAdapter + Exchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 + WholeStageCodegen (7) + HashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] + Expand [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] + InputAdapter + Union + WholeStageCodegen (2) + Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] + Filter [sales] + Subquery #3 + WholeStageCodegen (2) + HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] + InputAdapter + Exchange #15 + WholeStageCodegen (1) + HashAggregate [quantity,list_price] [sum,count,sum,count] + ColumnarToRow + InputAdapter + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk] #11 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk] #11 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk] #11 + HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),sales,number_sales,sum,isEmpty,count] + InputAdapter + Exchange [i_brand_id,i_class_id,i_category_id] #2 + WholeStageCodegen (1) + HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] ColumnarToRow InputAdapter - CometColumnarExchange #15 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [quantity,list_price] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometUnion [quantity,list_price] - CometProject [ss_quantity,ss_list_price] [quantity,list_price] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 - CometProject [cs_quantity,cs_list_price] [quantity,list_price] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 - CometProject [ws_quantity,ws_list_price] [quantity,list_price] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #11 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk] #4 - CometProject [i_item_sk] [ss_item_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #5 - CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #6 - CometHashAggregate [brand_id,class_id,category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] - CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #7 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #11 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #11 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #12 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 - ReusedExchange [d_date_sk] #11 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #4 - CometBroadcastExchange [d_date_sk] #14 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (4) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #3 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #16 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] - CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #4 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 - ReusedExchange [d_date_sk] #14 - WholeStageCodegen (6) - Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] - Filter [sales] - ReusedSubquery [average_sales] #3 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),sales,number_sales,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #17 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] - CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #4 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 - ReusedExchange [d_date_sk] #14 + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk] #4 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #5 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #6 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #7 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #8 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #9 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #11 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #11 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #12 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #10 + ReusedExchange [d_date_sk] #11 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #4 + CometBroadcastExchange [d_date_sk] #14 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (4) + Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] + Filter [sales] + ReusedSubquery [average_sales] #3 + HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),sales,number_sales,sum,isEmpty,count] + InputAdapter + Exchange [i_brand_id,i_class_id,i_category_id] #16 + WholeStageCodegen (3) + HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count] + ColumnarToRow + InputAdapter + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 + ReusedExchange [d_date_sk] #14 + WholeStageCodegen (6) + Project [sales,number_sales,i_brand_id,i_class_id,i_category_id] + Filter [sales] + ReusedSubquery [average_sales] #3 + HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),sales,number_sales,sum,isEmpty,count] + InputAdapter + Exchange [i_brand_id,i_class_id,i_category_id] #17 + WholeStageCodegen (5) + HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count] + ColumnarToRow + InputAdapter + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [ss_item_sk] #4 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #13 + ReusedExchange [d_date_sk] #14 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt index ce4cf849b..bbdb4144f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/explain.txt @@ -1,94 +1,90 @@ == Physical Plan == -TakeOrderedAndProject (90) -+- * BroadcastHashJoin Inner BuildRight (89) - :- * Filter (67) - : +- * HashAggregate (66) - : +- * ColumnarToRow (65) - : +- CometColumnarExchange (64) - : +- RowToColumnar (63) - : +- * HashAggregate (62) - : +- * ColumnarToRow (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (54) - : : +- CometBroadcastHashJoin (53) - : : :- CometBroadcastHashJoin (47) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (46) - : : : +- CometProject (45) - : : : +- CometBroadcastHashJoin (44) - : : : :- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.item (3) - : : : +- CometBroadcastExchange (43) - : : : +- CometBroadcastHashJoin (42) - : : : :- CometHashAggregate (32) - : : : : +- CometColumnarExchange (31) - : : : : +- CometHashAggregate (30) - : : : : +- CometProject (29) - : : : : +- CometBroadcastHashJoin (28) - : : : : :- CometProject (26) - : : : : : +- CometBroadcastHashJoin (25) - : : : : : :- CometFilter (6) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) - : : : : : +- CometBroadcastExchange (24) - : : : : : +- CometBroadcastHashJoin (23) - : : : : : :- CometFilter (8) - : : : : : : +- CometScan parquet spark_catalog.default.item (7) - : : : : : +- CometBroadcastExchange (22) - : : : : : +- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometProject (15) - : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) - : : : : : : +- CometBroadcastExchange (13) - : : : : : : +- CometFilter (12) - : : : : : : +- CometScan parquet spark_catalog.default.item (11) - : : : : : +- CometBroadcastExchange (19) - : : : : : +- CometProject (18) - : : : : : +- CometFilter (17) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) - : : : : +- ReusedExchange (27) - : : : +- CometBroadcastExchange (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometProject (37) - : : : : +- CometBroadcastHashJoin (36) - : : : : :- CometFilter (34) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) - : : : : +- ReusedExchange (35) - : : : +- ReusedExchange (38) - : : +- CometBroadcastExchange (52) - : : +- CometBroadcastHashJoin (51) - : : :- CometFilter (49) - : : : +- CometScan parquet spark_catalog.default.item (48) - : : +- ReusedExchange (50) - : +- CometBroadcastExchange (58) - : +- CometProject (57) - : +- CometFilter (56) - : +- CometScan parquet spark_catalog.default.date_dim (55) - +- BroadcastExchange (88) - +- * Filter (87) - +- * HashAggregate (86) - +- * ColumnarToRow (85) - +- CometColumnarExchange (84) - +- RowToColumnar (83) - +- * HashAggregate (82) - +- * ColumnarToRow (81) - +- CometProject (80) - +- CometBroadcastHashJoin (79) - :- CometProject (74) - : +- CometBroadcastHashJoin (73) - : :- CometBroadcastHashJoin (71) - : : :- CometFilter (69) - : : : +- CometScan parquet spark_catalog.default.store_sales (68) - : : +- ReusedExchange (70) - : +- ReusedExchange (72) - +- CometBroadcastExchange (78) - +- CometProject (77) - +- CometFilter (76) - +- CometScan parquet spark_catalog.default.date_dim (75) +TakeOrderedAndProject (86) ++- * BroadcastHashJoin Inner BuildRight (85) + :- * Filter (65) + : +- * HashAggregate (64) + : +- Exchange (63) + : +- * HashAggregate (62) + : +- * ColumnarToRow (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometScan parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometScan parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometScan parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan parquet spark_catalog.default.date_dim (55) + +- BroadcastExchange (84) + +- * Filter (83) + +- * HashAggregate (82) + +- Exchange (81) + +- * HashAggregate (80) + +- * ColumnarToRow (79) + +- CometProject (78) + +- CometBroadcastHashJoin (77) + :- CometProject (72) + : +- CometBroadcastHashJoin (71) + : :- CometBroadcastHashJoin (69) + : : :- CometFilter (67) + : : : +- CometScan parquet spark_catalog.default.store_sales (66) + : : +- ReusedExchange (68) + : +- ReusedExchange (70) + +- CometBroadcastExchange (76) + +- CometProject (75) + +- CometFilter (74) + +- CometScan parquet spark_catalog.default.date_dim (73) (1) Scan parquet spark_catalog.default.store_sales @@ -240,9 +236,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -386,28 +382,22 @@ Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_pric Aggregate Attributes [3]: [sum#47, isEmpty#48, count#49] Results [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#50, isEmpty#51, count#52] -(63) RowToColumnar +(63) Exchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#50, isEmpty#51, count#52] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(64) CometColumnarExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#50, isEmpty#51, count#52] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(65) ColumnarToRow [codegen id : 4] -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#50, isEmpty#51, count#52] - -(66) HashAggregate [codegen id : 4] +(64) HashAggregate [codegen id : 4] Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#50, isEmpty#51, count#52] Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#53, count(1)#54] Results [6]: [store AS channel#55, i_brand_id#40, i_class_id#41, i_category_id#42, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#53 AS sales#56, count(1)#54 AS number_sales#57] -(67) Filter [codegen id : 4] +(65) Filter [codegen id : 4] Input [6]: [channel#55, i_brand_id#40, i_class_id#41, i_category_id#42, sales#56, number_sales#57] Condition : (isnotnull(sales#56) AND (cast(sales#56 as decimal(32,6)) > cast(Subquery scalar-subquery#58, [id=#59] as decimal(32,6)))) -(68) Scan parquet spark_catalog.default.store_sales +(66) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] @@ -415,306 +405,292 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#63), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(69) CometFilter +(67) CometFilter Input [4]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63] Condition : isnotnull(ss_item_sk#60) -(70) ReusedExchange [Reuses operator id: 46] +(68) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#65] -(71) CometBroadcastHashJoin +(69) CometBroadcastHashJoin Left output [4]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63] Right output [1]: [ss_item_sk#65] Arguments: [ss_item_sk#60], [ss_item_sk#65], LeftSemi, BuildRight -(72) ReusedExchange [Reuses operator id: 52] +(70) ReusedExchange [Reuses operator id: 52] Output [4]: [i_item_sk#66, i_brand_id#67, i_class_id#68, i_category_id#69] -(73) CometBroadcastHashJoin +(71) CometBroadcastHashJoin Left output [4]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63] Right output [4]: [i_item_sk#66, i_brand_id#67, i_class_id#68, i_category_id#69] Arguments: [ss_item_sk#60], [i_item_sk#66], Inner, BuildRight -(74) CometProject +(72) CometProject Input [8]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_item_sk#66, i_brand_id#67, i_class_id#68, i_category_id#69] Arguments: [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69], [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69] -(75) Scan parquet spark_catalog.default.date_dim +(73) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#70, d_week_seq#71] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#72), IsNotNull(d_date_sk)] ReadSchema: struct -(76) CometFilter +(74) CometFilter Input [2]: [d_date_sk#70, d_week_seq#71] Condition : ((isnotnull(d_week_seq#71) AND (d_week_seq#71 = ReusedSubquery Subquery scalar-subquery#72, [id=#73])) AND isnotnull(d_date_sk#70)) -(77) CometProject +(75) CometProject Input [2]: [d_date_sk#70, d_week_seq#71] Arguments: [d_date_sk#70], [d_date_sk#70] -(78) CometBroadcastExchange +(76) CometBroadcastExchange Input [1]: [d_date_sk#70] Arguments: [d_date_sk#70] -(79) CometBroadcastHashJoin +(77) CometBroadcastHashJoin Left output [6]: [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69] Right output [1]: [d_date_sk#70] Arguments: [ss_sold_date_sk#63], [d_date_sk#70], Inner, BuildRight -(80) CometProject +(78) CometProject Input [7]: [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69, d_date_sk#70] Arguments: [ss_quantity#61, ss_list_price#62, i_brand_id#67, i_class_id#68, i_category_id#69], [ss_quantity#61, ss_list_price#62, i_brand_id#67, i_class_id#68, i_category_id#69] -(81) ColumnarToRow [codegen id : 2] +(79) ColumnarToRow [codegen id : 2] Input [5]: [ss_quantity#61, ss_list_price#62, i_brand_id#67, i_class_id#68, i_category_id#69] -(82) HashAggregate [codegen id : 2] +(80) HashAggregate [codegen id : 2] Input [5]: [ss_quantity#61, ss_list_price#62, i_brand_id#67, i_class_id#68, i_category_id#69] Keys [3]: [i_brand_id#67, i_class_id#68, i_category_id#69] Functions [2]: [partial_sum((cast(ss_quantity#61 as decimal(10,0)) * ss_list_price#62)), partial_count(1)] Aggregate Attributes [3]: [sum#74, isEmpty#75, count#76] Results [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#77, isEmpty#78, count#79] -(83) RowToColumnar +(81) Exchange Input [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#77, isEmpty#78, count#79] +Arguments: hashpartitioning(i_brand_id#67, i_class_id#68, i_category_id#69, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(84) CometColumnarExchange -Input [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#77, isEmpty#78, count#79] -Arguments: hashpartitioning(i_brand_id#67, i_class_id#68, i_category_id#69, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(85) ColumnarToRow [codegen id : 3] -Input [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#77, isEmpty#78, count#79] - -(86) HashAggregate [codegen id : 3] +(82) HashAggregate [codegen id : 3] Input [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#77, isEmpty#78, count#79] Keys [3]: [i_brand_id#67, i_class_id#68, i_category_id#69] Functions [2]: [sum((cast(ss_quantity#61 as decimal(10,0)) * ss_list_price#62)), count(1)] Aggregate Attributes [2]: [sum((cast(ss_quantity#61 as decimal(10,0)) * ss_list_price#62))#80, count(1)#81] Results [6]: [store AS channel#82, i_brand_id#67, i_class_id#68, i_category_id#69, sum((cast(ss_quantity#61 as decimal(10,0)) * ss_list_price#62))#80 AS sales#83, count(1)#81 AS number_sales#84] -(87) Filter [codegen id : 3] +(83) Filter [codegen id : 3] Input [6]: [channel#82, i_brand_id#67, i_class_id#68, i_category_id#69, sales#83, number_sales#84] Condition : (isnotnull(sales#83) AND (cast(sales#83 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#58, [id=#59] as decimal(32,6)))) -(88) BroadcastExchange +(84) BroadcastExchange Input [6]: [channel#82, i_brand_id#67, i_class_id#68, i_category_id#69, sales#83, number_sales#84] Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [plan_id=4] -(89) BroadcastHashJoin [codegen id : 4] +(85) BroadcastHashJoin [codegen id : 4] Left keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Right keys [3]: [i_brand_id#67, i_class_id#68, i_category_id#69] Join type: Inner Join condition: None -(90) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [12]: [channel#55, i_brand_id#40, i_class_id#41, i_category_id#42, sales#56, number_sales#57, channel#82, i_brand_id#67, i_class_id#68, i_category_id#69, sales#83, number_sales#84] Arguments: 100, [i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#55, i_brand_id#40, i_class_id#41, i_category_id#42, sales#56, number_sales#57, channel#82, i_brand_id#67, i_class_id#68, i_category_id#69, sales#83, number_sales#84] ===== Subqueries ===== -Subquery:1 Hosting operator id = 67 Hosting Expression = Subquery scalar-subquery#58, [id=#59] -* HashAggregate (109) -+- * ColumnarToRow (108) - +- CometColumnarExchange (107) - +- RowToColumnar (106) - +- * HashAggregate (105) - +- * ColumnarToRow (104) - +- CometUnion (103) - :- CometProject (94) - : +- CometBroadcastHashJoin (93) - : :- CometScan parquet spark_catalog.default.store_sales (91) - : +- ReusedExchange (92) - :- CometProject (98) - : +- CometBroadcastHashJoin (97) - : :- CometScan parquet spark_catalog.default.catalog_sales (95) - : +- ReusedExchange (96) - +- CometProject (102) - +- CometBroadcastHashJoin (101) - :- CometScan parquet spark_catalog.default.web_sales (99) - +- ReusedExchange (100) - - -(91) Scan parquet spark_catalog.default.store_sales +Subquery:1 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#58, [id=#59] +* HashAggregate (103) ++- Exchange (102) + +- * HashAggregate (101) + +- * ColumnarToRow (100) + +- CometUnion (99) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometScan parquet spark_catalog.default.store_sales (87) + : +- ReusedExchange (88) + :- CometProject (94) + : +- CometBroadcastHashJoin (93) + : :- CometScan parquet spark_catalog.default.catalog_sales (91) + : +- ReusedExchange (92) + +- CometProject (98) + +- CometBroadcastHashJoin (97) + :- CometScan parquet spark_catalog.default.web_sales (95) + +- ReusedExchange (96) + + +(87) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#85, ss_list_price#86, ss_sold_date_sk#87] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#87), dynamicpruningexpression(ss_sold_date_sk#87 IN dynamicpruning#88)] ReadSchema: struct -(92) ReusedExchange [Reuses operator id: 19] +(88) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#89] -(93) CometBroadcastHashJoin +(89) CometBroadcastHashJoin Left output [3]: [ss_quantity#85, ss_list_price#86, ss_sold_date_sk#87] Right output [1]: [d_date_sk#89] Arguments: [ss_sold_date_sk#87], [d_date_sk#89], Inner, BuildRight -(94) CometProject +(90) CometProject Input [4]: [ss_quantity#85, ss_list_price#86, ss_sold_date_sk#87, d_date_sk#89] Arguments: [quantity#90, list_price#91], [ss_quantity#85 AS quantity#90, ss_list_price#86 AS list_price#91] -(95) Scan parquet spark_catalog.default.catalog_sales +(91) Scan parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#92, cs_list_price#93, cs_sold_date_sk#94] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#94), dynamicpruningexpression(cs_sold_date_sk#94 IN dynamicpruning#95)] ReadSchema: struct -(96) ReusedExchange [Reuses operator id: 19] +(92) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#96] -(97) CometBroadcastHashJoin +(93) CometBroadcastHashJoin Left output [3]: [cs_quantity#92, cs_list_price#93, cs_sold_date_sk#94] Right output [1]: [d_date_sk#96] Arguments: [cs_sold_date_sk#94], [d_date_sk#96], Inner, BuildRight -(98) CometProject +(94) CometProject Input [4]: [cs_quantity#92, cs_list_price#93, cs_sold_date_sk#94, d_date_sk#96] Arguments: [quantity#97, list_price#98], [cs_quantity#92 AS quantity#97, cs_list_price#93 AS list_price#98] -(99) Scan parquet spark_catalog.default.web_sales +(95) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#99, ws_list_price#100, ws_sold_date_sk#101] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#101), dynamicpruningexpression(ws_sold_date_sk#101 IN dynamicpruning#102)] ReadSchema: struct -(100) ReusedExchange [Reuses operator id: 19] +(96) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#103] -(101) CometBroadcastHashJoin +(97) CometBroadcastHashJoin Left output [3]: [ws_quantity#99, ws_list_price#100, ws_sold_date_sk#101] Right output [1]: [d_date_sk#103] Arguments: [ws_sold_date_sk#101], [d_date_sk#103], Inner, BuildRight -(102) CometProject +(98) CometProject Input [4]: [ws_quantity#99, ws_list_price#100, ws_sold_date_sk#101, d_date_sk#103] Arguments: [quantity#104, list_price#105], [ws_quantity#99 AS quantity#104, ws_list_price#100 AS list_price#105] -(103) CometUnion +(99) CometUnion Child 0 Input [2]: [quantity#90, list_price#91] Child 1 Input [2]: [quantity#97, list_price#98] Child 2 Input [2]: [quantity#104, list_price#105] -(104) ColumnarToRow [codegen id : 1] +(100) ColumnarToRow [codegen id : 1] Input [2]: [quantity#90, list_price#91] -(105) HashAggregate [codegen id : 1] +(101) HashAggregate [codegen id : 1] Input [2]: [quantity#90, list_price#91] Keys: [] Functions [1]: [partial_avg((cast(quantity#90 as decimal(10,0)) * list_price#91))] Aggregate Attributes [2]: [sum#106, count#107] Results [2]: [sum#108, count#109] -(106) RowToColumnar -Input [2]: [sum#108, count#109] - -(107) CometColumnarExchange -Input [2]: [sum#108, count#109] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(108) ColumnarToRow [codegen id : 2] +(102) Exchange Input [2]: [sum#108, count#109] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] -(109) HashAggregate [codegen id : 2] +(103) HashAggregate [codegen id : 2] Input [2]: [sum#108, count#109] Keys: [] Functions [1]: [avg((cast(quantity#90 as decimal(10,0)) * list_price#91))] Aggregate Attributes [1]: [avg((cast(quantity#90 as decimal(10,0)) * list_price#91))#110] Results [1]: [avg((cast(quantity#90 as decimal(10,0)) * list_price#91))#110 AS average_sales#111] -Subquery:2 Hosting operator id = 91 Hosting Expression = ss_sold_date_sk#87 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 87 Hosting Expression = ss_sold_date_sk#87 IN dynamicpruning#12 -Subquery:3 Hosting operator id = 95 Hosting Expression = cs_sold_date_sk#94 IN dynamicpruning#12 +Subquery:3 Hosting operator id = 91 Hosting Expression = cs_sold_date_sk#94 IN dynamicpruning#12 -Subquery:4 Hosting operator id = 99 Hosting Expression = ws_sold_date_sk#101 IN dynamicpruning#12 +Subquery:4 Hosting operator id = 95 Hosting Expression = ws_sold_date_sk#101 IN dynamicpruning#12 Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (114) -+- * ColumnarToRow (113) - +- CometProject (112) - +- CometFilter (111) - +- CometScan parquet spark_catalog.default.date_dim (110) +BroadcastExchange (108) ++- * ColumnarToRow (107) + +- CometProject (106) + +- CometFilter (105) + +- CometScan parquet spark_catalog.default.date_dim (104) -(110) Scan parquet spark_catalog.default.date_dim +(104) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#45), IsNotNull(d_date_sk)] ReadSchema: struct -(111) CometFilter +(105) CometFilter Input [2]: [d_date_sk#43, d_week_seq#44] Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = ReusedSubquery Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) -(112) CometProject +(106) CometProject Input [2]: [d_date_sk#43, d_week_seq#44] Arguments: [d_date_sk#43], [d_date_sk#43] -(113) ColumnarToRow [codegen id : 1] +(107) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#43] -(114) BroadcastExchange +(108) BroadcastExchange Input [1]: [d_date_sk#43] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -Subquery:6 Hosting operator id = 111 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] +Subquery:6 Hosting operator id = 105 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] -Subquery:7 Hosting operator id = 110 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* ColumnarToRow (118) -+- CometProject (117) - +- CometFilter (116) - +- CometScan parquet spark_catalog.default.date_dim (115) +Subquery:7 Hosting operator id = 104 Hosting Expression = Subquery scalar-subquery#45, [id=#46] +* ColumnarToRow (112) ++- CometProject (111) + +- CometFilter (110) + +- CometScan parquet spark_catalog.default.date_dim (109) -(115) Scan parquet spark_catalog.default.date_dim +(109) Scan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#112, d_year#113, d_moy#114, d_dom#115] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,2000), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(116) CometFilter +(110) CometFilter Input [4]: [d_week_seq#112, d_year#113, d_moy#114, d_dom#115] Condition : (((((isnotnull(d_year#113) AND isnotnull(d_moy#114)) AND isnotnull(d_dom#115)) AND (d_year#113 = 2000)) AND (d_moy#114 = 12)) AND (d_dom#115 = 11)) -(117) CometProject +(111) CometProject Input [4]: [d_week_seq#112, d_year#113, d_moy#114, d_dom#115] Arguments: [d_week_seq#112], [d_week_seq#112] -(118) ColumnarToRow [codegen id : 1] +(112) ColumnarToRow [codegen id : 1] Input [1]: [d_week_seq#112] Subquery:8 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (123) -+- * ColumnarToRow (122) - +- CometProject (121) - +- CometFilter (120) - +- CometScan parquet spark_catalog.default.date_dim (119) +BroadcastExchange (117) ++- * ColumnarToRow (116) + +- CometProject (115) + +- CometFilter (114) + +- CometScan parquet spark_catalog.default.date_dim (113) -(119) Scan parquet spark_catalog.default.date_dim +(113) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#116] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(120) CometFilter +(114) CometFilter Input [2]: [d_date_sk#26, d_year#116] Condition : (((isnotnull(d_year#116) AND (d_year#116 >= 1999)) AND (d_year#116 <= 2001)) AND isnotnull(d_date_sk#26)) -(121) CometProject +(115) CometProject Input [2]: [d_date_sk#26, d_year#116] Arguments: [d_date_sk#26], [d_date_sk#26] -(122) ColumnarToRow [codegen id : 1] +(116) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#26] -(123) BroadcastExchange +(117) BroadcastExchange Input [1]: [d_date_sk#26] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] @@ -726,67 +702,67 @@ Subquery:11 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquer Subquery:12 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] -Subquery:13 Hosting operator id = 87 Hosting Expression = ReusedSubquery Subquery scalar-subquery#58, [id=#59] +Subquery:13 Hosting operator id = 83 Hosting Expression = ReusedSubquery Subquery scalar-subquery#58, [id=#59] -Subquery:14 Hosting operator id = 68 Hosting Expression = ss_sold_date_sk#63 IN dynamicpruning#64 -BroadcastExchange (128) -+- * ColumnarToRow (127) - +- CometProject (126) - +- CometFilter (125) - +- CometScan parquet spark_catalog.default.date_dim (124) +Subquery:14 Hosting operator id = 66 Hosting Expression = ss_sold_date_sk#63 IN dynamicpruning#64 +BroadcastExchange (122) ++- * ColumnarToRow (121) + +- CometProject (120) + +- CometFilter (119) + +- CometScan parquet spark_catalog.default.date_dim (118) -(124) Scan parquet spark_catalog.default.date_dim +(118) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#70, d_week_seq#71] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#72), IsNotNull(d_date_sk)] ReadSchema: struct -(125) CometFilter +(119) CometFilter Input [2]: [d_date_sk#70, d_week_seq#71] Condition : ((isnotnull(d_week_seq#71) AND (d_week_seq#71 = ReusedSubquery Subquery scalar-subquery#72, [id=#73])) AND isnotnull(d_date_sk#70)) -(126) CometProject +(120) CometProject Input [2]: [d_date_sk#70, d_week_seq#71] Arguments: [d_date_sk#70], [d_date_sk#70] -(127) ColumnarToRow [codegen id : 1] +(121) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#70] -(128) BroadcastExchange +(122) BroadcastExchange Input [1]: [d_date_sk#70] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:15 Hosting operator id = 125 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] +Subquery:15 Hosting operator id = 119 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] -Subquery:16 Hosting operator id = 124 Hosting Expression = Subquery scalar-subquery#72, [id=#73] -* ColumnarToRow (132) -+- CometProject (131) - +- CometFilter (130) - +- CometScan parquet spark_catalog.default.date_dim (129) +Subquery:16 Hosting operator id = 118 Hosting Expression = Subquery scalar-subquery#72, [id=#73] +* ColumnarToRow (126) ++- CometProject (125) + +- CometFilter (124) + +- CometScan parquet spark_catalog.default.date_dim (123) -(129) Scan parquet spark_catalog.default.date_dim +(123) Scan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#117, d_year#118, d_moy#119, d_dom#120] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,11)] ReadSchema: struct -(130) CometFilter +(124) CometFilter Input [4]: [d_week_seq#117, d_year#118, d_moy#119, d_dom#120] Condition : (((((isnotnull(d_year#118) AND isnotnull(d_moy#119)) AND isnotnull(d_dom#120)) AND (d_year#118 = 1999)) AND (d_moy#119 = 12)) AND (d_dom#120 = 11)) -(131) CometProject +(125) CometProject Input [4]: [d_week_seq#117, d_year#118, d_moy#119, d_dom#120] Arguments: [d_week_seq#117], [d_week_seq#117] -(132) ColumnarToRow [codegen id : 1] +(126) ColumnarToRow [codegen id : 1] Input [1]: [d_week_seq#117] -Subquery:17 Hosting operator id = 76 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] +Subquery:17 Hosting operator id = 74 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] -Subquery:18 Hosting operator id = 75 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] +Subquery:18 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt index 1210748c5..1fecc0fca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q14b/simplified.txt @@ -5,169 +5,163 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ Subquery #4 WholeStageCodegen (2) HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #14 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [quantity,list_price] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometUnion [quantity,list_price] - CometProject [ss_quantity,ss_list_price] [quantity,list_price] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 - CometProject [cs_quantity,cs_list_price] [quantity,list_price] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 - CometProject [ws_quantity,ws_list_price] [quantity,list_price] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #1 - RowToColumnar + InputAdapter + Exchange #14 WholeStageCodegen (1) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] + HashAggregate [quantity,list_price] [sum,count,sum,count] ColumnarToRow InputAdapter - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometBroadcastExchange [ss_item_sk] #3 - CometProject [i_item_sk] [ss_item_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 - CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #5 - CometHashAggregate [brand_id,class_id,category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] - CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #10 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + ReusedExchange [d_date_sk] #10 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + ReusedExchange [d_date_sk] #10 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + ReusedExchange [d_date_sk] #10 + HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] + InputAdapter + Exchange [i_brand_id,i_class_id,i_category_id] #1 + WholeStageCodegen (1) + HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #2 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + Subquery #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometBroadcastExchange [ss_item_sk] #3 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #13 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 + CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #5 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #3 + BroadcastExchange #6 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #10 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #3 + CometBroadcastExchange [d_date_sk] #13 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #2 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #2 InputAdapter BroadcastExchange #15 WholeStageCodegen (3) Filter [sales] ReusedSubquery [average_sales] #4 HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #16 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #5 - BroadcastExchange #17 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - Subquery #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ReusedExchange [ss_item_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - CometBroadcastExchange [d_date_sk] #18 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 + InputAdapter + Exchange [i_brand_id,i_class_id,i_category_id] #16 + WholeStageCodegen (2) + HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #5 + BroadcastExchange #17 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #6 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + Subquery #6 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + CometBroadcastExchange [d_date_sk] #18 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #6 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt index 9015f7e90..a22a52dc6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/explain.txt @@ -1,29 +1,27 @@ == Physical Plan == -TakeOrderedAndProject (25) -+- * HashAggregate (24) - +- * ColumnarToRow (23) - +- CometColumnarExchange (22) - +- RowToColumnar (21) - +- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.customer (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometScan parquet spark_catalog.default.customer_address (8) - +- CometBroadcastExchange (16) - +- CometProject (15) - +- CometFilter (14) - +- CometScan parquet spark_catalog.default.date_dim (13) +TakeOrderedAndProject (23) ++- * HashAggregate (22) + +- Exchange (21) + +- * HashAggregate (20) + +- * ColumnarToRow (19) + +- CometProject (18) + +- CometBroadcastHashJoin (17) + :- CometProject (12) + : +- CometBroadcastHashJoin (11) + : :- CometProject (7) + : : +- CometBroadcastHashJoin (6) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : +- CometBroadcastExchange (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.customer (3) + : +- CometBroadcastExchange (10) + : +- CometFilter (9) + : +- CometScan parquet spark_catalog.default.customer_address (8) + +- CometBroadcastExchange (16) + +- CometProject (15) + +- CometFilter (14) + +- CometScan parquet spark_catalog.default.date_dim (13) (1) Scan parquet spark_catalog.default.catalog_sales @@ -124,56 +122,50 @@ Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] Aggregate Attributes [1]: [sum#13] Results [2]: [ca_zip#9, sum#14] -(21) RowToColumnar +(21) Exchange Input [2]: [ca_zip#9, sum#14] +Arguments: hashpartitioning(ca_zip#9, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(22) CometColumnarExchange -Input [2]: [ca_zip#9, sum#14] -Arguments: hashpartitioning(ca_zip#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(23) ColumnarToRow [codegen id : 2] -Input [2]: [ca_zip#9, sum#14] - -(24) HashAggregate [codegen id : 2] +(22) HashAggregate [codegen id : 2] Input [2]: [ca_zip#9, sum#14] Keys [1]: [ca_zip#9] Functions [1]: [sum(UnscaledValue(cs_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#2))#15] Results [2]: [ca_zip#9, MakeDecimal(sum(UnscaledValue(cs_sales_price#2))#15,17,2) AS sum(cs_sales_price)#16] -(25) TakeOrderedAndProject +(23) TakeOrderedAndProject Input [2]: [ca_zip#9, sum(cs_sales_price)#16] Arguments: 100, [ca_zip#9 ASC NULLS FIRST], [ca_zip#9, sum(cs_sales_price)#16] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (30) -+- * ColumnarToRow (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan parquet spark_catalog.default.date_dim (26) +BroadcastExchange (28) ++- * ColumnarToRow (27) + +- CometProject (26) + +- CometFilter (25) + +- CometScan parquet spark_catalog.default.date_dim (24) -(26) Scan parquet spark_catalog.default.date_dim +(24) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_year#11, d_qoy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(27) CometFilter +(25) CometFilter Input [3]: [d_date_sk#10, d_year#11, d_qoy#12] Condition : ((((isnotnull(d_qoy#12) AND isnotnull(d_year#11)) AND (d_qoy#12 = 2)) AND (d_year#11 = 2001)) AND isnotnull(d_date_sk#10)) -(28) CometProject +(26) CometProject Input [3]: [d_date_sk#10, d_year#11, d_qoy#12] Arguments: [d_date_sk#10], [d_date_sk#10] -(29) ColumnarToRow [codegen id : 1] +(27) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#10] -(30) BroadcastExchange +(28) BroadcastExchange Input [1]: [d_date_sk#10] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt index f466e8189..a34cd9c1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q15/simplified.txt @@ -1,37 +1,35 @@ TakeOrderedAndProject [ca_zip,sum(cs_sales_price)] WholeStageCodegen (2) HashAggregate [ca_zip,sum] [sum(UnscaledValue(cs_sales_price)),sum(cs_sales_price),sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_zip] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ca_zip,cs_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_sales_price,ca_zip] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] - CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_state,ca_zip] - CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 - CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #4 - CometFilter [ca_address_sk,ca_state,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + InputAdapter + Exchange [ca_zip] #1 + WholeStageCodegen (1) + HashAggregate [ca_zip,cs_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [cs_sales_price,ca_zip] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] + CometProject [cs_sales_price,cs_sold_date_sk,ca_zip] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_state,ca_zip] + CometProject [cs_sales_price,cs_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sales_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_state,ca_zip] #4 + CometFilter [ca_address_sk,ca_state,ca_zip] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt index ebff95d06..4890b4436 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/explain.txt @@ -1,51 +1,49 @@ == Physical Plan == -* HashAggregate (47) -+- * ColumnarToRow (46) - +- CometColumnarExchange (45) - +- RowToColumnar (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * SortMergeJoin LeftAnti (19) - : : : :- * Project (13) - : : : : +- * SortMergeJoin LeftSemi (12) - : : : : :- * ColumnarToRow (6) - : : : : : +- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : +- * ColumnarToRow (11) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (7) - : : : +- * ColumnarToRow (18) - : : : +- CometSort (17) - : : : +- CometColumnarExchange (16) - : : : +- CometProject (15) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (14) - : : +- BroadcastExchange (24) - : : +- * ColumnarToRow (23) - : : +- CometProject (22) - : : +- CometFilter (21) - : : +- CometScan parquet spark_catalog.default.date_dim (20) - : +- BroadcastExchange (31) - : +- * ColumnarToRow (30) - : +- CometProject (29) - : +- CometFilter (28) - : +- CometScan parquet spark_catalog.default.customer_address (27) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan parquet spark_catalog.default.call_center (34) +* HashAggregate (45) ++- Exchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (33) + : +- * BroadcastHashJoin Inner BuildRight (32) + : :- * Project (26) + : : +- * BroadcastHashJoin Inner BuildRight (25) + : : :- * SortMergeJoin LeftAnti (19) + : : : :- * Project (13) + : : : : +- * SortMergeJoin LeftSemi (12) + : : : : :- * ColumnarToRow (6) + : : : : : +- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- * ColumnarToRow (11) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (7) + : : : +- * ColumnarToRow (18) + : : : +- CometSort (17) + : : : +- CometExchange (16) + : : : +- CometProject (15) + : : : +- CometScan parquet spark_catalog.default.catalog_returns (14) + : : +- BroadcastExchange (24) + : : +- * ColumnarToRow (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometScan parquet spark_catalog.default.date_dim (20) + : +- BroadcastExchange (31) + : +- * ColumnarToRow (30) + : +- CometProject (29) + : +- CometFilter (28) + : +- CometScan parquet spark_catalog.default.customer_address (27) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan parquet spark_catalog.default.call_center (34) (1) Scan parquet spark_catalog.default.catalog_sales @@ -63,9 +61,9 @@ Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(4) CometColumnarExchange +(4) CometExchange Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] @@ -84,9 +82,9 @@ ReadSchema: struct Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order_number#10] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [cs_warehouse_sk#9, cs_order_number#10] @@ -115,9 +113,9 @@ ReadSchema: struct Input [2]: [cr_order_number#12, cr_returned_date_sk#13] Arguments: [cr_order_number#12], [cr_order_number#12] -(16) CometColumnarExchange +(16) CometExchange Input [1]: [cr_order_number#12] -Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (17) CometSort Input [1]: [cr_order_number#12] @@ -249,17 +247,11 @@ Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] Results [3]: [sum#22, sum#23, count#25] -(44) RowToColumnar +(44) Exchange Input [3]: [sum#22, sum#23, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(45) CometColumnarExchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(46) ColumnarToRow [codegen id : 9] -Input [3]: [sum#22, sum#23, count#25] - -(47) HashAggregate [codegen id : 9] +(45) HashAggregate [codegen id : 9] Input [3]: [sum#22, sum#23, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt index 605ce3703..18ba39326 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q16/simplified.txt @@ -1,70 +1,68 @@ WholeStageCodegen (9) HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (8) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - Project [cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk] - Project [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk] - SortMergeJoin [cs_order_number,cr_order_number] - InputAdapter - WholeStageCodegen (3) - Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometColumnarExchange [cs_order_number] #2 - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [cs_warehouse_sk,cs_order_number] - CometColumnarExchange [cs_order_number] #3 - CometProject [cs_warehouse_sk,cs_order_number] - CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [cr_order_number] - CometColumnarExchange [cr_order_number] #4 - CometProject [cr_order_number] - CometScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (8) + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + Project [cs_order_number,cs_ext_ship_cost,cs_net_profit] + BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] + Project [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk] + Project [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + BroadcastHashJoin [cs_ship_date_sk,d_date_sk] + SortMergeJoin [cs_order_number,cr_order_number] + InputAdapter + WholeStageCodegen (3) + Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometExchange [cs_order_number] #2 + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + InputAdapter + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometSort [cs_warehouse_sk,cs_order_number] + CometExchange [cs_order_number] #3 + CometProject [cs_warehouse_sk,cs_order_number] + CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + InputAdapter + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [cr_order_number] + CometExchange [cr_order_number] #4 + CometProject [cr_order_number] + CometScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) + BroadcastExchange #5 + WholeStageCodegen (5) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) + BroadcastExchange #6 + WholeStageCodegen (6) ColumnarToRow InputAdapter - CometProject [cc_call_center_sk] - CometFilter [cc_call_center_sk,cc_county] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometProject [cc_call_center_sk] + CometFilter [cc_call_center_sk,cc_county] + CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt index 2da3a5a21..d3739ed75 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/explain.txt @@ -1,48 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * HashAggregate (43) - +- * ColumnarToRow (42) - +- CometColumnarExchange (41) - +- RowToColumnar (40) - +- * HashAggregate (39) - +- * ColumnarToRow (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store_returns (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (8) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometProject (15) - : : : : +- CometFilter (14) - : : : : +- CometScan parquet spark_catalog.default.date_dim (13) - : : : +- CometBroadcastExchange (22) - : : : +- CometProject (21) - : : : +- CometFilter (20) - : : : +- CometScan parquet spark_catalog.default.date_dim (19) - : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometScan parquet spark_catalog.default.store (28) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometScan parquet spark_catalog.default.item (33) +TakeOrderedAndProject (42) ++- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * ColumnarToRow (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (32) + : +- CometBroadcastHashJoin (31) + : :- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometScan parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometScan parquet spark_catalog.default.date_dim (19) + : : +- ReusedExchange (25) + : +- CometBroadcastExchange (30) + : +- CometFilter (29) + : +- CometScan parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (35) + +- CometFilter (34) + +- CometScan parquet spark_catalog.default.item (33) (1) Scan parquet spark_catalog.default.store_sales @@ -233,86 +231,80 @@ Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partia Aggregate Attributes [18]: [count#29, sum#30, count#31, n#32, avg#33, m2#34, count#35, sum#36, count#37, n#38, avg#39, m2#40, count#41, sum#42, count#43, n#44, avg#45, m2#46] Results [21]: [i_item_id#27, i_item_desc#28, s_state#25, count#47, sum#48, count#49, n#50, avg#51, m2#52, count#53, sum#54, count#55, n#56, avg#57, m2#58, count#59, sum#60, count#61, n#62, avg#63, m2#64] -(40) RowToColumnar +(40) Exchange Input [21]: [i_item_id#27, i_item_desc#28, s_state#25, count#47, sum#48, count#49, n#50, avg#51, m2#52, count#53, sum#54, count#55, n#56, avg#57, m2#58, count#59, sum#60, count#61, n#62, avg#63, m2#64] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_state#25, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(41) CometColumnarExchange -Input [21]: [i_item_id#27, i_item_desc#28, s_state#25, count#47, sum#48, count#49, n#50, avg#51, m2#52, count#53, sum#54, count#55, n#56, avg#57, m2#58, count#59, sum#60, count#61, n#62, avg#63, m2#64] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_state#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(42) ColumnarToRow [codegen id : 2] -Input [21]: [i_item_id#27, i_item_desc#28, s_state#25, count#47, sum#48, count#49, n#50, avg#51, m2#52, count#53, sum#54, count#55, n#56, avg#57, m2#58, count#59, sum#60, count#61, n#62, avg#63, m2#64] - -(43) HashAggregate [codegen id : 2] +(41) HashAggregate [codegen id : 2] Input [21]: [i_item_id#27, i_item_desc#28, s_state#25, count#47, sum#48, count#49, n#50, avg#51, m2#52, count#53, sum#54, count#55, n#56, avg#57, m2#58, count#59, sum#60, count#61, n#62, avg#63, m2#64] Keys [3]: [i_item_id#27, i_item_desc#28, s_state#25] Functions [9]: [count(ss_quantity#5), avg(ss_quantity#5), stddev_samp(cast(ss_quantity#5 as double)), count(sr_return_quantity#11), avg(sr_return_quantity#11), stddev_samp(cast(sr_return_quantity#11 as double)), count(cs_quantity#16), avg(cs_quantity#16), stddev_samp(cast(cs_quantity#16 as double))] Aggregate Attributes [9]: [count(ss_quantity#5)#65, avg(ss_quantity#5)#66, stddev_samp(cast(ss_quantity#5 as double))#67, count(sr_return_quantity#11)#68, avg(sr_return_quantity#11)#69, stddev_samp(cast(sr_return_quantity#11 as double))#70, count(cs_quantity#16)#71, avg(cs_quantity#16)#72, stddev_samp(cast(cs_quantity#16 as double))#73] Results [15]: [i_item_id#27, i_item_desc#28, s_state#25, count(ss_quantity#5)#65 AS store_sales_quantitycount#74, avg(ss_quantity#5)#66 AS store_sales_quantityave#75, stddev_samp(cast(ss_quantity#5 as double))#67 AS store_sales_quantitystdev#76, (stddev_samp(cast(ss_quantity#5 as double))#67 / avg(ss_quantity#5)#66) AS store_sales_quantitycov#77, count(sr_return_quantity#11)#68 AS as_store_returns_quantitycount#78, avg(sr_return_quantity#11)#69 AS as_store_returns_quantityave#79, stddev_samp(cast(sr_return_quantity#11 as double))#70 AS as_store_returns_quantitystdev#80, (stddev_samp(cast(sr_return_quantity#11 as double))#70 / avg(sr_return_quantity#11)#69) AS store_returns_quantitycov#81, count(cs_quantity#16)#71 AS catalog_sales_quantitycount#82, avg(cs_quantity#16)#72 AS catalog_sales_quantityave#83, (stddev_samp(cast(cs_quantity#16 as double))#73 / avg(cs_quantity#16)#72) AS catalog_sales_quantitystdev#84, (stddev_samp(cast(cs_quantity#16 as double))#73 / avg(cs_quantity#16)#72) AS catalog_sales_quantitycov#85] -(44) TakeOrderedAndProject +(42) TakeOrderedAndProject Input [15]: [i_item_id#27, i_item_desc#28, s_state#25, store_sales_quantitycount#74, store_sales_quantityave#75, store_sales_quantitystdev#76, store_sales_quantitycov#77, as_store_returns_quantitycount#78, as_store_returns_quantityave#79, as_store_returns_quantitystdev#80, store_returns_quantitycov#81, catalog_sales_quantitycount#82, catalog_sales_quantityave#83, catalog_sales_quantitystdev#84, catalog_sales_quantitycov#85] Arguments: 100, [i_item_id#27 ASC NULLS FIRST, i_item_desc#28 ASC NULLS FIRST, s_state#25 ASC NULLS FIRST], [i_item_id#27, i_item_desc#28, s_state#25, store_sales_quantitycount#74, store_sales_quantityave#75, store_sales_quantitystdev#76, store_sales_quantitycov#77, as_store_returns_quantitycount#78, as_store_returns_quantityave#79, as_store_returns_quantitystdev#80, store_returns_quantitycov#81, catalog_sales_quantitycount#82, catalog_sales_quantityave#83, catalog_sales_quantitystdev#84, catalog_sales_quantitycov#85] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (49) -+- * ColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan parquet spark_catalog.default.date_dim (45) +BroadcastExchange (47) ++- * ColumnarToRow (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan parquet spark_catalog.default.date_dim (43) -(45) Scan parquet spark_catalog.default.date_dim +(43) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_quarter_name#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_quarter_name), EqualTo(d_quarter_name,2001Q1), IsNotNull(d_date_sk)] ReadSchema: struct -(46) CometFilter +(44) CometFilter Input [2]: [d_date_sk#19, d_quarter_name#20] Condition : ((isnotnull(d_quarter_name#20) AND (d_quarter_name#20 = 2001Q1)) AND isnotnull(d_date_sk#19)) -(47) CometProject +(45) CometProject Input [2]: [d_date_sk#19, d_quarter_name#20] Arguments: [d_date_sk#19], [d_date_sk#19] -(48) ColumnarToRow [codegen id : 1] +(46) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#19] -(49) BroadcastExchange +(47) BroadcastExchange Input [1]: [d_date_sk#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (54) -+- * ColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.date_dim (50) +BroadcastExchange (52) ++- * ColumnarToRow (51) + +- CometProject (50) + +- CometFilter (49) + +- CometScan parquet spark_catalog.default.date_dim (48) -(50) Scan parquet spark_catalog.default.date_dim +(48) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#21, d_quarter_name#22] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_quarter_name, [2001Q1,2001Q2,2001Q3]), IsNotNull(d_date_sk)] ReadSchema: struct -(51) CometFilter +(49) CometFilter Input [2]: [d_date_sk#21, d_quarter_name#22] Condition : (d_quarter_name#22 IN (2001Q1,2001Q2,2001Q3) AND isnotnull(d_date_sk#21)) -(52) CometProject +(50) CometProject Input [2]: [d_date_sk#21, d_quarter_name#22] Arguments: [d_date_sk#21], [d_date_sk#21] -(53) ColumnarToRow [codegen id : 1] +(51) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#21] -(54) BroadcastExchange +(52) BroadcastExchange Input [1]: [d_date_sk#21] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt index 0604dcd57..4a634b069 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q17/simplified.txt @@ -1,65 +1,63 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] WholeStageCodegen (2) HashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] [count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double)),store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,i_item_desc,s_state] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,s_state,ss_quantity,sr_return_quantity,cs_quantity] [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_state] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_quarter_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - CometBroadcastExchange [d_date_sk] #7 + InputAdapter + Exchange [i_item_id,i_item_desc,s_state] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,s_state,ss_quantity,sr_return_quantity,cs_quantity] [count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_quarter_name] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_quarter_name] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_quarter_name] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [s_store_sk,s_state] #8 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 - CometFilter [i_item_sk,i_item_id,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_quarter_name] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_quarter_name] + ReusedExchange [d_date_sk] #7 + CometBroadcastExchange [s_store_sk,s_state] #8 + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt index 49d68974b..acbc8e654 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/explain.txt @@ -1,47 +1,45 @@ == Physical Plan == -TakeOrderedAndProject (43) -+- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometColumnarExchange (40) - +- RowToColumnar (39) - +- * HashAggregate (38) - +- * ColumnarToRow (37) - +- CometExpand (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (19) - : : : +- CometBroadcastHashJoin (18) - : : : :- CometProject (14) - : : : : +- CometBroadcastHashJoin (13) - : : : : :- CometProject (8) - : : : : : +- CometBroadcastHashJoin (7) - : : : : : :- CometFilter (2) - : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : : +- CometBroadcastExchange (6) - : : : : : +- CometProject (5) - : : : : : +- CometFilter (4) - : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) - : : : : +- CometBroadcastExchange (12) - : : : : +- CometProject (11) - : : : : +- CometFilter (10) - : : : : +- CometScan parquet spark_catalog.default.customer (9) - : : : +- CometBroadcastExchange (17) - : : : +- CometFilter (16) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (15) - : : +- CometBroadcastExchange (22) - : : +- CometFilter (21) - : : +- CometScan parquet spark_catalog.default.customer_address (20) - : +- CometBroadcastExchange (28) - : +- CometProject (27) - : +- CometFilter (26) - : +- CometScan parquet spark_catalog.default.date_dim (25) - +- CometBroadcastExchange (33) - +- CometFilter (32) - +- CometScan parquet spark_catalog.default.item (31) +TakeOrderedAndProject (41) ++- * HashAggregate (40) + +- Exchange (39) + +- * HashAggregate (38) + +- * ColumnarToRow (37) + +- CometExpand (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (19) + : : : +- CometBroadcastHashJoin (18) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan parquet spark_catalog.default.customer (9) + : : : +- CometBroadcastExchange (17) + : : : +- CometFilter (16) + : : : +- CometScan parquet spark_catalog.default.customer_demographics (15) + : : +- CometBroadcastExchange (22) + : : +- CometFilter (21) + : : +- CometScan parquet spark_catalog.default.customer_address (20) + : +- CometBroadcastExchange (28) + : +- CometProject (27) + : +- CometFilter (26) + : +- CometScan parquet spark_catalog.default.date_dim (25) + +- CometBroadcastExchange (33) + +- CometFilter (32) + +- CometScan parquet spark_catalog.default.item (31) (1) Scan parquet spark_catalog.default.catalog_sales @@ -226,56 +224,50 @@ Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(c Aggregate Attributes [14]: [sum#34, count#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47] Results [19]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61] -(39) RowToColumnar +(39) Exchange Input [19]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61] +Arguments: hashpartitioning(i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(40) CometColumnarExchange -Input [19]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61] -Arguments: hashpartitioning(i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(41) ColumnarToRow [codegen id : 2] -Input [19]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61] - -(42) HashAggregate [codegen id : 2] +(40) HashAggregate [codegen id : 2] Input [19]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, sum#48, count#49, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61] Keys [5]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33] Functions [7]: [avg(cast(cs_quantity#4 as decimal(12,2))), avg(cast(cs_list_price#5 as decimal(12,2))), avg(cast(cs_coupon_amt#7 as decimal(12,2))), avg(cast(cs_sales_price#6 as decimal(12,2))), avg(cast(cs_net_profit#8 as decimal(12,2))), avg(cast(c_birth_year#19 as decimal(12,2))), avg(cast(cd_dep_count#14 as decimal(12,2)))] Aggregate Attributes [7]: [avg(cast(cs_quantity#4 as decimal(12,2)))#62, avg(cast(cs_list_price#5 as decimal(12,2)))#63, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#64, avg(cast(cs_sales_price#6 as decimal(12,2)))#65, avg(cast(cs_net_profit#8 as decimal(12,2)))#66, avg(cast(c_birth_year#19 as decimal(12,2)))#67, avg(cast(cd_dep_count#14 as decimal(12,2)))#68] Results [11]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, avg(cast(cs_quantity#4 as decimal(12,2)))#62 AS agg1#69, avg(cast(cs_list_price#5 as decimal(12,2)))#63 AS agg2#70, avg(cast(cs_coupon_amt#7 as decimal(12,2)))#64 AS agg3#71, avg(cast(cs_sales_price#6 as decimal(12,2)))#65 AS agg4#72, avg(cast(cs_net_profit#8 as decimal(12,2)))#66 AS agg5#73, avg(cast(c_birth_year#19 as decimal(12,2)))#67 AS agg6#74, avg(cast(cd_dep_count#14 as decimal(12,2)))#68 AS agg7#75] -(43) TakeOrderedAndProject +(41) TakeOrderedAndProject Input [11]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, agg1#69, agg2#70, agg3#71, agg4#72, agg5#73, agg6#74, agg7#75] Arguments: 100, [ca_country#30 ASC NULLS FIRST, ca_state#31 ASC NULLS FIRST, ca_county#32 ASC NULLS FIRST, i_item_id#29 ASC NULLS FIRST], [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, agg1#69, agg2#70, agg3#71, agg4#72, agg5#73, agg6#74, agg7#75] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (48) -+- * ColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan parquet spark_catalog.default.date_dim (44) +BroadcastExchange (46) ++- * ColumnarToRow (45) + +- CometProject (44) + +- CometFilter (43) + +- CometScan parquet spark_catalog.default.date_dim (42) -(44) Scan parquet spark_catalog.default.date_dim +(42) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(45) CometFilter +(43) CometFilter Input [2]: [d_date_sk#25, d_year#26] Condition : ((isnotnull(d_year#26) AND (d_year#26 = 1998)) AND isnotnull(d_date_sk#25)) -(46) CometProject +(44) CometProject Input [2]: [d_date_sk#25, d_year#26] Arguments: [d_date_sk#25], [d_date_sk#25] -(47) ColumnarToRow [codegen id : 1] +(45) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#25] -(48) BroadcastExchange +(46) BroadcastExchange Input [1]: [d_date_sk#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt index 660161251..c49bfc6c8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q18/simplified.txt @@ -1,55 +1,53 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] WholeStageCodegen (2) HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2))),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] - CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #3 - CometProject [cd_demo_sk,cd_dep_count] - CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #5 - CometFilter [cd_demo_sk] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #6 - CometFilter [ca_address_sk,ca_county,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #8 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + Exchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] + CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #3 + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometBroadcastExchange [cd_demo_sk] #5 + CometFilter [cd_demo_sk] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #6 + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #8 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt index 270facf74..0353b1c2f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/explain.txt @@ -1,40 +1,38 @@ == Physical Plan == -TakeOrderedAndProject (36) -+- * HashAggregate (35) - +- * ColumnarToRow (34) - +- CometColumnarExchange (33) - +- RowToColumnar (32) - +- * HashAggregate (31) - +- * ColumnarToRow (30) - +- CometProject (29) - +- CometBroadcastHashJoin (28) - :- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometProject (14) - : : : +- CometBroadcastHashJoin (13) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.store_sales (4) - : : : +- CometBroadcastExchange (12) - : : : +- CometProject (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.item (9) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.customer (15) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometScan parquet spark_catalog.default.customer_address (20) - +- CometBroadcastExchange (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.store (25) +TakeOrderedAndProject (34) ++- * HashAggregate (33) + +- Exchange (32) + +- * HashAggregate (31) + +- * ColumnarToRow (30) + +- CometProject (29) + +- CometBroadcastHashJoin (28) + :- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.store_sales (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan parquet spark_catalog.default.item (9) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometScan parquet spark_catalog.default.customer (15) + : +- CometBroadcastExchange (22) + : +- CometFilter (21) + : +- CometScan parquet spark_catalog.default.customer_address (20) + +- CometBroadcastExchange (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.store (25) (1) Scan parquet spark_catalog.default.date_dim @@ -187,24 +185,18 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] Aggregate Attributes [1]: [sum#21] Results [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -(32) RowToColumnar +(32) Exchange Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] +Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(33) CometColumnarExchange -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(34) ColumnarToRow [codegen id : 2] -Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] - -(35) HashAggregate [codegen id : 2] +(33) HashAggregate [codegen id : 2] Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#22] Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#7))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#7))#23] Results [5]: [i_brand_id#10 AS brand_id#24, i_brand#11 AS brand#25, i_manufact_id#12, i_manufact#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#7))#23,17,2) AS ext_price#26] -(36) TakeOrderedAndProject +(34) TakeOrderedAndProject Input [5]: [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] Arguments: 100, [ext_price#26 DESC NULLS LAST, brand#25 ASC NULLS FIRST, brand_id#24 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST, i_manufact#13 ASC NULLS FIRST], [brand_id#24, brand#25, i_manufact_id#12, i_manufact#13, ext_price#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/simplified.txt index 082614b7f..60caecf91 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q19/simplified.txt @@ -1,40 +1,38 @@ TakeOrderedAndProject [ext_price,brand,brand_id,i_manufact_id,i_manufact] WholeStageCodegen (2) HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] - CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk,ca_address_sk,ca_zip] - CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_customer_sk,c_current_addr_sk] - CometProject [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 - CometProject [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 - CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_zip] #5 - CometFilter [ca_address_sk,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - CometBroadcastExchange [s_store_sk,s_zip] #6 - CometFilter [s_store_sk,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_zip] + InputAdapter + Exchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 + WholeStageCodegen (1) + HashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] + CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk,ca_address_sk,ca_zip] + CometProject [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_current_addr_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,c_customer_sk,c_current_addr_sk] + CometProject [ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] #3 + CometProject [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id,i_manufact,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #4 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_zip] #5 + CometFilter [ca_address_sk,ca_zip] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [s_store_sk,s_zip] #6 + CometFilter [s_store_sk,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt index 70af0c5f4..015b241fa 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/explain.txt @@ -1,44 +1,41 @@ == Physical Plan == -* ColumnarToRow (40) -+- CometSort (39) - +- CometColumnarExchange (38) - +- RowToColumnar (37) - +- * Project (36) - +- * BroadcastHashJoin Inner BuildRight (35) - :- * Project (23) - : +- * BroadcastHashJoin Inner BuildRight (22) - : :- * HashAggregate (16) - : : +- * ColumnarToRow (15) - : : +- CometColumnarExchange (14) - : : +- RowToColumnar (13) - : : +- * HashAggregate (12) - : : +- * ColumnarToRow (11) - : : +- CometProject (10) - : : +- CometBroadcastHashJoin (9) - : : :- CometUnion (5) - : : : :- CometProject (2) - : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : +- CometProject (4) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (3) - : : +- CometBroadcastExchange (8) - : : +- CometFilter (7) - : : +- CometScan parquet spark_catalog.default.date_dim (6) - : +- BroadcastExchange (21) - : +- * ColumnarToRow (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometScan parquet spark_catalog.default.date_dim (17) - +- BroadcastExchange (34) - +- * Project (33) - +- * BroadcastHashJoin Inner BuildRight (32) - :- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- ReusedExchange (24) - +- BroadcastExchange (31) - +- * ColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.date_dim (27) +* ColumnarToRow (37) ++- CometSort (36) + +- CometColumnarExchange (35) + +- RowToColumnar (34) + +- * Project (33) + +- * BroadcastHashJoin Inner BuildRight (32) + :- * Project (21) + : +- * BroadcastHashJoin Inner BuildRight (20) + : :- * HashAggregate (14) + : : +- Exchange (13) + : : +- * HashAggregate (12) + : : +- * ColumnarToRow (11) + : : +- CometProject (10) + : : +- CometBroadcastHashJoin (9) + : : :- CometUnion (5) + : : : :- CometProject (2) + : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : +- CometProject (4) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (3) + : : +- CometBroadcastExchange (8) + : : +- CometFilter (7) + : : +- CometScan parquet spark_catalog.default.date_dim (6) + : +- BroadcastExchange (19) + : +- * ColumnarToRow (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan parquet spark_catalog.default.date_dim (15) + +- BroadcastExchange (31) + +- * Project (30) + +- * BroadcastHashJoin Inner BuildRight (29) + :- * HashAggregate (23) + : +- ReusedExchange (22) + +- BroadcastExchange (28) + +- * ColumnarToRow (27) + +- CometProject (26) + +- CometFilter (25) + +- CometScan parquet spark_catalog.default.date_dim (24) (1) Scan parquet spark_catalog.default.web_sales @@ -101,125 +98,116 @@ Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) Aggregate Attributes [7]: [sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] Results [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] -(13) RowToColumnar +(13) Exchange Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] +Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(14) CometColumnarExchange -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] -Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(15) ColumnarToRow [codegen id : 6] -Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] - -(16) HashAggregate [codegen id : 6] +(14) HashAggregate [codegen id : 6] Input [8]: [d_week_seq#10, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] Keys [1]: [d_week_seq#10] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32] Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END))#26,17,2) AS sun_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END))#27,17,2) AS mon_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END))#28,17,2) AS tue_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#29,17,2) AS wed_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END))#30,17,2) AS thu_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END))#31,17,2) AS fri_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))#32,17,2) AS sat_sales#39] -(17) Scan parquet spark_catalog.default.date_dim +(15) Scan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#40, d_year#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)] ReadSchema: struct -(18) CometFilter +(16) CometFilter Input [2]: [d_week_seq#40, d_year#41] Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2001)) AND isnotnull(d_week_seq#40)) -(19) CometProject +(17) CometProject Input [2]: [d_week_seq#40, d_year#41] Arguments: [d_week_seq#40], [d_week_seq#40] -(20) ColumnarToRow [codegen id : 2] +(18) ColumnarToRow [codegen id : 2] Input [1]: [d_week_seq#40] -(21) BroadcastExchange +(19) BroadcastExchange Input [1]: [d_week_seq#40] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(22) BroadcastHashJoin [codegen id : 6] +(20) BroadcastHashJoin [codegen id : 6] Left keys [1]: [d_week_seq#10] Right keys [1]: [d_week_seq#40] Join type: Inner Join condition: None -(23) Project [codegen id : 6] +(21) Project [codegen id : 6] Output [8]: [d_week_seq#10 AS d_week_seq1#42, sun_sales#33 AS sun_sales1#43, mon_sales#34 AS mon_sales1#44, tue_sales#35 AS tue_sales1#45, wed_sales#36 AS wed_sales1#46, thu_sales#37 AS thu_sales1#47, fri_sales#38 AS fri_sales1#48, sat_sales#39 AS sat_sales1#49] Input [9]: [d_week_seq#10, sun_sales#33, mon_sales#34, tue_sales#35, wed_sales#36, thu_sales#37, fri_sales#38, sat_sales#39, d_week_seq#40] -(24) ReusedExchange [Reuses operator id: 14] +(22) ReusedExchange [Reuses operator id: 13] Output [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57] -(25) ColumnarToRow [codegen id : 5] -Input [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57] - -(26) HashAggregate [codegen id : 5] +(23) HashAggregate [codegen id : 5] Input [8]: [d_week_seq#50, sum#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57] Keys [1]: [d_week_seq#50] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END)), sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))] Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))#32] Results [8]: [d_week_seq#50, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Sunday ) THEN sales_price#59 END))#26,17,2) AS sun_sales#60, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Monday ) THEN sales_price#59 END))#27,17,2) AS mon_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Tuesday ) THEN sales_price#59 END))#28,17,2) AS tue_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Wednesday) THEN sales_price#59 END))#29,17,2) AS wed_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Thursday ) THEN sales_price#59 END))#30,17,2) AS thu_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Friday ) THEN sales_price#59 END))#31,17,2) AS fri_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#58 = Saturday ) THEN sales_price#59 END))#32,17,2) AS sat_sales#66] -(27) Scan parquet spark_catalog.default.date_dim +(24) Scan parquet spark_catalog.default.date_dim Output [2]: [d_week_seq#67, d_year#68] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)] ReadSchema: struct -(28) CometFilter +(25) CometFilter Input [2]: [d_week_seq#67, d_year#68] Condition : ((isnotnull(d_year#68) AND (d_year#68 = 2002)) AND isnotnull(d_week_seq#67)) -(29) CometProject +(26) CometProject Input [2]: [d_week_seq#67, d_year#68] Arguments: [d_week_seq#67], [d_week_seq#67] -(30) ColumnarToRow [codegen id : 4] +(27) ColumnarToRow [codegen id : 4] Input [1]: [d_week_seq#67] -(31) BroadcastExchange +(28) BroadcastExchange Input [1]: [d_week_seq#67] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(32) BroadcastHashJoin [codegen id : 5] +(29) BroadcastHashJoin [codegen id : 5] Left keys [1]: [d_week_seq#50] Right keys [1]: [d_week_seq#67] Join type: Inner Join condition: None -(33) Project [codegen id : 5] +(30) Project [codegen id : 5] Output [8]: [d_week_seq#50 AS d_week_seq2#69, sun_sales#60 AS sun_sales2#70, mon_sales#61 AS mon_sales2#71, tue_sales#62 AS tue_sales2#72, wed_sales#63 AS wed_sales2#73, thu_sales#64 AS thu_sales2#74, fri_sales#65 AS fri_sales2#75, sat_sales#66 AS sat_sales2#76] Input [9]: [d_week_seq#50, sun_sales#60, mon_sales#61, tue_sales#62, wed_sales#63, thu_sales#64, fri_sales#65, sat_sales#66, d_week_seq#67] -(34) BroadcastExchange +(31) BroadcastExchange Input [8]: [d_week_seq2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] Arguments: HashedRelationBroadcastMode(List(cast((input[0, int, true] - 53) as bigint)),false), [plan_id=4] -(35) BroadcastHashJoin [codegen id : 6] +(32) BroadcastHashJoin [codegen id : 6] Left keys [1]: [d_week_seq1#42] Right keys [1]: [(d_week_seq2#69 - 53)] Join type: Inner Join condition: None -(36) Project [codegen id : 6] +(33) Project [codegen id : 6] Output [8]: [d_week_seq1#42, round((sun_sales1#43 / sun_sales2#70), 2) AS round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1#44 / mon_sales2#71), 2) AS round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1#45 / tue_sales2#72), 2) AS round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1#46 / wed_sales2#73), 2) AS round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1#47 / thu_sales2#74), 2) AS round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1#48 / fri_sales2#75), 2) AS round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1#49 / sat_sales2#76), 2) AS round((sat_sales1 / sat_sales2), 2)#83] Input [16]: [d_week_seq1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#69, sun_sales2#70, mon_sales2#71, tue_sales2#72, wed_sales2#73, thu_sales2#74, fri_sales2#75, sat_sales2#76] -(37) RowToColumnar +(34) RowToColumnar Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] -(38) CometColumnarExchange +(35) CometColumnarExchange Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] Arguments: rangepartitioning(d_week_seq1#42 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] -(39) CometSort +(36) CometSort Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] Arguments: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83], [d_week_seq1#42 ASC NULLS FIRST] -(40) ColumnarToRow [codegen id : 7] +(37) ColumnarToRow [codegen id : 7] Input [8]: [d_week_seq1#42, round((sun_sales1 / sun_sales2), 2)#77, round((mon_sales1 / mon_sales2), 2)#78, round((tue_sales1 / tue_sales2), 2)#79, round((wed_sales1 / wed_sales2), 2)#80, round((thu_sales1 / thu_sales2), 2)#81, round((fri_sales1 / fri_sales2), 2)#82, round((sat_sales1 / sat_sales2), 2)#83] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt index a4f4107c6..e8d6a74f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q2/simplified.txt @@ -10,24 +10,22 @@ WholeStageCodegen (7) Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] BroadcastHashJoin [d_week_seq,d_week_seq] HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_week_seq] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [d_week_seq,d_day_name,sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,d_week_seq,d_day_name] - CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] - CometUnion [sold_date_sk,sales_price] - CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] - CometScan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] - CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] - CometScan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + InputAdapter + Exchange [d_week_seq] #2 + WholeStageCodegen (1) + HashAggregate [d_week_seq,d_day_name,sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [sales_price,d_week_seq,d_day_name] + CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] + CometUnion [sold_date_sk,sales_price] + CometProject [ws_sold_date_sk,ws_ext_sales_price] [sold_date_sk,sales_price] + CometScan parquet spark_catalog.default.web_sales [ws_ext_sales_price,ws_sold_date_sk] + CometProject [cs_sold_date_sk,cs_ext_sales_price] [sold_date_sk,sales_price] + CometScan parquet spark_catalog.default.catalog_sales [cs_ext_sales_price,cs_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #3 + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] InputAdapter BroadcastExchange #4 WholeStageCodegen (2) @@ -42,9 +40,8 @@ WholeStageCodegen (7) Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] BroadcastHashJoin [d_week_seq,d_week_seq] HashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 + InputAdapter + ReusedExchange [d_week_seq,sum,sum,sum,sum,sum,sum,sum] #2 InputAdapter BroadcastExchange #6 WholeStageCodegen (4) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt index b88b1983b..4bb7f5ca5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/explain.txt @@ -1,30 +1,26 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * Project (25) - +- Window (24) - +- * ColumnarToRow (23) - +- CometSort (22) - +- CometColumnarExchange (21) - +- RowToColumnar (20) - +- * HashAggregate (19) - +- * ColumnarToRow (18) - +- CometColumnarExchange (17) - +- RowToColumnar (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) +TakeOrderedAndProject (22) ++- * Project (21) + +- Window (20) + +- * Sort (19) + +- Exchange (18) + +- * HashAggregate (17) + +- Exchange (16) + +- * HashAggregate (15) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometBroadcastHashJoin (12) + :- CometProject (7) + : +- CometBroadcastHashJoin (6) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (11) + +- CometProject (10) + +- CometFilter (9) + +- CometScan parquet spark_catalog.default.date_dim (8) (1) Scan parquet spark_catalog.default.catalog_sales @@ -101,78 +97,66 @@ Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] Aggregate Attributes [1]: [sum#13] Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -(16) RowToColumnar +(16) Exchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 2] -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] - -(19) HashAggregate [codegen id : 2] +(17) HashAggregate [codegen id : 2] Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#15] Results [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#15,17,2) AS itemrevenue#16, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#15,17,2) AS _w0#17, i_item_id#6] -(20) RowToColumnar -Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] - -(21) CometColumnarExchange -Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(22) CometSort +(18) Exchange Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] -Arguments: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6], [i_class#9 ASC NULLS FIRST] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(23) ColumnarToRow [codegen id : 3] +(19) Sort [codegen id : 3] Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] +Arguments: [i_class#9 ASC NULLS FIRST], false, 0 -(24) Window +(20) Window Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] Arguments: [sum(_w0#17) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#9] -(25) Project [codegen id : 4] +(21) Project [codegen id : 4] Output [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#6] Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6, _we0#18] -(26) TakeOrderedAndProject +(22) TakeOrderedAndProject Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19, i_item_id#6] Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * ColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.date_dim (27) +BroadcastExchange (27) ++- * ColumnarToRow (26) + +- CometProject (25) + +- CometFilter (24) + +- CometScan parquet spark_catalog.default.date_dim (23) -(27) Scan parquet spark_catalog.default.date_dim +(23) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(24) CometFilter Input [2]: [d_date_sk#11, d_date#12] Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 1999-02-22)) AND (d_date#12 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(29) CometProject +(25) CometProject Input [2]: [d_date_sk#11, d_date#12] Arguments: [d_date_sk#11], [d_date_sk#11] -(30) ColumnarToRow [codegen id : 1] +(26) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#11] -(31) BroadcastExchange +(27) BroadcastExchange Input [1]: [d_date_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt index 9eaa25219..5bfd9a907 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q20/simplified.txt @@ -4,39 +4,35 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (3) - ColumnarToRow + Sort [i_class] InputAdapter - CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometColumnarExchange [i_class] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + Exchange [i_class] #1 + WholeStageCodegen (2) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] + InputAdapter + Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt index 5b422574b..b2f81dbc1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/explain.txt @@ -1,30 +1,28 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * Filter (25) - +- * HashAggregate (24) - +- * ColumnarToRow (23) - +- CometColumnarExchange (22) - +- RowToColumnar (21) - +- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.inventory (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.warehouse (3) - : +- CometBroadcastExchange (11) - : +- CometProject (10) - : +- CometFilter (9) - : +- CometScan parquet spark_catalog.default.item (8) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometScan parquet spark_catalog.default.date_dim (14) +TakeOrderedAndProject (24) ++- * Filter (23) + +- * HashAggregate (22) + +- Exchange (21) + +- * HashAggregate (20) + +- * ColumnarToRow (19) + +- CometProject (18) + +- CometBroadcastHashJoin (17) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (7) + : : +- CometBroadcastHashJoin (6) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.warehouse (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometScan parquet spark_catalog.default.item (8) + +- CometBroadcastExchange (16) + +- CometFilter (15) + +- CometScan parquet spark_catalog.default.date_dim (14) (1) Scan parquet spark_catalog.default.inventory @@ -125,55 +123,49 @@ Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity Aggregate Attributes [2]: [sum#13, sum#14] Results [4]: [w_warehouse_name#7, i_item_id#9, sum#15, sum#16] -(21) RowToColumnar +(21) Exchange Input [4]: [w_warehouse_name#7, i_item_id#9, sum#15, sum#16] +Arguments: hashpartitioning(w_warehouse_name#7, i_item_id#9, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(22) CometColumnarExchange -Input [4]: [w_warehouse_name#7, i_item_id#9, sum#15, sum#16] -Arguments: hashpartitioning(w_warehouse_name#7, i_item_id#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(23) ColumnarToRow [codegen id : 2] -Input [4]: [w_warehouse_name#7, i_item_id#9, sum#15, sum#16] - -(24) HashAggregate [codegen id : 2] +(22) HashAggregate [codegen id : 2] Input [4]: [w_warehouse_name#7, i_item_id#9, sum#15, sum#16] Keys [2]: [w_warehouse_name#7, i_item_id#9] Functions [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17, sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#18] Results [4]: [w_warehouse_name#7, i_item_id#9, sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#17 AS inv_before#19, sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)#18 AS inv_after#20] -(25) Filter [codegen id : 2] +(23) Filter [codegen id : 2] Input [4]: [w_warehouse_name#7, i_item_id#9, inv_before#19, inv_after#20] Condition : (CASE WHEN (inv_before#19 > 0) THEN ((cast(inv_after#20 as double) / cast(inv_before#19 as double)) >= 0.666667) END AND CASE WHEN (inv_before#19 > 0) THEN ((cast(inv_after#20 as double) / cast(inv_before#19 as double)) <= 1.5) END) -(26) TakeOrderedAndProject +(24) TakeOrderedAndProject Input [4]: [w_warehouse_name#7, i_item_id#9, inv_before#19, inv_after#20] Arguments: 100, [w_warehouse_name#7 ASC NULLS FIRST, i_item_id#9 ASC NULLS FIRST], [w_warehouse_name#7, i_item_id#9, inv_before#19, inv_after#20] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (30) -+- * ColumnarToRow (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.date_dim (27) +BroadcastExchange (28) ++- * ColumnarToRow (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.date_dim (25) -(27) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [2]: [d_date_sk#11, d_date#12] Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 2000-02-10)) AND (d_date#12 <= 2000-04-10)) AND isnotnull(d_date_sk#11)) -(29) ColumnarToRow [codegen id : 1] +(27) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#11, d_date#12] -(30) BroadcastExchange +(28) BroadcastExchange Input [2]: [d_date_sk#11, d_date#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt index 7747dd616..2c8892b68 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q21/simplified.txt @@ -2,36 +2,34 @@ TakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] WholeStageCodegen (2) Filter [inv_before,inv_after] HashAggregate [w_warehouse_name,i_item_id,sum,sum] [sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),inv_before,inv_after,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [w_warehouse_name,i_item_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,i_item_id,d_date,inv_quantity_on_hand] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] - CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_sk,i_item_id] - CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #3 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometProject [i_item_sk,i_item_id] - CometFilter [i_item_sk,i_item_id,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #5 - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + Exchange [w_warehouse_name,i_item_id] #1 + WholeStageCodegen (1) + HashAggregate [w_warehouse_name,i_item_id,d_date,inv_quantity_on_hand] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] + CometProject [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_sk,i_item_id] + CometProject [inv_item_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #3 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometProject [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id,i_current_price] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometBroadcastExchange [d_date_sk,d_date] #5 + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt index 8e3bf2731..342926b93 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/explain.txt @@ -1,30 +1,28 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometColumnarExchange (23) - +- RowToColumnar (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.inventory (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.item (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometScan parquet spark_catalog.default.warehouse (14) +TakeOrderedAndProject (24) ++- * HashAggregate (23) + +- Exchange (22) + +- * HashAggregate (21) + +- * ColumnarToRow (20) + +- CometExpand (19) + +- CometProject (18) + +- CometBroadcastHashJoin (17) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (16) + +- CometFilter (15) + +- CometScan parquet spark_catalog.default.warehouse (14) (1) Scan parquet spark_catalog.default.inventory @@ -129,56 +127,50 @@ Functions [1]: [partial_avg(inv_quantity_on_hand#3)] Aggregate Attributes [2]: [sum#19, count#20] Results [7]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, sum#21, count#22] -(22) RowToColumnar +(22) Exchange Input [7]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, sum#21, count#22] +Arguments: hashpartitioning(i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(23) CometColumnarExchange -Input [7]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, sum#21, count#22] -Arguments: hashpartitioning(i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(24) ColumnarToRow [codegen id : 2] -Input [7]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, sum#21, count#22] - -(25) HashAggregate [codegen id : 2] +(23) HashAggregate [codegen id : 2] Input [7]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, sum#21, count#22] Keys [5]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18] Functions [1]: [avg(inv_quantity_on_hand#3)] Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#23] Results [5]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, avg(inv_quantity_on_hand#3)#23 AS qoh#24] -(26) TakeOrderedAndProject +(24) TakeOrderedAndProject Input [5]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, qoh#24] Arguments: 100, [qoh#24 ASC NULLS FIRST, i_product_name#14 ASC NULLS FIRST, i_brand#15 ASC NULLS FIRST, i_class#16 ASC NULLS FIRST, i_category#17 ASC NULLS FIRST], [i_product_name#14, i_brand#15, i_class#16, i_category#17, qoh#24] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (31) -+- * ColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.date_dim (27) +BroadcastExchange (29) ++- * ColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.date_dim (25) -(27) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [2]: [d_date_sk#6, d_month_seq#7] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#6)) -(29) CometProject +(27) CometProject Input [2]: [d_date_sk#6, d_month_seq#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(30) ColumnarToRow [codegen id : 1] +(28) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(31) BroadcastExchange +(29) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt index 2bb7b519c..eb25d6d3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q22/simplified.txt @@ -1,38 +1,36 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] WholeStageCodegen (2) HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] - CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #5 - CometFilter [w_warehouse_sk] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk] + InputAdapter + Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + WholeStageCodegen (1) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] + ColumnarToRow + InputAdapter + CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] + CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometBroadcastExchange [w_warehouse_sk] #5 + CometFilter [w_warehouse_sk] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt index a849066c6..7308b0165 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/explain.txt @@ -1,75 +1,70 @@ == Physical Plan == -* HashAggregate (71) -+- * ColumnarToRow (70) - +- CometColumnarExchange (69) - +- RowToColumnar (68) - +- * HashAggregate (67) - +- Union (66) - :- * Project (47) - : +- * BroadcastHashJoin Inner BuildRight (46) - : :- * Project (44) - : : +- * SortMergeJoin LeftSemi (43) - : : :- * ColumnarToRow (25) - : : : +- CometSort (24) - : : : +- CometColumnarExchange (23) - : : : +- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : +- CometBroadcastExchange (20) - : : : +- CometProject (19) - : : : +- CometFilter (18) - : : : +- CometHashAggregate (17) - : : : +- CometColumnarExchange (16) - : : : +- CometHashAggregate (15) - : : : +- CometProject (14) - : : : +- CometBroadcastHashJoin (13) - : : : :- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometFilter (3) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (2) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- CometBroadcastExchange (12) - : : : +- CometFilter (11) - : : : +- CometScan parquet spark_catalog.default.item (10) - : : +- * Sort (42) - : : +- * Project (41) - : : +- * Filter (40) - : : +- * HashAggregate (39) - : : +- * ColumnarToRow (38) - : : +- CometColumnarExchange (37) - : : +- RowToColumnar (36) - : : +- * HashAggregate (35) - : : +- * ColumnarToRow (34) - : : +- CometProject (33) - : : +- CometBroadcastHashJoin (32) - : : :- CometProject (28) - : : : +- CometFilter (27) - : : : +- CometScan parquet spark_catalog.default.store_sales (26) - : : +- CometBroadcastExchange (31) - : : +- CometFilter (30) - : : +- CometScan parquet spark_catalog.default.customer (29) - : +- ReusedExchange (45) - +- * Project (65) - +- * BroadcastHashJoin Inner BuildRight (64) - :- * Project (62) - : +- * SortMergeJoin LeftSemi (61) - : :- * ColumnarToRow (54) - : : +- CometSort (53) - : : +- CometColumnarExchange (52) - : : +- CometProject (51) - : : +- CometBroadcastHashJoin (50) - : : :- CometScan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (49) - : +- * Sort (60) - : +- * Project (59) - : +- * Filter (58) - : +- * HashAggregate (57) - : +- * ColumnarToRow (56) - : +- ReusedExchange (55) - +- ReusedExchange (63) +* HashAggregate (66) ++- Exchange (65) + +- * HashAggregate (64) + +- Union (63) + :- * Project (45) + : +- * BroadcastHashJoin Inner BuildRight (44) + : :- * Project (42) + : : +- * SortMergeJoin LeftSemi (41) + : : :- * ColumnarToRow (25) + : : : +- CometSort (24) + : : : +- CometExchange (23) + : : : +- CometProject (22) + : : : +- CometBroadcastHashJoin (21) + : : : :- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (20) + : : : +- CometProject (19) + : : : +- CometFilter (18) + : : : +- CometHashAggregate (17) + : : : +- CometExchange (16) + : : : +- CometHashAggregate (15) + : : : +- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometFilter (3) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (2) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- CometBroadcastExchange (12) + : : : +- CometFilter (11) + : : : +- CometScan parquet spark_catalog.default.item (10) + : : +- * Sort (40) + : : +- * Project (39) + : : +- * Filter (38) + : : +- * HashAggregate (37) + : : +- Exchange (36) + : : +- * HashAggregate (35) + : : +- * ColumnarToRow (34) + : : +- CometProject (33) + : : +- CometBroadcastHashJoin (32) + : : :- CometProject (28) + : : : +- CometFilter (27) + : : : +- CometScan parquet spark_catalog.default.store_sales (26) + : : +- CometBroadcastExchange (31) + : : +- CometFilter (30) + : : +- CometScan parquet spark_catalog.default.customer (29) + : +- ReusedExchange (43) + +- * Project (62) + +- * BroadcastHashJoin Inner BuildRight (61) + :- * Project (59) + : +- * SortMergeJoin LeftSemi (58) + : :- * ColumnarToRow (52) + : : +- CometSort (51) + : : +- CometExchange (50) + : : +- CometProject (49) + : : +- CometBroadcastHashJoin (48) + : : :- CometScan parquet spark_catalog.default.web_sales (46) + : : +- ReusedExchange (47) + : +- * Sort (57) + : +- * Project (56) + : +- * Filter (55) + : +- * HashAggregate (54) + : +- ReusedExchange (53) + +- ReusedExchange (60) (1) Scan parquet spark_catalog.default.catalog_sales @@ -148,9 +143,9 @@ Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] Functions [1]: [partial_count(1)] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] @@ -178,9 +173,9 @@ Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(23) CometColumnarExchange +(23) CometExchange Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -238,156 +233,141 @@ Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_pr Aggregate Attributes [2]: [sum#24, isEmpty#25] Results [3]: [c_customer_sk#23, sum#26, isEmpty#27] -(36) RowToColumnar -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(37) CometColumnarExchange +(36) Exchange Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(38) ColumnarToRow [codegen id : 3] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(39) HashAggregate [codegen id : 3] +(37) HashAggregate [codegen id : 3] Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] Keys [1]: [c_customer_sk#23] Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] -(40) Filter [codegen id : 3] +(38) Filter [codegen id : 3] Input [2]: [c_customer_sk#23, ssales#29] Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#30, [id=#31]))) -(41) Project [codegen id : 3] +(39) Project [codegen id : 3] Output [1]: [c_customer_sk#23] Input [2]: [c_customer_sk#23, ssales#29] -(42) Sort [codegen id : 3] +(40) Sort [codegen id : 3] Input [1]: [c_customer_sk#23] Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 -(43) SortMergeJoin [codegen id : 5] +(41) SortMergeJoin [codegen id : 5] Left keys [1]: [cs_bill_customer_sk#1] Right keys [1]: [c_customer_sk#23] Join type: LeftSemi Join condition: None -(44) Project [codegen id : 5] +(42) Project [codegen id : 5] Output [3]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(45) ReusedExchange [Reuses operator id: 76] +(43) ReusedExchange [Reuses operator id: 71] Output [1]: [d_date_sk#32] -(46) BroadcastHashJoin [codegen id : 5] +(44) BroadcastHashJoin [codegen id : 5] Left keys [1]: [cs_sold_date_sk#5] Right keys [1]: [d_date_sk#32] Join type: Inner Join condition: None -(47) Project [codegen id : 5] +(45) Project [codegen id : 5] Output [1]: [(cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4) AS sales#33] Input [4]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, d_date_sk#32] -(48) Scan parquet spark_catalog.default.web_sales +(46) Scan parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#38), dynamicpruningexpression(ws_sold_date_sk#38 IN dynamicpruning#39)] ReadSchema: struct -(49) ReusedExchange [Reuses operator id: 20] +(47) ReusedExchange [Reuses operator id: 20] Output [1]: [item_sk#40] -(50) CometBroadcastHashJoin +(48) CometBroadcastHashJoin Left output [5]: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] Right output [1]: [item_sk#40] Arguments: [ws_item_sk#34], [item_sk#40], LeftSemi, BuildRight -(51) CometProject +(49) CometProject Input [5]: [ws_item_sk#34, ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] Arguments: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38], [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -(52) CometColumnarExchange +(50) CometExchange Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -Arguments: hashpartitioning(ws_bill_customer_sk#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ws_bill_customer_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(53) CometSort +(51) CometSort Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] Arguments: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38], [ws_bill_customer_sk#35 ASC NULLS FIRST] -(54) ColumnarToRow [codegen id : 6] +(52) ColumnarToRow [codegen id : 6] Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -(55) ReusedExchange [Reuses operator id: 37] +(53) ReusedExchange [Reuses operator id: 36] Output [3]: [c_customer_sk#41, sum#42, isEmpty#43] -(56) ColumnarToRow [codegen id : 8] -Input [3]: [c_customer_sk#41, sum#42, isEmpty#43] - -(57) HashAggregate [codegen id : 8] +(54) HashAggregate [codegen id : 8] Input [3]: [c_customer_sk#41, sum#42, isEmpty#43] Keys [1]: [c_customer_sk#41] Functions [1]: [sum((cast(ss_quantity#44 as decimal(10,0)) * ss_sales_price#45))] Aggregate Attributes [1]: [sum((cast(ss_quantity#44 as decimal(10,0)) * ss_sales_price#45))#28] Results [2]: [c_customer_sk#41, sum((cast(ss_quantity#44 as decimal(10,0)) * ss_sales_price#45))#28 AS ssales#46] -(58) Filter [codegen id : 8] +(55) Filter [codegen id : 8] Input [2]: [c_customer_sk#41, ssales#46] Condition : (isnotnull(ssales#46) AND (cast(ssales#46 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) -(59) Project [codegen id : 8] +(56) Project [codegen id : 8] Output [1]: [c_customer_sk#41] Input [2]: [c_customer_sk#41, ssales#46] -(60) Sort [codegen id : 8] +(57) Sort [codegen id : 8] Input [1]: [c_customer_sk#41] Arguments: [c_customer_sk#41 ASC NULLS FIRST], false, 0 -(61) SortMergeJoin [codegen id : 10] +(58) SortMergeJoin [codegen id : 10] Left keys [1]: [ws_bill_customer_sk#35] Right keys [1]: [c_customer_sk#41] Join type: LeftSemi Join condition: None -(62) Project [codegen id : 10] +(59) Project [codegen id : 10] Output [3]: [ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] Input [4]: [ws_bill_customer_sk#35, ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38] -(63) ReusedExchange [Reuses operator id: 76] +(60) ReusedExchange [Reuses operator id: 71] Output [1]: [d_date_sk#47] -(64) BroadcastHashJoin [codegen id : 10] +(61) BroadcastHashJoin [codegen id : 10] Left keys [1]: [ws_sold_date_sk#38] Right keys [1]: [d_date_sk#47] Join type: Inner Join condition: None -(65) Project [codegen id : 10] +(62) Project [codegen id : 10] Output [1]: [(cast(ws_quantity#36 as decimal(10,0)) * ws_list_price#37) AS sales#48] Input [4]: [ws_quantity#36, ws_list_price#37, ws_sold_date_sk#38, d_date_sk#47] -(66) Union +(63) Union -(67) HashAggregate [codegen id : 11] +(64) HashAggregate [codegen id : 11] Input [1]: [sales#33] Keys: [] Functions [1]: [partial_sum(sales#33)] Aggregate Attributes [2]: [sum#49, isEmpty#50] Results [2]: [sum#51, isEmpty#52] -(68) RowToColumnar +(65) Exchange Input [2]: [sum#51, isEmpty#52] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] -(69) CometColumnarExchange -Input [2]: [sum#51, isEmpty#52] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(70) ColumnarToRow [codegen id : 12] -Input [2]: [sum#51, isEmpty#52] - -(71) HashAggregate [codegen id : 12] +(66) HashAggregate [codegen id : 12] Input [2]: [sum#51, isEmpty#52] Keys: [] Functions [1]: [sum(sales#33)] @@ -397,91 +377,87 @@ Results [1]: [sum(sales#33)#53 AS sum(sales)#54] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (76) -+- * ColumnarToRow (75) - +- CometProject (74) - +- CometFilter (73) - +- CometScan parquet spark_catalog.default.date_dim (72) +BroadcastExchange (71) ++- * ColumnarToRow (70) + +- CometProject (69) + +- CometFilter (68) + +- CometScan parquet spark_catalog.default.date_dim (67) -(72) Scan parquet spark_catalog.default.date_dim +(67) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#32, d_year#55, d_moy#56] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(73) CometFilter +(68) CometFilter Input [3]: [d_date_sk#32, d_year#55, d_moy#56] Condition : ((((isnotnull(d_year#55) AND isnotnull(d_moy#56)) AND (d_year#55 = 2000)) AND (d_moy#56 = 2)) AND isnotnull(d_date_sk#32)) -(74) CometProject +(69) CometProject Input [3]: [d_date_sk#32, d_year#55, d_moy#56] Arguments: [d_date_sk#32], [d_date_sk#32] -(75) ColumnarToRow [codegen id : 1] +(70) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#32] -(76) BroadcastExchange +(71) BroadcastExchange Input [1]: [d_date_sk#32] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] Subquery:2 Hosting operator id = 2 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (81) -+- * ColumnarToRow (80) - +- CometProject (79) - +- CometFilter (78) - +- CometScan parquet spark_catalog.default.date_dim (77) +BroadcastExchange (76) ++- * ColumnarToRow (75) + +- CometProject (74) + +- CometFilter (73) + +- CometScan parquet spark_catalog.default.date_dim (72) -(77) Scan parquet spark_catalog.default.date_dim +(72) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(78) CometFilter +(73) CometFilter Input [3]: [d_date_sk#10, d_date#11, d_year#12] Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) -(79) CometProject +(74) CometProject Input [3]: [d_date_sk#10, d_date#11, d_year#12] Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] -(80) ColumnarToRow [codegen id : 1] +(75) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#10, d_date#11] -(81) BroadcastExchange +(76) BroadcastExchange Input [2]: [d_date_sk#10, d_date#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] -Subquery:3 Hosting operator id = 40 Hosting Expression = Subquery scalar-subquery#30, [id=#31] -* HashAggregate (103) -+- * ColumnarToRow (102) - +- CometColumnarExchange (101) - +- RowToColumnar (100) - +- * HashAggregate (99) - +- * HashAggregate (98) - +- * ColumnarToRow (97) - +- CometColumnarExchange (96) - +- RowToColumnar (95) - +- * HashAggregate (94) - +- * ColumnarToRow (93) - +- CometProject (92) - +- CometBroadcastHashJoin (91) - :- CometProject (86) - : +- CometBroadcastHashJoin (85) - : :- CometFilter (83) - : : +- CometScan parquet spark_catalog.default.store_sales (82) - : +- ReusedExchange (84) - +- CometBroadcastExchange (90) - +- CometProject (89) - +- CometFilter (88) - +- CometScan parquet spark_catalog.default.date_dim (87) - - -(82) Scan parquet spark_catalog.default.store_sales +Subquery:3 Hosting operator id = 38 Hosting Expression = Subquery scalar-subquery#30, [id=#31] +* HashAggregate (94) ++- Exchange (93) + +- * HashAggregate (92) + +- * HashAggregate (91) + +- Exchange (90) + +- * HashAggregate (89) + +- * ColumnarToRow (88) + +- CometProject (87) + +- CometBroadcastHashJoin (86) + :- CometProject (81) + : +- CometBroadcastHashJoin (80) + : :- CometFilter (78) + : : +- CometScan parquet spark_catalog.default.store_sales (77) + : +- ReusedExchange (79) + +- CometBroadcastExchange (85) + +- CometProject (84) + +- CometFilter (83) + +- CometScan parquet spark_catalog.default.date_dim (82) + + +(77) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] Batched: true Location: InMemoryFileIndex [] @@ -489,133 +465,121 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#60), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(83) CometFilter +(78) CometFilter Input [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] Condition : isnotnull(ss_customer_sk#57) -(84) ReusedExchange [Reuses operator id: 31] +(79) ReusedExchange [Reuses operator id: 31] Output [1]: [c_customer_sk#62] -(85) CometBroadcastHashJoin +(80) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60] Right output [1]: [c_customer_sk#62] Arguments: [ss_customer_sk#57], [c_customer_sk#62], Inner, BuildRight -(86) CometProject +(81) CometProject Input [5]: [ss_customer_sk#57, ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#62] Arguments: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#62], [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#62] -(87) Scan parquet spark_catalog.default.date_dim +(82) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#63, d_year#64] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(88) CometFilter +(83) CometFilter Input [2]: [d_date_sk#63, d_year#64] Condition : (d_year#64 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#63)) -(89) CometProject +(84) CometProject Input [2]: [d_date_sk#63, d_year#64] Arguments: [d_date_sk#63], [d_date_sk#63] -(90) CometBroadcastExchange +(85) CometBroadcastExchange Input [1]: [d_date_sk#63] Arguments: [d_date_sk#63] -(91) CometBroadcastHashJoin +(86) CometBroadcastHashJoin Left output [4]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#62] Right output [1]: [d_date_sk#63] Arguments: [ss_sold_date_sk#60], [d_date_sk#63], Inner, BuildRight -(92) CometProject +(87) CometProject Input [5]: [ss_quantity#58, ss_sales_price#59, ss_sold_date_sk#60, c_customer_sk#62, d_date_sk#63] Arguments: [ss_quantity#58, ss_sales_price#59, c_customer_sk#62], [ss_quantity#58, ss_sales_price#59, c_customer_sk#62] -(93) ColumnarToRow [codegen id : 1] +(88) ColumnarToRow [codegen id : 1] Input [3]: [ss_quantity#58, ss_sales_price#59, c_customer_sk#62] -(94) HashAggregate [codegen id : 1] +(89) HashAggregate [codegen id : 1] Input [3]: [ss_quantity#58, ss_sales_price#59, c_customer_sk#62] Keys [1]: [c_customer_sk#62] Functions [1]: [partial_sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] Aggregate Attributes [2]: [sum#65, isEmpty#66] Results [3]: [c_customer_sk#62, sum#67, isEmpty#68] -(95) RowToColumnar +(90) Exchange Input [3]: [c_customer_sk#62, sum#67, isEmpty#68] +Arguments: hashpartitioning(c_customer_sk#62, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(96) CometColumnarExchange -Input [3]: [c_customer_sk#62, sum#67, isEmpty#68] -Arguments: hashpartitioning(c_customer_sk#62, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(97) ColumnarToRow [codegen id : 2] -Input [3]: [c_customer_sk#62, sum#67, isEmpty#68] - -(98) HashAggregate [codegen id : 2] +(91) HashAggregate [codegen id : 2] Input [3]: [c_customer_sk#62, sum#67, isEmpty#68] Keys [1]: [c_customer_sk#62] Functions [1]: [sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))] Aggregate Attributes [1]: [sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))#69] Results [1]: [sum((cast(ss_quantity#58 as decimal(10,0)) * ss_sales_price#59))#69 AS csales#70] -(99) HashAggregate [codegen id : 2] +(92) HashAggregate [codegen id : 2] Input [1]: [csales#70] Keys: [] Functions [1]: [partial_max(csales#70)] Aggregate Attributes [1]: [max#71] Results [1]: [max#72] -(100) RowToColumnar -Input [1]: [max#72] - -(101) CometColumnarExchange -Input [1]: [max#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(102) ColumnarToRow [codegen id : 3] +(93) Exchange Input [1]: [max#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] -(103) HashAggregate [codegen id : 3] +(94) HashAggregate [codegen id : 3] Input [1]: [max#72] Keys: [] Functions [1]: [max(csales#70)] Aggregate Attributes [1]: [max(csales#70)#73] Results [1]: [max(csales#70)#73 AS tpcds_cmax#74] -Subquery:4 Hosting operator id = 82 Hosting Expression = ss_sold_date_sk#60 IN dynamicpruning#61 -BroadcastExchange (108) -+- * ColumnarToRow (107) - +- CometProject (106) - +- CometFilter (105) - +- CometScan parquet spark_catalog.default.date_dim (104) +Subquery:4 Hosting operator id = 77 Hosting Expression = ss_sold_date_sk#60 IN dynamicpruning#61 +BroadcastExchange (99) ++- * ColumnarToRow (98) + +- CometProject (97) + +- CometFilter (96) + +- CometScan parquet spark_catalog.default.date_dim (95) -(104) Scan parquet spark_catalog.default.date_dim +(95) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#63, d_year#64] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(105) CometFilter +(96) CometFilter Input [2]: [d_date_sk#63, d_year#64] Condition : (d_year#64 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#63)) -(106) CometProject +(97) CometProject Input [2]: [d_date_sk#63, d_year#64] Arguments: [d_date_sk#63], [d_date_sk#63] -(107) ColumnarToRow [codegen id : 1] +(98) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#63] -(108) BroadcastExchange +(99) BroadcastExchange Input [1]: [d_date_sk#63] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] -Subquery:5 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#38 IN dynamicpruning#6 +Subquery:5 Hosting operator id = 46 Hosting Expression = ws_sold_date_sk#38 IN dynamicpruning#6 -Subquery:6 Hosting operator id = 58 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] +Subquery:6 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt index 317d50c28..760cbdd8b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23a/simplified.txt @@ -1,148 +1,139 @@ WholeStageCodegen (12) HashAggregate [sum,isEmpty] [sum(sales),sum(sales),sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (11) - HashAggregate [sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (5) - Project [cs_quantity,cs_list_price] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometColumnarExchange [cs_bill_customer_sk] #2 - CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [item_sk] #4 - CometProject [item_sk] - CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometColumnarExchange [_groupingexpression,i_item_sk,d_date] #5 - CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] - CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] - CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] - CometProject [ss_item_sk,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [d_date_sk,d_date] #7 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #8 - CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (3) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #3 - WholeStageCodegen (3) - HashAggregate [max] [max(csales),tpcds_cmax,max] - ColumnarToRow - InputAdapter - CometColumnarExchange #11 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_sk] #12 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange #1 + WholeStageCodegen (11) + HashAggregate [sales] [sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (5) + Project [cs_quantity,cs_list_price] + BroadcastHashJoin [cs_sold_date_sk,d_date_sk] + Project [cs_quantity,cs_list_price,cs_sold_date_sk] + SortMergeJoin [cs_bill_customer_sk,c_customer_sk] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometExchange [cs_bill_customer_sk] #2 + CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [item_sk] #4 + CometProject [item_sk] + CometFilter [item_sk,cnt] + CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] + CometExchange [_groupingexpression,i_item_sk,d_date] #5 + CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] + CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] + CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] + CometProject [ss_item_sk,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #6 + WholeStageCodegen (1) ColumnarToRow InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] - CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #13 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [c_customer_sk] #10 - CometBroadcastExchange [d_date_sk] #14 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_sk] #9 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #10 - CometFilter [c_customer_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk] + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [d_date_sk,d_date] #7 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [i_item_sk,i_item_desc] #8 + CometFilter [i_item_sk,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + InputAdapter + WholeStageCodegen (3) + Sort [c_customer_sk] + Project [c_customer_sk] + Filter [ssales] + Subquery #3 + WholeStageCodegen (3) + HashAggregate [max] [max(csales),tpcds_cmax,max] + InputAdapter + Exchange #11 + WholeStageCodegen (2) + HashAggregate [csales] [max,max] + HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] + InputAdapter + Exchange [c_customer_sk] #12 + WholeStageCodegen (1) + HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] + CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #4 + BroadcastExchange #13 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [c_customer_sk] #10 + CometBroadcastExchange [d_date_sk] #14 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] + InputAdapter + Exchange [c_customer_sk] #9 + WholeStageCodegen (2) + HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [c_customer_sk] #10 + CometFilter [c_customer_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk] + InputAdapter + ReusedExchange [d_date_sk] #3 + WholeStageCodegen (10) + Project [ws_quantity,ws_list_price] + BroadcastHashJoin [ws_sold_date_sk,d_date_sk] + Project [ws_quantity,ws_list_price,ws_sold_date_sk] + SortMergeJoin [ws_bill_customer_sk,c_customer_sk] InputAdapter - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (10) - Project [ws_quantity,ws_list_price] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometColumnarExchange [ws_bill_customer_sk] #15 - CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [item_sk] #4 - InputAdapter - WholeStageCodegen (8) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #3 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #9 + WholeStageCodegen (6) + ColumnarToRow + InputAdapter + CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometExchange [ws_bill_customer_sk] #15 + CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [item_sk] #4 InputAdapter - ReusedExchange [d_date_sk] #3 + WholeStageCodegen (8) + Sort [c_customer_sk] + Project [c_customer_sk] + Filter [ssales] + ReusedSubquery [tpcds_cmax] #3 + HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] + InputAdapter + ReusedExchange [c_customer_sk,sum,isEmpty] #9 + InputAdapter + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt index 876b496c5..4e2bbdace 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/explain.txt @@ -1,99 +1,91 @@ == Physical Plan == -TakeOrderedAndProject (95) -+- Union (94) - :- * HashAggregate (67) - : +- * ColumnarToRow (66) - : +- CometColumnarExchange (65) - : +- RowToColumnar (64) - : +- * HashAggregate (63) - : +- * Project (62) - : +- * BroadcastHashJoin Inner BuildRight (61) - : :- * Project (59) - : : +- * BroadcastHashJoin Inner BuildRight (58) - : : :- * SortMergeJoin LeftSemi (44) - : : : :- * ColumnarToRow (26) - : : : : +- CometSort (25) - : : : : +- CometColumnarExchange (24) - : : : : +- CometProject (23) - : : : : +- CometBroadcastHashJoin (22) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : +- CometBroadcastExchange (21) - : : : : +- CometProject (20) - : : : : +- CometFilter (19) - : : : : +- CometHashAggregate (18) - : : : : +- CometColumnarExchange (17) - : : : : +- CometHashAggregate (16) - : : : : +- CometProject (15) - : : : : +- CometBroadcastHashJoin (14) - : : : : :- CometProject (10) - : : : : : +- CometBroadcastHashJoin (9) - : : : : : :- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : : : +- CometBroadcastExchange (8) - : : : : : +- CometProject (7) - : : : : : +- CometFilter (6) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (5) - : : : : +- CometBroadcastExchange (13) - : : : : +- CometFilter (12) - : : : : +- CometScan parquet spark_catalog.default.item (11) - : : : +- * Sort (43) - : : : +- * Project (42) - : : : +- * Filter (41) - : : : +- * HashAggregate (40) - : : : +- * ColumnarToRow (39) - : : : +- CometColumnarExchange (38) - : : : +- RowToColumnar (37) - : : : +- * HashAggregate (36) - : : : +- * ColumnarToRow (35) - : : : +- CometProject (34) - : : : +- CometBroadcastHashJoin (33) - : : : :- CometProject (29) - : : : : +- CometFilter (28) - : : : : +- CometScan parquet spark_catalog.default.store_sales (27) - : : : +- CometBroadcastExchange (32) - : : : +- CometFilter (31) - : : : +- CometScan parquet spark_catalog.default.customer (30) - : : +- BroadcastExchange (57) - : : +- * SortMergeJoin LeftSemi (56) - : : :- * ColumnarToRow (49) - : : : +- CometSort (48) - : : : +- CometColumnarExchange (47) - : : : +- CometFilter (46) - : : : +- CometScan parquet spark_catalog.default.customer (45) - : : +- * Sort (55) - : : +- * Project (54) - : : +- * Filter (53) - : : +- * HashAggregate (52) - : : +- * ColumnarToRow (51) - : : +- ReusedExchange (50) - : +- ReusedExchange (60) - +- * HashAggregate (93) - +- * ColumnarToRow (92) - +- CometColumnarExchange (91) - +- RowToColumnar (90) - +- * HashAggregate (89) - +- * Project (88) - +- * BroadcastHashJoin Inner BuildRight (87) - :- * Project (85) - : +- * BroadcastHashJoin Inner BuildRight (84) - : :- * SortMergeJoin LeftSemi (82) - : : :- * ColumnarToRow (75) - : : : +- CometSort (74) - : : : +- CometColumnarExchange (73) - : : : +- CometProject (72) - : : : +- CometBroadcastHashJoin (71) - : : : :- CometFilter (69) - : : : : +- CometScan parquet spark_catalog.default.web_sales (68) - : : : +- ReusedExchange (70) - : : +- * Sort (81) - : : +- * Project (80) - : : +- * Filter (79) - : : +- * HashAggregate (78) - : : +- * ColumnarToRow (77) - : : +- ReusedExchange (76) - : +- ReusedExchange (83) - +- ReusedExchange (86) +TakeOrderedAndProject (87) ++- Union (86) + :- * HashAggregate (62) + : +- Exchange (61) + : +- * HashAggregate (60) + : +- * Project (59) + : +- * BroadcastHashJoin Inner BuildRight (58) + : :- * Project (56) + : : +- * BroadcastHashJoin Inner BuildRight (55) + : : :- * SortMergeJoin LeftSemi (42) + : : : :- * ColumnarToRow (26) + : : : : +- CometSort (25) + : : : : +- CometExchange (24) + : : : : +- CometProject (23) + : : : : +- CometBroadcastHashJoin (22) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (21) + : : : : +- CometProject (20) + : : : : +- CometFilter (19) + : : : : +- CometHashAggregate (18) + : : : : +- CometExchange (17) + : : : : +- CometHashAggregate (16) + : : : : +- CometProject (15) + : : : : +- CometBroadcastHashJoin (14) + : : : : :- CometProject (10) + : : : : : +- CometBroadcastHashJoin (9) + : : : : : :- CometFilter (4) + : : : : : : +- CometScan parquet spark_catalog.default.store_sales (3) + : : : : : +- CometBroadcastExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (5) + : : : : +- CometBroadcastExchange (13) + : : : : +- CometFilter (12) + : : : : +- CometScan parquet spark_catalog.default.item (11) + : : : +- * Sort (41) + : : : +- * Project (40) + : : : +- * Filter (39) + : : : +- * HashAggregate (38) + : : : +- Exchange (37) + : : : +- * HashAggregate (36) + : : : +- * ColumnarToRow (35) + : : : +- CometProject (34) + : : : +- CometBroadcastHashJoin (33) + : : : :- CometProject (29) + : : : : +- CometFilter (28) + : : : : +- CometScan parquet spark_catalog.default.store_sales (27) + : : : +- CometBroadcastExchange (32) + : : : +- CometFilter (31) + : : : +- CometScan parquet spark_catalog.default.customer (30) + : : +- BroadcastExchange (54) + : : +- * SortMergeJoin LeftSemi (53) + : : :- * ColumnarToRow (47) + : : : +- CometSort (46) + : : : +- CometExchange (45) + : : : +- CometFilter (44) + : : : +- CometScan parquet spark_catalog.default.customer (43) + : : +- * Sort (52) + : : +- * Project (51) + : : +- * Filter (50) + : : +- * HashAggregate (49) + : : +- ReusedExchange (48) + : +- ReusedExchange (57) + +- * HashAggregate (85) + +- Exchange (84) + +- * HashAggregate (83) + +- * Project (82) + +- * BroadcastHashJoin Inner BuildRight (81) + :- * Project (79) + : +- * BroadcastHashJoin Inner BuildRight (78) + : :- * SortMergeJoin LeftSemi (76) + : : :- * ColumnarToRow (70) + : : : +- CometSort (69) + : : : +- CometExchange (68) + : : : +- CometProject (67) + : : : +- CometBroadcastHashJoin (66) + : : : :- CometFilter (64) + : : : : +- CometScan parquet spark_catalog.default.web_sales (63) + : : : +- ReusedExchange (65) + : : +- * Sort (75) + : : +- * Project (74) + : : +- * Filter (73) + : : +- * HashAggregate (72) + : : +- ReusedExchange (71) + : +- ReusedExchange (77) + +- ReusedExchange (80) (1) Scan parquet spark_catalog.default.catalog_sales @@ -177,9 +169,9 @@ Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] Functions [1]: [partial_count(1)] -(17) CometColumnarExchange +(17) CometExchange Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] @@ -207,9 +199,9 @@ Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(24) CometColumnarExchange +(24) CometExchange Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (25) CometSort Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -267,146 +259,131 @@ Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_pr Aggregate Attributes [2]: [sum#24, isEmpty#25] Results [3]: [c_customer_sk#23, sum#26, isEmpty#27] -(37) RowToColumnar +(37) Exchange Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] +Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(38) CometColumnarExchange -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(39) ColumnarToRow [codegen id : 3] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(40) HashAggregate [codegen id : 3] +(38) HashAggregate [codegen id : 3] Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] Keys [1]: [c_customer_sk#23] Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] -(41) Filter [codegen id : 3] +(39) Filter [codegen id : 3] Input [2]: [c_customer_sk#23, ssales#29] Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * Subquery scalar-subquery#30, [id=#31]))) -(42) Project [codegen id : 3] +(40) Project [codegen id : 3] Output [1]: [c_customer_sk#23] Input [2]: [c_customer_sk#23, ssales#29] -(43) Sort [codegen id : 3] +(41) Sort [codegen id : 3] Input [1]: [c_customer_sk#23] Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 -(44) SortMergeJoin [codegen id : 9] +(42) SortMergeJoin [codegen id : 9] Left keys [1]: [cs_bill_customer_sk#1] Right keys [1]: [c_customer_sk#23] Join type: LeftSemi Join condition: None -(45) Scan parquet spark_catalog.default.customer +(43) Scan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(46) CometFilter +(44) CometFilter Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] Condition : isnotnull(c_customer_sk#32) -(47) CometColumnarExchange +(45) CometExchange Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -Arguments: hashpartitioning(c_customer_sk#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_sk#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(48) CometSort +(46) CometSort Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] Arguments: [c_customer_sk#32, c_first_name#33, c_last_name#34], [c_customer_sk#32 ASC NULLS FIRST] -(49) ColumnarToRow [codegen id : 4] +(47) ColumnarToRow [codegen id : 4] Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] -(50) ReusedExchange [Reuses operator id: 38] +(48) ReusedExchange [Reuses operator id: 37] Output [3]: [c_customer_sk#23, sum#26, isEmpty#27] -(51) ColumnarToRow [codegen id : 6] -Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] - -(52) HashAggregate [codegen id : 6] +(49) HashAggregate [codegen id : 6] Input [3]: [c_customer_sk#23, sum#26, isEmpty#27] Keys [1]: [c_customer_sk#23] Functions [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] Aggregate Attributes [1]: [sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28] Results [2]: [c_customer_sk#23, sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))#28 AS ssales#29] -(53) Filter [codegen id : 6] +(50) Filter [codegen id : 6] Input [2]: [c_customer_sk#23, ssales#29] Condition : (isnotnull(ssales#29) AND (cast(ssales#29 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) -(54) Project [codegen id : 6] +(51) Project [codegen id : 6] Output [1]: [c_customer_sk#23] Input [2]: [c_customer_sk#23, ssales#29] -(55) Sort [codegen id : 6] +(52) Sort [codegen id : 6] Input [1]: [c_customer_sk#23] Arguments: [c_customer_sk#23 ASC NULLS FIRST], false, 0 -(56) SortMergeJoin [codegen id : 7] +(53) SortMergeJoin [codegen id : 7] Left keys [1]: [c_customer_sk#32] Right keys [1]: [c_customer_sk#23] Join type: LeftSemi Join condition: None -(57) BroadcastExchange +(54) BroadcastExchange Input [3]: [c_customer_sk#32, c_first_name#33, c_last_name#34] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(58) BroadcastHashJoin [codegen id : 9] +(55) BroadcastHashJoin [codegen id : 9] Left keys [1]: [cs_bill_customer_sk#1] Right keys [1]: [c_customer_sk#32] Join type: Inner Join condition: None -(59) Project [codegen id : 9] +(56) Project [codegen id : 9] Output [5]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#33, c_last_name#34] Input [7]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_customer_sk#32, c_first_name#33, c_last_name#34] -(60) ReusedExchange [Reuses operator id: 100] +(57) ReusedExchange [Reuses operator id: 92] Output [1]: [d_date_sk#35] -(61) BroadcastHashJoin [codegen id : 9] +(58) BroadcastHashJoin [codegen id : 9] Left keys [1]: [cs_sold_date_sk#5] Right keys [1]: [d_date_sk#35] Join type: Inner Join condition: None -(62) Project [codegen id : 9] +(59) Project [codegen id : 9] Output [4]: [cs_quantity#3, cs_list_price#4, c_first_name#33, c_last_name#34] Input [6]: [cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5, c_first_name#33, c_last_name#34, d_date_sk#35] -(63) HashAggregate [codegen id : 9] +(60) HashAggregate [codegen id : 9] Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#33, c_last_name#34] Keys [2]: [c_last_name#34, c_first_name#33] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] Aggregate Attributes [2]: [sum#36, isEmpty#37] Results [4]: [c_last_name#34, c_first_name#33, sum#38, isEmpty#39] -(64) RowToColumnar -Input [4]: [c_last_name#34, c_first_name#33, sum#38, isEmpty#39] - -(65) CometColumnarExchange -Input [4]: [c_last_name#34, c_first_name#33, sum#38, isEmpty#39] -Arguments: hashpartitioning(c_last_name#34, c_first_name#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(66) ColumnarToRow [codegen id : 10] +(61) Exchange Input [4]: [c_last_name#34, c_first_name#33, sum#38, isEmpty#39] +Arguments: hashpartitioning(c_last_name#34, c_first_name#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(67) HashAggregate [codegen id : 10] +(62) HashAggregate [codegen id : 10] Input [4]: [c_last_name#34, c_first_name#33, sum#38, isEmpty#39] Keys [2]: [c_last_name#34, c_first_name#33] Functions [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] Aggregate Attributes [1]: [sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#40] Results [3]: [c_last_name#34, c_first_name#33, sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))#40 AS sales#41] -(68) Scan parquet spark_catalog.default.web_sales +(63) Scan parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] @@ -414,208 +391,195 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#46), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(69) CometFilter +(64) CometFilter Input [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] Condition : isnotnull(ws_bill_customer_sk#43) -(70) ReusedExchange [Reuses operator id: 21] +(65) ReusedExchange [Reuses operator id: 21] Output [1]: [item_sk#48] -(71) CometBroadcastHashJoin +(66) CometBroadcastHashJoin Left output [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] Right output [1]: [item_sk#48] Arguments: [ws_item_sk#42], [item_sk#48], LeftSemi, BuildRight -(72) CometProject +(67) CometProject Input [5]: [ws_item_sk#42, ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] Arguments: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46], [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -(73) CometColumnarExchange +(68) CometExchange Input [4]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -Arguments: hashpartitioning(ws_bill_customer_sk#43, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(ws_bill_customer_sk#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(74) CometSort +(69) CometSort Input [4]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] Arguments: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46], [ws_bill_customer_sk#43 ASC NULLS FIRST] -(75) ColumnarToRow [codegen id : 11] +(70) ColumnarToRow [codegen id : 11] Input [4]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46] -(76) ReusedExchange [Reuses operator id: 38] +(71) ReusedExchange [Reuses operator id: 37] Output [3]: [c_customer_sk#49, sum#50, isEmpty#51] -(77) ColumnarToRow [codegen id : 13] -Input [3]: [c_customer_sk#49, sum#50, isEmpty#51] - -(78) HashAggregate [codegen id : 13] +(72) HashAggregate [codegen id : 13] Input [3]: [c_customer_sk#49, sum#50, isEmpty#51] Keys [1]: [c_customer_sk#49] Functions [1]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_sales_price#53))] Aggregate Attributes [1]: [sum((cast(ss_quantity#52 as decimal(10,0)) * ss_sales_price#53))#28] Results [2]: [c_customer_sk#49, sum((cast(ss_quantity#52 as decimal(10,0)) * ss_sales_price#53))#28 AS ssales#54] -(79) Filter [codegen id : 13] +(73) Filter [codegen id : 13] Input [2]: [c_customer_sk#49, ssales#54] Condition : (isnotnull(ssales#54) AND (cast(ssales#54 as decimal(38,8)) > (0.500000 * ReusedSubquery Subquery scalar-subquery#30, [id=#31]))) -(80) Project [codegen id : 13] +(74) Project [codegen id : 13] Output [1]: [c_customer_sk#49] Input [2]: [c_customer_sk#49, ssales#54] -(81) Sort [codegen id : 13] +(75) Sort [codegen id : 13] Input [1]: [c_customer_sk#49] Arguments: [c_customer_sk#49 ASC NULLS FIRST], false, 0 -(82) SortMergeJoin [codegen id : 19] +(76) SortMergeJoin [codegen id : 19] Left keys [1]: [ws_bill_customer_sk#43] Right keys [1]: [c_customer_sk#49] Join type: LeftSemi Join condition: None -(83) ReusedExchange [Reuses operator id: 57] +(77) ReusedExchange [Reuses operator id: 54] Output [3]: [c_customer_sk#55, c_first_name#56, c_last_name#57] -(84) BroadcastHashJoin [codegen id : 19] +(78) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ws_bill_customer_sk#43] Right keys [1]: [c_customer_sk#55] Join type: Inner Join condition: None -(85) Project [codegen id : 19] +(79) Project [codegen id : 19] Output [5]: [ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46, c_first_name#56, c_last_name#57] Input [7]: [ws_bill_customer_sk#43, ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46, c_customer_sk#55, c_first_name#56, c_last_name#57] -(86) ReusedExchange [Reuses operator id: 100] +(80) ReusedExchange [Reuses operator id: 92] Output [1]: [d_date_sk#58] -(87) BroadcastHashJoin [codegen id : 19] +(81) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ws_sold_date_sk#46] Right keys [1]: [d_date_sk#58] Join type: Inner Join condition: None -(88) Project [codegen id : 19] +(82) Project [codegen id : 19] Output [4]: [ws_quantity#44, ws_list_price#45, c_first_name#56, c_last_name#57] Input [6]: [ws_quantity#44, ws_list_price#45, ws_sold_date_sk#46, c_first_name#56, c_last_name#57, d_date_sk#58] -(89) HashAggregate [codegen id : 19] +(83) HashAggregate [codegen id : 19] Input [4]: [ws_quantity#44, ws_list_price#45, c_first_name#56, c_last_name#57] Keys [2]: [c_last_name#57, c_first_name#56] Functions [1]: [partial_sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))] Aggregate Attributes [2]: [sum#59, isEmpty#60] Results [4]: [c_last_name#57, c_first_name#56, sum#61, isEmpty#62] -(90) RowToColumnar -Input [4]: [c_last_name#57, c_first_name#56, sum#61, isEmpty#62] - -(91) CometColumnarExchange +(84) Exchange Input [4]: [c_last_name#57, c_first_name#56, sum#61, isEmpty#62] -Arguments: hashpartitioning(c_last_name#57, c_first_name#56, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(c_last_name#57, c_first_name#56, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(92) ColumnarToRow [codegen id : 20] -Input [4]: [c_last_name#57, c_first_name#56, sum#61, isEmpty#62] - -(93) HashAggregate [codegen id : 20] +(85) HashAggregate [codegen id : 20] Input [4]: [c_last_name#57, c_first_name#56, sum#61, isEmpty#62] Keys [2]: [c_last_name#57, c_first_name#56] Functions [1]: [sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))] Aggregate Attributes [1]: [sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))#63] Results [3]: [c_last_name#57, c_first_name#56, sum((cast(ws_quantity#44 as decimal(10,0)) * ws_list_price#45))#63 AS sales#64] -(94) Union +(86) Union -(95) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [3]: [c_last_name#34, c_first_name#33, sales#41] Arguments: 100, [c_last_name#34 ASC NULLS FIRST, c_first_name#33 ASC NULLS FIRST, sales#41 ASC NULLS FIRST], [c_last_name#34, c_first_name#33, sales#41] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (100) -+- * ColumnarToRow (99) - +- CometProject (98) - +- CometFilter (97) - +- CometScan parquet spark_catalog.default.date_dim (96) +BroadcastExchange (92) ++- * ColumnarToRow (91) + +- CometProject (90) + +- CometFilter (89) + +- CometScan parquet spark_catalog.default.date_dim (88) -(96) Scan parquet spark_catalog.default.date_dim +(88) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#35, d_year#65, d_moy#66] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(97) CometFilter +(89) CometFilter Input [3]: [d_date_sk#35, d_year#65, d_moy#66] Condition : ((((isnotnull(d_year#65) AND isnotnull(d_moy#66)) AND (d_year#65 = 2000)) AND (d_moy#66 = 2)) AND isnotnull(d_date_sk#35)) -(98) CometProject +(90) CometProject Input [3]: [d_date_sk#35, d_year#65, d_moy#66] Arguments: [d_date_sk#35], [d_date_sk#35] -(99) ColumnarToRow [codegen id : 1] +(91) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#35] -(100) BroadcastExchange +(92) BroadcastExchange Input [1]: [d_date_sk#35] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] Subquery:2 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (105) -+- * ColumnarToRow (104) - +- CometProject (103) - +- CometFilter (102) - +- CometScan parquet spark_catalog.default.date_dim (101) +BroadcastExchange (97) ++- * ColumnarToRow (96) + +- CometProject (95) + +- CometFilter (94) + +- CometScan parquet spark_catalog.default.date_dim (93) -(101) Scan parquet spark_catalog.default.date_dim +(93) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_date#11, d_year#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(102) CometFilter +(94) CometFilter Input [3]: [d_date_sk#10, d_date#11, d_year#12] Condition : (d_year#12 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#10)) -(103) CometProject +(95) CometProject Input [3]: [d_date_sk#10, d_date#11, d_year#12] Arguments: [d_date_sk#10, d_date#11], [d_date_sk#10, d_date#11] -(104) ColumnarToRow [codegen id : 1] +(96) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#10, d_date#11] -(105) BroadcastExchange +(97) BroadcastExchange Input [2]: [d_date_sk#10, d_date#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=10] -Subquery:3 Hosting operator id = 41 Hosting Expression = Subquery scalar-subquery#30, [id=#31] -* HashAggregate (127) -+- * ColumnarToRow (126) - +- CometColumnarExchange (125) - +- RowToColumnar (124) - +- * HashAggregate (123) - +- * HashAggregate (122) - +- * ColumnarToRow (121) - +- CometColumnarExchange (120) - +- RowToColumnar (119) - +- * HashAggregate (118) - +- * ColumnarToRow (117) - +- CometProject (116) - +- CometBroadcastHashJoin (115) - :- CometProject (110) - : +- CometBroadcastHashJoin (109) - : :- CometFilter (107) - : : +- CometScan parquet spark_catalog.default.store_sales (106) - : +- ReusedExchange (108) - +- CometBroadcastExchange (114) - +- CometProject (113) - +- CometFilter (112) - +- CometScan parquet spark_catalog.default.date_dim (111) - - -(106) Scan parquet spark_catalog.default.store_sales +Subquery:3 Hosting operator id = 39 Hosting Expression = Subquery scalar-subquery#30, [id=#31] +* HashAggregate (115) ++- Exchange (114) + +- * HashAggregate (113) + +- * HashAggregate (112) + +- Exchange (111) + +- * HashAggregate (110) + +- * ColumnarToRow (109) + +- CometProject (108) + +- CometBroadcastHashJoin (107) + :- CometProject (102) + : +- CometBroadcastHashJoin (101) + : :- CometFilter (99) + : : +- CometScan parquet spark_catalog.default.store_sales (98) + : +- ReusedExchange (100) + +- CometBroadcastExchange (106) + +- CometProject (105) + +- CometFilter (104) + +- CometScan parquet spark_catalog.default.date_dim (103) + + +(98) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#67, ss_quantity#68, ss_sales_price#69, ss_sold_date_sk#70] Batched: true Location: InMemoryFileIndex [] @@ -623,135 +587,123 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#70), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(107) CometFilter +(99) CometFilter Input [4]: [ss_customer_sk#67, ss_quantity#68, ss_sales_price#69, ss_sold_date_sk#70] Condition : isnotnull(ss_customer_sk#67) -(108) ReusedExchange [Reuses operator id: 32] +(100) ReusedExchange [Reuses operator id: 32] Output [1]: [c_customer_sk#72] -(109) CometBroadcastHashJoin +(101) CometBroadcastHashJoin Left output [4]: [ss_customer_sk#67, ss_quantity#68, ss_sales_price#69, ss_sold_date_sk#70] Right output [1]: [c_customer_sk#72] Arguments: [ss_customer_sk#67], [c_customer_sk#72], Inner, BuildRight -(110) CometProject +(102) CometProject Input [5]: [ss_customer_sk#67, ss_quantity#68, ss_sales_price#69, ss_sold_date_sk#70, c_customer_sk#72] Arguments: [ss_quantity#68, ss_sales_price#69, ss_sold_date_sk#70, c_customer_sk#72], [ss_quantity#68, ss_sales_price#69, ss_sold_date_sk#70, c_customer_sk#72] -(111) Scan parquet spark_catalog.default.date_dim +(103) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(112) CometFilter +(104) CometFilter Input [2]: [d_date_sk#73, d_year#74] Condition : (d_year#74 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#73)) -(113) CometProject +(105) CometProject Input [2]: [d_date_sk#73, d_year#74] Arguments: [d_date_sk#73], [d_date_sk#73] -(114) CometBroadcastExchange +(106) CometBroadcastExchange Input [1]: [d_date_sk#73] Arguments: [d_date_sk#73] -(115) CometBroadcastHashJoin +(107) CometBroadcastHashJoin Left output [4]: [ss_quantity#68, ss_sales_price#69, ss_sold_date_sk#70, c_customer_sk#72] Right output [1]: [d_date_sk#73] Arguments: [ss_sold_date_sk#70], [d_date_sk#73], Inner, BuildRight -(116) CometProject +(108) CometProject Input [5]: [ss_quantity#68, ss_sales_price#69, ss_sold_date_sk#70, c_customer_sk#72, d_date_sk#73] Arguments: [ss_quantity#68, ss_sales_price#69, c_customer_sk#72], [ss_quantity#68, ss_sales_price#69, c_customer_sk#72] -(117) ColumnarToRow [codegen id : 1] +(109) ColumnarToRow [codegen id : 1] Input [3]: [ss_quantity#68, ss_sales_price#69, c_customer_sk#72] -(118) HashAggregate [codegen id : 1] +(110) HashAggregate [codegen id : 1] Input [3]: [ss_quantity#68, ss_sales_price#69, c_customer_sk#72] Keys [1]: [c_customer_sk#72] Functions [1]: [partial_sum((cast(ss_quantity#68 as decimal(10,0)) * ss_sales_price#69))] Aggregate Attributes [2]: [sum#75, isEmpty#76] Results [3]: [c_customer_sk#72, sum#77, isEmpty#78] -(119) RowToColumnar -Input [3]: [c_customer_sk#72, sum#77, isEmpty#78] - -(120) CometColumnarExchange -Input [3]: [c_customer_sk#72, sum#77, isEmpty#78] -Arguments: hashpartitioning(c_customer_sk#72, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] - -(121) ColumnarToRow [codegen id : 2] +(111) Exchange Input [3]: [c_customer_sk#72, sum#77, isEmpty#78] +Arguments: hashpartitioning(c_customer_sk#72, 5), ENSURE_REQUIREMENTS, [plan_id=11] -(122) HashAggregate [codegen id : 2] +(112) HashAggregate [codegen id : 2] Input [3]: [c_customer_sk#72, sum#77, isEmpty#78] Keys [1]: [c_customer_sk#72] Functions [1]: [sum((cast(ss_quantity#68 as decimal(10,0)) * ss_sales_price#69))] Aggregate Attributes [1]: [sum((cast(ss_quantity#68 as decimal(10,0)) * ss_sales_price#69))#79] Results [1]: [sum((cast(ss_quantity#68 as decimal(10,0)) * ss_sales_price#69))#79 AS csales#80] -(123) HashAggregate [codegen id : 2] +(113) HashAggregate [codegen id : 2] Input [1]: [csales#80] Keys: [] Functions [1]: [partial_max(csales#80)] Aggregate Attributes [1]: [max#81] Results [1]: [max#82] -(124) RowToColumnar -Input [1]: [max#82] - -(125) CometColumnarExchange -Input [1]: [max#82] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] - -(126) ColumnarToRow [codegen id : 3] +(114) Exchange Input [1]: [max#82] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] -(127) HashAggregate [codegen id : 3] +(115) HashAggregate [codegen id : 3] Input [1]: [max#82] Keys: [] Functions [1]: [max(csales#80)] Aggregate Attributes [1]: [max(csales#80)#83] Results [1]: [max(csales#80)#83 AS tpcds_cmax#84] -Subquery:4 Hosting operator id = 106 Hosting Expression = ss_sold_date_sk#70 IN dynamicpruning#71 -BroadcastExchange (132) -+- * ColumnarToRow (131) - +- CometProject (130) - +- CometFilter (129) - +- CometScan parquet spark_catalog.default.date_dim (128) +Subquery:4 Hosting operator id = 98 Hosting Expression = ss_sold_date_sk#70 IN dynamicpruning#71 +BroadcastExchange (120) ++- * ColumnarToRow (119) + +- CometProject (118) + +- CometFilter (117) + +- CometScan parquet spark_catalog.default.date_dim (116) -(128) Scan parquet spark_catalog.default.date_dim +(116) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#73, d_year#74] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [2000,2001,2002,2003]), IsNotNull(d_date_sk)] ReadSchema: struct -(129) CometFilter +(117) CometFilter Input [2]: [d_date_sk#73, d_year#74] Condition : (d_year#74 IN (2000,2001,2002,2003) AND isnotnull(d_date_sk#73)) -(130) CometProject +(118) CometProject Input [2]: [d_date_sk#73, d_year#74] Arguments: [d_date_sk#73], [d_date_sk#73] -(131) ColumnarToRow [codegen id : 1] +(119) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#73] -(132) BroadcastExchange +(120) BroadcastExchange Input [1]: [d_date_sk#73] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] -Subquery:5 Hosting operator id = 53 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] +Subquery:5 Hosting operator id = 50 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] -Subquery:6 Hosting operator id = 68 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 +Subquery:6 Hosting operator id = 63 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 -Subquery:7 Hosting operator id = 79 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] +Subquery:7 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#30, [id=#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt index 53beb8bb6..1184e5085 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q23b/simplified.txt @@ -2,181 +2,169 @@ TakeOrderedAndProject [c_last_name,c_first_name,sales] Union WholeStageCodegen (10) HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name] #1 - RowToColumnar - WholeStageCodegen (9) - HashAggregate [c_last_name,c_first_name,cs_quantity,cs_list_price] [sum,isEmpty,sum,isEmpty] - Project [cs_quantity,cs_list_price,c_first_name,c_last_name] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] - SortMergeJoin [cs_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometColumnarExchange [cs_bill_customer_sk] #2 - CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [item_sk] #4 - CometProject [item_sk] - CometFilter [item_sk,cnt] - CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometColumnarExchange [_groupingexpression,i_item_sk,d_date] #5 - CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] - CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] - CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] - CometProject [ss_item_sk,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [d_date_sk,d_date] #7 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] - CometBroadcastExchange [i_item_sk,i_item_desc] #8 - CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - InputAdapter - WholeStageCodegen (3) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - Subquery #3 - WholeStageCodegen (3) - HashAggregate [max] [max(csales),tpcds_cmax,max] - ColumnarToRow - InputAdapter - CometColumnarExchange #11 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [csales] [max,max] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_sk] #12 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name] #1 + WholeStageCodegen (9) + HashAggregate [c_last_name,c_first_name,cs_quantity,cs_list_price] [sum,isEmpty,sum,isEmpty] + Project [cs_quantity,cs_list_price,c_first_name,c_last_name] + BroadcastHashJoin [cs_sold_date_sk,d_date_sk] + Project [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name] + BroadcastHashJoin [cs_bill_customer_sk,c_customer_sk] + SortMergeJoin [cs_bill_customer_sk,c_customer_sk] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometExchange [cs_bill_customer_sk] #2 + CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [item_sk] #4 + CometProject [item_sk] + CometFilter [item_sk,cnt] + CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] + CometExchange [_groupingexpression,i_item_sk,d_date] #5 + CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] + CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] + CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] + CometProject [ss_item_sk,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #6 + WholeStageCodegen (1) ColumnarToRow InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] - CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #13 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [c_customer_sk] #10 - CometBroadcastExchange [d_date_sk] #14 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_sk] #9 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,c_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] - CometProject [ss_customer_sk,ss_quantity,ss_sales_price] - CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [c_customer_sk] #10 - CometFilter [c_customer_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (7) - SortMergeJoin [c_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (4) - ColumnarToRow + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [d_date_sk,d_date] #7 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_year] + CometBroadcastExchange [i_item_sk,i_item_desc] #8 + CometFilter [i_item_sk,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + InputAdapter + WholeStageCodegen (3) + Sort [c_customer_sk] + Project [c_customer_sk] + Filter [ssales] + Subquery #3 + WholeStageCodegen (3) + HashAggregate [max] [max(csales),tpcds_cmax,max] InputAdapter - CometSort [c_customer_sk,c_first_name,c_last_name] - CometColumnarExchange [c_customer_sk] #16 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - WholeStageCodegen (6) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #3 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [c_customer_sk,sum,isEmpty] #9 + Exchange #11 + WholeStageCodegen (2) + HashAggregate [csales] [max,max] + HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),csales,sum,isEmpty] + InputAdapter + Exchange [c_customer_sk] #12 + WholeStageCodegen (1) + HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] + CometProject [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #4 + BroadcastExchange #13 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [c_customer_sk] #10 + CometBroadcastExchange [d_date_sk] #14 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] + InputAdapter + Exchange [c_customer_sk] #9 + WholeStageCodegen (2) + HashAggregate [c_customer_sk,ss_quantity,ss_sales_price] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_sales_price,c_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] + CometProject [ss_customer_sk,ss_quantity,ss_sales_price] + CometFilter [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [c_customer_sk] #10 + CometFilter [c_customer_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk] InputAdapter - ReusedExchange [d_date_sk] #3 - WholeStageCodegen (20) - HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name] #17 - RowToColumnar - WholeStageCodegen (19) - HashAggregate [c_last_name,c_first_name,ws_quantity,ws_list_price] [sum,isEmpty,sum,isEmpty] - Project [ws_quantity,ws_list_price,c_first_name,c_last_name] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] - BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] - SortMergeJoin [ws_bill_customer_sk,c_customer_sk] - InputAdapter - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometColumnarExchange [ws_bill_customer_sk] #18 - CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [item_sk] #4 - InputAdapter - WholeStageCodegen (13) - Sort [c_customer_sk] - Project [c_customer_sk] - Filter [ssales] - ReusedSubquery [tpcds_cmax] #3 - HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] - ColumnarToRow + BroadcastExchange #15 + WholeStageCodegen (7) + SortMergeJoin [c_customer_sk,c_customer_sk] + InputAdapter + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [c_customer_sk,c_first_name,c_last_name] + CometExchange [c_customer_sk] #16 + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + InputAdapter + WholeStageCodegen (6) + Sort [c_customer_sk] + Project [c_customer_sk] + Filter [ssales] + ReusedSubquery [tpcds_cmax] #3 + HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] InputAdapter ReusedExchange [c_customer_sk,sum,isEmpty] #9 - InputAdapter - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #15 + InputAdapter + ReusedExchange [d_date_sk] #3 + WholeStageCodegen (20) + HashAggregate [c_last_name,c_first_name,sum,isEmpty] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),sales,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name] #17 + WholeStageCodegen (19) + HashAggregate [c_last_name,c_first_name,ws_quantity,ws_list_price] [sum,isEmpty,sum,isEmpty] + Project [ws_quantity,ws_list_price,c_first_name,c_last_name] + BroadcastHashJoin [ws_sold_date_sk,d_date_sk] + Project [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name] + BroadcastHashJoin [ws_bill_customer_sk,c_customer_sk] + SortMergeJoin [ws_bill_customer_sk,c_customer_sk] + InputAdapter + WholeStageCodegen (11) + ColumnarToRow + InputAdapter + CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometExchange [ws_bill_customer_sk] #18 + CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [item_sk] #4 + InputAdapter + WholeStageCodegen (13) + Sort [c_customer_sk] + Project [c_customer_sk] + Filter [ssales] + ReusedSubquery [tpcds_cmax] #3 + HashAggregate [c_customer_sk,sum,isEmpty] [sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price)),ssales,sum,isEmpty] + InputAdapter + ReusedExchange [c_customer_sk,sum,isEmpty] #9 InputAdapter - ReusedExchange [d_date_sk] #3 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #15 + InputAdapter + ReusedExchange [d_date_sk] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt index 7872cef74..3094c4e07 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/explain.txt @@ -1,50 +1,46 @@ == Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometColumnarExchange (38) - +- RowToColumnar (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) +* Filter (42) ++- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (21) + : : +- CometFilter (20) + : : +- CometScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.customer (24) + +- BroadcastExchange (33) + +- * ColumnarToRow (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.customer_address (30) (1) Scan parquet spark_catalog.default.store_sales @@ -62,9 +58,9 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(4) CometColumnarExchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -85,9 +81,9 @@ Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -216,216 +212,188 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#28] Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -(37) RowToColumnar +(37) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(38) CometColumnarExchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(39) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(40) HashAggregate [codegen id : 3] +(38) HashAggregate [codegen id : 3] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [partial_sum(netpaid#31)] Aggregate Attributes [2]: [sum#32, isEmpty#33] Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -(42) RowToColumnar -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(43) CometColumnarExchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [sum(netpaid#31)] Aggregate Attributes [1]: [sum(netpaid#31)#36] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (77) -+- * ColumnarToRow (76) - +- CometColumnarExchange (75) - +- RowToColumnar (74) - +- * HashAggregate (73) - +- * HashAggregate (72) - +- * ColumnarToRow (71) - +- CometColumnarExchange (70) - +- RowToColumnar (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * ColumnarToRow (64) - : +- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometProject (52) - : : : : +- CometSortMergeJoin (51) - : : : : :- CometSort (48) - : : : : : +- ReusedExchange (47) - : : : : +- CometSort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- CometBroadcastExchange (58) - : : +- CometFilter (57) - : : +- CometScan parquet spark_catalog.default.item (56) - : +- ReusedExchange (61) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 4] +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] +* HashAggregate (69) ++- Exchange (68) + +- * HashAggregate (67) + +- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * BroadcastHashJoin Inner BuildRight (62) + :- * ColumnarToRow (60) + : +- CometProject (59) + : +- CometBroadcastHashJoin (58) + : :- CometProject (56) + : : +- CometBroadcastHashJoin (55) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometProject (48) + : : : : +- CometSortMergeJoin (47) + : : : : :- CometSort (44) + : : : : : +- ReusedExchange (43) + : : : : +- CometSort (46) + : : : : +- ReusedExchange (45) + : : : +- ReusedExchange (49) + : : +- CometBroadcastExchange (54) + : : +- CometFilter (53) + : : +- CometScan parquet spark_catalog.default.item (52) + : +- ReusedExchange (57) + +- ReusedExchange (61) + + +(43) ReusedExchange [Reuses operator id: 4] Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -(48) CometSort +(44) CometSort Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] -(49) ReusedExchange [Reuses operator id: 9] +(45) ReusedExchange [Reuses operator id: 9] Output [2]: [sr_item_sk#45, sr_ticket_number#46] -(50) CometSort +(46) CometSort Input [2]: [sr_item_sk#45, sr_ticket_number#46] Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] -(51) CometSortMergeJoin +(47) CometSortMergeJoin Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] Right output [2]: [sr_item_sk#45, sr_ticket_number#46] Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner -(52) CometProject +(48) CometProject Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -(53) ReusedExchange [Reuses operator id: 16] +(49) ReusedExchange [Reuses operator id: 16] Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight -(55) CometProject +(51) CometProject Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -(56) Scan parquet spark_catalog.default.item +(52) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) CometFilter +(53) CometFilter Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Condition : isnotnull(i_item_sk#51) -(58) CometBroadcastExchange +(54) CometBroadcastExchange Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -(59) CometBroadcastHashJoin +(55) CometBroadcastHashJoin Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight -(60) CometProject +(56) CometProject Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -(61) ReusedExchange [Reuses operator id: 26] +(57) ReusedExchange [Reuses operator id: 26] Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -(62) CometBroadcastHashJoin +(58) CometBroadcastHashJoin Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight -(63) CometProject +(59) CometProject Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] -(64) ColumnarToRow [codegen id : 2] +(60) ColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] -(65) ReusedExchange [Reuses operator id: 33] +(61) ReusedExchange [Reuses operator id: 33] Output [3]: [ca_state#61, ca_zip#62, ca_country#63] -(66) BroadcastHashJoin [codegen id : 2] +(62) BroadcastHashJoin [codegen id : 2] Left keys [2]: [c_birth_country#60, s_zip#50] Right keys [2]: [upper(ca_country#63), ca_zip#62] Join type: Inner Join condition: None -(67) Project [codegen id : 2] +(63) Project [codegen id : 2] Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] -(68) HashAggregate [codegen id : 2] +(64) HashAggregate [codegen id : 2] Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] Aggregate Attributes [1]: [sum#64] Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -(69) RowToColumnar +(65) Exchange Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] +Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(70) CometColumnarExchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(71) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(72) HashAggregate [codegen id : 3] +(66) HashAggregate [codegen id : 3] Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] -(73) HashAggregate [codegen id : 3] +(67) HashAggregate [codegen id : 3] Input [1]: [netpaid#66] Keys: [] Functions [1]: [partial_avg(netpaid#66)] Aggregate Attributes [2]: [sum#67, count#68] Results [2]: [sum#69, count#70] -(74) RowToColumnar -Input [2]: [sum#69, count#70] - -(75) CometColumnarExchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(76) ColumnarToRow [codegen id : 4] +(68) Exchange Input [2]: [sum#69, count#70] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(77) HashAggregate [codegen id : 4] +(69) HashAggregate [codegen id : 4] Input [2]: [sum#69, count#70] Keys: [] Functions [1]: [avg(netpaid#66)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt index 0b53b6fc9..bd14d8250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24a/simplified.txt @@ -3,92 +3,84 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #9 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 - RowToColumnar + InputAdapter + Exchange #9 WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [netpaid] [sum,count,sum,count] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (1) + ColumnarToRow InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt index b5f3faefb..846c7a14a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/explain.txt @@ -1,50 +1,46 @@ == Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometColumnarExchange (38) - +- RowToColumnar (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) +* Filter (42) ++- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (21) + : : +- CometFilter (20) + : : +- CometScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.customer (24) + +- BroadcastExchange (33) + +- * ColumnarToRow (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.customer_address (30) (1) Scan parquet spark_catalog.default.store_sales @@ -62,9 +58,9 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(4) CometColumnarExchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -85,9 +81,9 @@ Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -216,216 +212,188 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#28] Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -(37) RowToColumnar +(37) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(38) CometColumnarExchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(39) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(40) HashAggregate [codegen id : 3] +(38) HashAggregate [codegen id : 3] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [partial_sum(netpaid#31)] Aggregate Attributes [2]: [sum#32, isEmpty#33] Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -(42) RowToColumnar -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(43) CometColumnarExchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [sum(netpaid#31)] Aggregate Attributes [1]: [sum(netpaid#31)#36] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (77) -+- * ColumnarToRow (76) - +- CometColumnarExchange (75) - +- RowToColumnar (74) - +- * HashAggregate (73) - +- * HashAggregate (72) - +- * ColumnarToRow (71) - +- CometColumnarExchange (70) - +- RowToColumnar (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * ColumnarToRow (64) - : +- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometProject (52) - : : : : +- CometSortMergeJoin (51) - : : : : :- CometSort (48) - : : : : : +- ReusedExchange (47) - : : : : +- CometSort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- CometBroadcastExchange (58) - : : +- CometFilter (57) - : : +- CometScan parquet spark_catalog.default.item (56) - : +- ReusedExchange (61) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 4] +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] +* HashAggregate (69) ++- Exchange (68) + +- * HashAggregate (67) + +- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * BroadcastHashJoin Inner BuildRight (62) + :- * ColumnarToRow (60) + : +- CometProject (59) + : +- CometBroadcastHashJoin (58) + : :- CometProject (56) + : : +- CometBroadcastHashJoin (55) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometProject (48) + : : : : +- CometSortMergeJoin (47) + : : : : :- CometSort (44) + : : : : : +- ReusedExchange (43) + : : : : +- CometSort (46) + : : : : +- ReusedExchange (45) + : : : +- ReusedExchange (49) + : : +- CometBroadcastExchange (54) + : : +- CometFilter (53) + : : +- CometScan parquet spark_catalog.default.item (52) + : +- ReusedExchange (57) + +- ReusedExchange (61) + + +(43) ReusedExchange [Reuses operator id: 4] Output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] -(48) CometSort +(44) CometSort Input [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44], [ss_ticket_number#43 ASC NULLS FIRST, ss_item_sk#40 ASC NULLS FIRST] -(49) ReusedExchange [Reuses operator id: 9] +(45) ReusedExchange [Reuses operator id: 9] Output [2]: [sr_item_sk#45, sr_ticket_number#46] -(50) CometSort +(46) CometSort Input [2]: [sr_item_sk#45, sr_ticket_number#46] Arguments: [sr_item_sk#45, sr_ticket_number#46], [sr_ticket_number#46 ASC NULLS FIRST, sr_item_sk#45 ASC NULLS FIRST] -(51) CometSortMergeJoin +(47) CometSortMergeJoin Left output [5]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44] Right output [2]: [sr_item_sk#45, sr_ticket_number#46] Arguments: [ss_ticket_number#43, ss_item_sk#40], [sr_ticket_number#46, sr_item_sk#45], Inner -(52) CometProject +(48) CometProject Input [7]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_ticket_number#43, ss_net_paid#44, sr_item_sk#45, sr_ticket_number#46] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44], [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] -(53) ReusedExchange [Reuses operator id: 16] +(49) ReusedExchange [Reuses operator id: 16] Output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [4]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44] Right output [4]: [s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] Arguments: [ss_store_sk#42], [s_store_sk#47], Inner, BuildRight -(55) CometProject +(51) CometProject Input [8]: [ss_item_sk#40, ss_customer_sk#41, ss_store_sk#42, ss_net_paid#44, s_store_sk#47, s_store_name#48, s_state#49, s_zip#50] Arguments: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50], [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] -(56) Scan parquet spark_catalog.default.item +(52) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) CometFilter +(53) CometFilter Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Condition : isnotnull(i_item_sk#51) -(58) CometBroadcastExchange +(54) CometBroadcastExchange Input [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -(59) CometBroadcastHashJoin +(55) CometBroadcastHashJoin Left output [6]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50] Right output [6]: [i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [ss_item_sk#40], [i_item_sk#51], Inner, BuildRight -(60) CometProject +(56) CometProject Input [12]: [ss_item_sk#40, ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_item_sk#51, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Arguments: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56], [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] -(61) ReusedExchange [Reuses operator id: 26] +(57) ReusedExchange [Reuses operator id: 26] Output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] -(62) CometBroadcastHashJoin +(58) CometBroadcastHashJoin Left output [10]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56] Right output [4]: [c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] Arguments: [ss_customer_sk#41], [c_customer_sk#57], Inner, BuildRight -(63) CometProject +(59) CometProject Input [14]: [ss_customer_sk#41, ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_customer_sk#57, c_first_name#58, c_last_name#59, c_birth_country#60] Arguments: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60], [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] -(64) ColumnarToRow [codegen id : 2] +(60) ColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60] -(65) ReusedExchange [Reuses operator id: 33] +(61) ReusedExchange [Reuses operator id: 33] Output [3]: [ca_state#61, ca_zip#62, ca_country#63] -(66) BroadcastHashJoin [codegen id : 2] +(62) BroadcastHashJoin [codegen id : 2] Left keys [2]: [c_birth_country#60, s_zip#50] Right keys [2]: [upper(ca_country#63), ca_zip#62] Join type: Inner Join condition: None -(67) Project [codegen id : 2] +(63) Project [codegen id : 2] Output [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] Input [15]: [ss_net_paid#44, s_store_name#48, s_state#49, s_zip#50, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, c_birth_country#60, ca_state#61, ca_zip#62, ca_country#63] -(68) HashAggregate [codegen id : 2] +(64) HashAggregate [codegen id : 2] Input [11]: [ss_net_paid#44, s_store_name#48, s_state#49, i_current_price#52, i_size#53, i_color#54, i_units#55, i_manager_id#56, c_first_name#58, c_last_name#59, ca_state#61] Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#44))] Aggregate Attributes [1]: [sum#64] Results [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -(69) RowToColumnar +(65) Exchange Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] +Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(70) CometColumnarExchange -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] -Arguments: hashpartitioning(c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(71) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] - -(72) HashAggregate [codegen id : 3] +(66) HashAggregate [codegen id : 3] Input [11]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53, sum#65] Keys [10]: [c_last_name#59, c_first_name#58, s_store_name#48, ca_state#61, s_state#49, i_color#54, i_current_price#52, i_manager_id#56, i_units#55, i_size#53] Functions [1]: [sum(UnscaledValue(ss_net_paid#44))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#44))#30] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#44))#30,17,2) AS netpaid#66] -(73) HashAggregate [codegen id : 3] +(67) HashAggregate [codegen id : 3] Input [1]: [netpaid#66] Keys: [] Functions [1]: [partial_avg(netpaid#66)] Aggregate Attributes [2]: [sum#67, count#68] Results [2]: [sum#69, count#70] -(74) RowToColumnar -Input [2]: [sum#69, count#70] - -(75) CometColumnarExchange -Input [2]: [sum#69, count#70] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(76) ColumnarToRow [codegen id : 4] +(68) Exchange Input [2]: [sum#69, count#70] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(77) HashAggregate [codegen id : 4] +(69) HashAggregate [codegen id : 4] Input [2]: [sum#69, count#70] Keys: [] Functions [1]: [avg(netpaid#66)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt index 0b53b6fc9..bd14d8250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q24b/simplified.txt @@ -3,92 +3,84 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #9 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 - RowToColumnar + InputAdapter + Exchange #9 WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [netpaid] [sum,count,sum,count] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (1) + ColumnarToRow InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt index b0f287458..0ea8e3ef4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/explain.txt @@ -1,48 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * HashAggregate (43) - +- * ColumnarToRow (42) - +- CometColumnarExchange (41) - +- RowToColumnar (40) - +- * HashAggregate (39) - +- * ColumnarToRow (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (32) - : +- CometBroadcastHashJoin (31) - : :- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store_returns (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (8) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometProject (15) - : : : : +- CometFilter (14) - : : : : +- CometScan parquet spark_catalog.default.date_dim (13) - : : : +- CometBroadcastExchange (22) - : : : +- CometProject (21) - : : : +- CometFilter (20) - : : : +- CometScan parquet spark_catalog.default.date_dim (19) - : : +- ReusedExchange (25) - : +- CometBroadcastExchange (30) - : +- CometFilter (29) - : +- CometScan parquet spark_catalog.default.store (28) - +- CometBroadcastExchange (35) - +- CometFilter (34) - +- CometScan parquet spark_catalog.default.item (33) +TakeOrderedAndProject (42) ++- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * ColumnarToRow (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (32) + : +- CometBroadcastHashJoin (31) + : :- CometProject (27) + : : +- CometBroadcastHashJoin (26) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometScan parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometScan parquet spark_catalog.default.date_dim (19) + : : +- ReusedExchange (25) + : +- CometBroadcastExchange (30) + : +- CometFilter (29) + : +- CometScan parquet spark_catalog.default.store (28) + +- CometBroadcastExchange (35) + +- CometFilter (34) + +- CometScan parquet spark_catalog.default.item (33) (1) Scan parquet spark_catalog.default.store_sales @@ -233,86 +231,80 @@ Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(Unscale Aggregate Attributes [3]: [sum#32, sum#33, sum#34] Results [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, sum#35, sum#36, sum#37] -(40) RowToColumnar +(40) Exchange Input [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, sum#35, sum#36, sum#37] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(41) CometColumnarExchange -Input [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, sum#35, sum#36, sum#37] -Arguments: hashpartitioning(i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(42) ColumnarToRow [codegen id : 2] -Input [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, sum#35, sum#36, sum#37] - -(43) HashAggregate [codegen id : 2] +(41) HashAggregate [codegen id : 2] Input [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, sum#35, sum#36, sum#37] Keys [4]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28] Functions [3]: [sum(UnscaledValue(ss_net_profit#5)), sum(UnscaledValue(sr_net_loss#11)), sum(UnscaledValue(cs_net_profit#16))] Aggregate Attributes [3]: [sum(UnscaledValue(ss_net_profit#5))#38, sum(UnscaledValue(sr_net_loss#11))#39, sum(UnscaledValue(cs_net_profit#16))#40] Results [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, MakeDecimal(sum(UnscaledValue(ss_net_profit#5))#38,17,2) AS store_sales_profit#41, MakeDecimal(sum(UnscaledValue(sr_net_loss#11))#39,17,2) AS store_returns_loss#42, MakeDecimal(sum(UnscaledValue(cs_net_profit#16))#40,17,2) AS catalog_sales_profit#43] -(44) TakeOrderedAndProject +(42) TakeOrderedAndProject Input [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, store_sales_profit#41, store_returns_loss#42, catalog_sales_profit#43] Arguments: 100, [i_item_id#30 ASC NULLS FIRST, i_item_desc#31 ASC NULLS FIRST, s_store_id#27 ASC NULLS FIRST, s_store_name#28 ASC NULLS FIRST], [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, store_sales_profit#41, store_returns_loss#42, catalog_sales_profit#43] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (49) -+- * ColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan parquet spark_catalog.default.date_dim (45) +BroadcastExchange (47) ++- * ColumnarToRow (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan parquet spark_catalog.default.date_dim (43) -(45) Scan parquet spark_catalog.default.date_dim +(43) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,4), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(46) CometFilter +(44) CometFilter Input [3]: [d_date_sk#19, d_year#20, d_moy#21] Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 4)) AND (d_year#20 = 2001)) AND isnotnull(d_date_sk#19)) -(47) CometProject +(45) CometProject Input [3]: [d_date_sk#19, d_year#20, d_moy#21] Arguments: [d_date_sk#19], [d_date_sk#19] -(48) ColumnarToRow [codegen id : 1] +(46) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#19] -(49) BroadcastExchange +(47) BroadcastExchange Input [1]: [d_date_sk#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (54) -+- * ColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.date_dim (50) +BroadcastExchange (52) ++- * ColumnarToRow (51) + +- CometProject (50) + +- CometFilter (49) + +- CometScan parquet spark_catalog.default.date_dim (48) -(50) Scan parquet spark_catalog.default.date_dim +(48) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#22, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,10), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(51) CometFilter +(49) CometFilter Input [3]: [d_date_sk#22, d_year#23, d_moy#24] Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 4)) AND (d_moy#24 <= 10)) AND (d_year#23 = 2001)) AND isnotnull(d_date_sk#22)) -(52) CometProject +(50) CometProject Input [3]: [d_date_sk#22, d_year#23, d_moy#24] Arguments: [d_date_sk#22], [d_date_sk#22] -(53) ColumnarToRow [codegen id : 1] +(51) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#22] -(54) BroadcastExchange +(52) BroadcastExchange Input [1]: [d_date_sk#22] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt index e435f53e7..6335e50ff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q25/simplified.txt @@ -1,65 +1,63 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] WholeStageCodegen (2) HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit)),store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_net_profit,sr_net_loss,cs_net_profit] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_sk,s_store_id,s_store_name] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #3 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #5 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #7 + InputAdapter + Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_net_profit,sr_net_loss,cs_net_profit] [sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_sk,s_store_id,s_store_name] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_net_loss,sr_returned_date_sk,cs_net_profit,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_net_loss,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] #3 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_net_loss,sr_returned_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] #5 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + CometBroadcastExchange [d_date_sk] #6 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #8 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 - CometFilter [i_item_sk,i_item_id,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + ReusedExchange [d_date_sk] #7 + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #8 + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #9 + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt index b98a5ce41..d0bc41bcf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/explain.txt @@ -1,36 +1,34 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- * ColumnarToRow (30) - +- CometColumnarExchange (29) - +- RowToColumnar (28) - +- * HashAggregate (27) - +- * ColumnarToRow (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.date_dim (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.item (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometScan parquet spark_catalog.default.promotion (20) +TakeOrderedAndProject (30) ++- * HashAggregate (29) + +- Exchange (28) + +- * HashAggregate (27) + +- * ColumnarToRow (26) + +- CometProject (25) + +- CometBroadcastHashJoin (24) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometScan parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (23) + +- CometProject (22) + +- CometFilter (21) + +- CometScan parquet spark_catalog.default.promotion (20) (1) Scan parquet spark_catalog.default.catalog_sales @@ -163,56 +161,50 @@ Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_pr Aggregate Attributes [8]: [sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] Results [9]: [i_item_id#17, sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] -(28) RowToColumnar +(28) Exchange Input [9]: [i_item_id#17, sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(29) CometColumnarExchange -Input [9]: [i_item_id#17, sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] -Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(30) ColumnarToRow [codegen id : 2] -Input [9]: [i_item_id#17, sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] - -(31) HashAggregate [codegen id : 2] +(29) HashAggregate [codegen id : 2] Input [9]: [i_item_id#17, sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] Keys [1]: [i_item_id#17] Functions [4]: [avg(cs_quantity#4), avg(UnscaledValue(cs_list_price#5)), avg(UnscaledValue(cs_coupon_amt#7)), avg(UnscaledValue(cs_sales_price#6))] Aggregate Attributes [4]: [avg(cs_quantity#4)#37, avg(UnscaledValue(cs_list_price#5))#38, avg(UnscaledValue(cs_coupon_amt#7))#39, avg(UnscaledValue(cs_sales_price#6))#40] Results [5]: [i_item_id#17, avg(cs_quantity#4)#37 AS agg1#41, cast((avg(UnscaledValue(cs_list_price#5))#38 / 100.0) as decimal(11,6)) AS agg2#42, cast((avg(UnscaledValue(cs_coupon_amt#7))#39 / 100.0) as decimal(11,6)) AS agg3#43, cast((avg(UnscaledValue(cs_sales_price#6))#40 / 100.0) as decimal(11,6)) AS agg4#44] -(32) TakeOrderedAndProject +(30) TakeOrderedAndProject Input [5]: [i_item_id#17, agg1#41, agg2#42, agg3#43, agg4#44] Arguments: 100, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#41, agg2#42, agg3#43, agg4#44] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (37) -+- * ColumnarToRow (36) - +- CometProject (35) - +- CometFilter (34) - +- CometScan parquet spark_catalog.default.date_dim (33) +BroadcastExchange (35) ++- * ColumnarToRow (34) + +- CometProject (33) + +- CometFilter (32) + +- CometScan parquet spark_catalog.default.date_dim (31) -(33) Scan parquet spark_catalog.default.date_dim +(31) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_year#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(34) CometFilter +(32) CometFilter Input [2]: [d_date_sk#14, d_year#15] Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) -(35) CometProject +(33) CometProject Input [2]: [d_date_sk#14, d_year#15] Arguments: [d_date_sk#14], [d_date_sk#14] -(36) ColumnarToRow [codegen id : 1] +(34) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(37) BroadcastExchange +(35) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt index 21f7e6b68..b98e61704 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q26/simplified.txt @@ -1,44 +1,42 @@ TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] WholeStageCodegen (2) HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] - CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,d_date_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] - CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #6 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] + InputAdapter + Exchange [i_item_id] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] + CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] + CometProject [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,d_date_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometBroadcastHashJoin [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk,cd_demo_sk] + CometFilter [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_cdemo_sk,cs_item_sk,cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk] #3 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [p_promo_sk] #6 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_email,p_channel_event] + CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt index 9a41c57e1..16af0e3a4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/explain.txt @@ -1,36 +1,34 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- * ColumnarToRow (30) - +- CometColumnarExchange (29) - +- RowToColumnar (28) - +- * HashAggregate (27) - +- * ColumnarToRow (26) - +- CometExpand (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.date_dim (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.store (15) - +- CometBroadcastExchange (22) - +- CometFilter (21) - +- CometScan parquet spark_catalog.default.item (20) +TakeOrderedAndProject (30) ++- * HashAggregate (29) + +- Exchange (28) + +- * HashAggregate (27) + +- * ColumnarToRow (26) + +- CometExpand (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometScan parquet spark_catalog.default.store (15) + +- CometBroadcastExchange (22) + +- CometFilter (21) + +- CometScan parquet spark_catalog.default.item (20) (1) Scan parquet spark_catalog.default.store_sales @@ -163,56 +161,50 @@ Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_pr Aggregate Attributes [8]: [sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] Results [11]: [i_item_id#20, s_state#21, spark_grouping_id#22, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] -(28) RowToColumnar +(28) Exchange Input [11]: [i_item_id#20, s_state#21, spark_grouping_id#22, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] +Arguments: hashpartitioning(i_item_id#20, s_state#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(29) CometColumnarExchange -Input [11]: [i_item_id#20, s_state#21, spark_grouping_id#22, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] -Arguments: hashpartitioning(i_item_id#20, s_state#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(30) ColumnarToRow [codegen id : 2] -Input [11]: [i_item_id#20, s_state#21, spark_grouping_id#22, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] - -(31) HashAggregate [codegen id : 2] +(29) HashAggregate [codegen id : 2] Input [11]: [i_item_id#20, s_state#21, spark_grouping_id#22, sum#31, count#32, sum#33, count#34, sum#35, count#36, sum#37, count#38] Keys [3]: [i_item_id#20, s_state#21, spark_grouping_id#22] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] Aggregate Attributes [4]: [avg(ss_quantity#4)#39, avg(UnscaledValue(ss_list_price#5))#40, avg(UnscaledValue(ss_coupon_amt#7))#41, avg(UnscaledValue(ss_sales_price#6))#42] Results [7]: [i_item_id#20, s_state#21, cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) AS g_state#43, avg(ss_quantity#4)#39 AS agg1#44, cast((avg(UnscaledValue(ss_list_price#5))#40 / 100.0) as decimal(11,6)) AS agg2#45, cast((avg(UnscaledValue(ss_coupon_amt#7))#41 / 100.0) as decimal(11,6)) AS agg3#46, cast((avg(UnscaledValue(ss_sales_price#6))#42 / 100.0) as decimal(11,6)) AS agg4#47] -(32) TakeOrderedAndProject +(30) TakeOrderedAndProject Input [7]: [i_item_id#20, s_state#21, g_state#43, agg1#44, agg2#45, agg3#46, agg4#47] Arguments: 100, [i_item_id#20 ASC NULLS FIRST, s_state#21 ASC NULLS FIRST], [i_item_id#20, s_state#21, g_state#43, agg1#44, agg2#45, agg3#46, agg4#47] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (37) -+- * ColumnarToRow (36) - +- CometProject (35) - +- CometFilter (34) - +- CometScan parquet spark_catalog.default.date_dim (33) +BroadcastExchange (35) ++- * ColumnarToRow (34) + +- CometProject (33) + +- CometFilter (32) + +- CometScan parquet spark_catalog.default.date_dim (31) -(33) Scan parquet spark_catalog.default.date_dim +(31) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_year#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(34) CometFilter +(32) CometFilter Input [2]: [d_date_sk#14, d_year#15] Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2002)) AND isnotnull(d_date_sk#14)) -(35) CometProject +(33) CometProject Input [2]: [d_date_sk#14, d_year#15] Arguments: [d_date_sk#14], [d_date_sk#14] -(36) ColumnarToRow [codegen id : 1] +(34) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(37) BroadcastExchange +(35) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt index 894d4215e..4d4c824bc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q27/simplified.txt @@ -1,44 +1,42 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] WholeStageCodegen (2) HashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,s_state,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,s_state,spark_grouping_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] - CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #5 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + Exchange [i_item_id,s_state,spark_grouping_id] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,s_state,spark_grouping_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] + CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk] #3 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_state] #5 + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/explain.txt index 615bb79b9..de4ab3a2c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/explain.txt @@ -1,98 +1,74 @@ == Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (94) -:- * BroadcastNestedLoopJoin Inner BuildRight (78) -: :- * BroadcastNestedLoopJoin Inner BuildRight (62) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (46) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (30) -: : : : :- * HashAggregate (14) -: : : : : +- * ColumnarToRow (13) -: : : : : +- CometColumnarExchange (12) -: : : : : +- RowToColumnar (11) -: : : : : +- * HashAggregate (10) -: : : : : +- * HashAggregate (9) -: : : : : +- * ColumnarToRow (8) -: : : : : +- CometColumnarExchange (7) -: : : : : +- RowToColumnar (6) -: : : : : +- * HashAggregate (5) -: : : : : +- * ColumnarToRow (4) -: : : : : +- CometProject (3) -: : : : : +- CometFilter (2) -: : : : : +- CometScan parquet spark_catalog.default.store_sales (1) -: : : : +- BroadcastExchange (29) -: : : : +- * HashAggregate (28) -: : : : +- * ColumnarToRow (27) -: : : : +- CometColumnarExchange (26) -: : : : +- RowToColumnar (25) -: : : : +- * HashAggregate (24) -: : : : +- * HashAggregate (23) -: : : : +- * ColumnarToRow (22) -: : : : +- CometColumnarExchange (21) -: : : : +- RowToColumnar (20) -: : : : +- * HashAggregate (19) -: : : : +- * ColumnarToRow (18) -: : : : +- CometProject (17) -: : : : +- CometFilter (16) -: : : : +- CometScan parquet spark_catalog.default.store_sales (15) -: : : +- BroadcastExchange (45) -: : : +- * HashAggregate (44) -: : : +- * ColumnarToRow (43) -: : : +- CometColumnarExchange (42) -: : : +- RowToColumnar (41) -: : : +- * HashAggregate (40) -: : : +- * HashAggregate (39) -: : : +- * ColumnarToRow (38) -: : : +- CometColumnarExchange (37) -: : : +- RowToColumnar (36) -: : : +- * HashAggregate (35) -: : : +- * ColumnarToRow (34) -: : : +- CometProject (33) -: : : +- CometFilter (32) -: : : +- CometScan parquet spark_catalog.default.store_sales (31) -: : +- BroadcastExchange (61) -: : +- * HashAggregate (60) -: : +- * ColumnarToRow (59) -: : +- CometColumnarExchange (58) -: : +- RowToColumnar (57) -: : +- * HashAggregate (56) -: : +- * HashAggregate (55) -: : +- * ColumnarToRow (54) -: : +- CometColumnarExchange (53) -: : +- RowToColumnar (52) -: : +- * HashAggregate (51) -: : +- * ColumnarToRow (50) -: : +- CometProject (49) -: : +- CometFilter (48) -: : +- CometScan parquet spark_catalog.default.store_sales (47) -: +- BroadcastExchange (77) -: +- * HashAggregate (76) -: +- * ColumnarToRow (75) -: +- CometColumnarExchange (74) -: +- RowToColumnar (73) -: +- * HashAggregate (72) -: +- * HashAggregate (71) -: +- * ColumnarToRow (70) -: +- CometColumnarExchange (69) -: +- RowToColumnar (68) -: +- * HashAggregate (67) -: +- * ColumnarToRow (66) -: +- CometProject (65) -: +- CometFilter (64) -: +- CometScan parquet spark_catalog.default.store_sales (63) -+- BroadcastExchange (93) - +- * HashAggregate (92) - +- * ColumnarToRow (91) - +- CometColumnarExchange (90) - +- RowToColumnar (89) - +- * HashAggregate (88) - +- * HashAggregate (87) - +- * ColumnarToRow (86) - +- CometColumnarExchange (85) - +- RowToColumnar (84) - +- * HashAggregate (83) - +- * ColumnarToRow (82) - +- CometProject (81) - +- CometFilter (80) - +- CometScan parquet spark_catalog.default.store_sales (79) +* BroadcastNestedLoopJoin Inner BuildRight (70) +:- * BroadcastNestedLoopJoin Inner BuildRight (58) +: :- * BroadcastNestedLoopJoin Inner BuildRight (46) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) +: : : : :- * HashAggregate (10) +: : : : : +- Exchange (9) +: : : : : +- * HashAggregate (8) +: : : : : +- * HashAggregate (7) +: : : : : +- Exchange (6) +: : : : : +- * HashAggregate (5) +: : : : : +- * ColumnarToRow (4) +: : : : : +- CometProject (3) +: : : : : +- CometFilter (2) +: : : : : +- CometScan parquet spark_catalog.default.store_sales (1) +: : : : +- BroadcastExchange (21) +: : : : +- * HashAggregate (20) +: : : : +- Exchange (19) +: : : : +- * HashAggregate (18) +: : : : +- * HashAggregate (17) +: : : : +- Exchange (16) +: : : : +- * HashAggregate (15) +: : : : +- * ColumnarToRow (14) +: : : : +- CometProject (13) +: : : : +- CometFilter (12) +: : : : +- CometScan parquet spark_catalog.default.store_sales (11) +: : : +- BroadcastExchange (33) +: : : +- * HashAggregate (32) +: : : +- Exchange (31) +: : : +- * HashAggregate (30) +: : : +- * HashAggregate (29) +: : : +- Exchange (28) +: : : +- * HashAggregate (27) +: : : +- * ColumnarToRow (26) +: : : +- CometProject (25) +: : : +- CometFilter (24) +: : : +- CometScan parquet spark_catalog.default.store_sales (23) +: : +- BroadcastExchange (45) +: : +- * HashAggregate (44) +: : +- Exchange (43) +: : +- * HashAggregate (42) +: : +- * HashAggregate (41) +: : +- Exchange (40) +: : +- * HashAggregate (39) +: : +- * ColumnarToRow (38) +: : +- CometProject (37) +: : +- CometFilter (36) +: : +- CometScan parquet spark_catalog.default.store_sales (35) +: +- BroadcastExchange (57) +: +- * HashAggregate (56) +: +- Exchange (55) +: +- * HashAggregate (54) +: +- * HashAggregate (53) +: +- Exchange (52) +: +- * HashAggregate (51) +: +- * ColumnarToRow (50) +: +- CometProject (49) +: +- CometFilter (48) +: +- CometScan parquet spark_catalog.default.store_sales (47) ++- BroadcastExchange (69) + +- * HashAggregate (68) + +- Exchange (67) + +- * HashAggregate (66) + +- * HashAggregate (65) + +- Exchange (64) + +- * HashAggregate (63) + +- * ColumnarToRow (62) + +- CometProject (61) + +- CometFilter (60) + +- CometScan parquet spark_catalog.default.store_sales (59) (1) Scan parquet spark_catalog.default.store_sales @@ -120,414 +96,342 @@ Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_li Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] Results [4]: [ss_list_price#3, sum#8, count#9, count#10] -(6) RowToColumnar +(6) Exchange Input [4]: [ss_list_price#3, sum#8, count#9, count#10] +Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(7) CometColumnarExchange -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] -Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(8) ColumnarToRow [codegen id : 2] -Input [4]: [ss_list_price#3, sum#8, count#9, count#10] - -(9) HashAggregate [codegen id : 2] +(7) HashAggregate [codegen id : 2] Input [4]: [ss_list_price#3, sum#8, count#9, count#10] Keys [1]: [ss_list_price#3] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7] Results [4]: [ss_list_price#3, sum#8, count#9, count#10] -(10) HashAggregate [codegen id : 2] +(8) HashAggregate [codegen id : 2] Input [4]: [ss_list_price#3, sum#8, count#9, count#10] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_price#3), partial_count(distinct ss_list_price#3)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] Results [4]: [sum#8, count#9, count#10, count#12] -(11) RowToColumnar -Input [4]: [sum#8, count#9, count#10, count#12] - -(12) CometColumnarExchange -Input [4]: [sum#8, count#9, count#10, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(13) ColumnarToRow [codegen id : 18] +(9) Exchange Input [4]: [sum#8, count#9, count#10, count#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(14) HashAggregate [codegen id : 18] +(10) HashAggregate [codegen id : 18] Input [4]: [sum#8, count#9, count#10, count#12] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#6, count(ss_list_price#3)#7, count(ss_list_price#3)#11] Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#6 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#7 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] -(15) Scan parquet spark_catalog.default.store_sales +(11) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] ReadSchema: struct -(16) CometFilter +(12) CometFilter Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) -(17) CometProject +(13) CometProject Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Arguments: [ss_list_price#18], [ss_list_price#18] -(18) ColumnarToRow [codegen id : 3] +(14) ColumnarToRow [codegen id : 3] Input [1]: [ss_list_price#18] -(19) HashAggregate [codegen id : 3] +(15) HashAggregate [codegen id : 3] Input [1]: [ss_list_price#18] Keys [1]: [ss_list_price#18] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] Results [4]: [ss_list_price#18, sum#23, count#24, count#25] -(20) RowToColumnar +(16) Exchange Input [4]: [ss_list_price#18, sum#23, count#24, count#25] +Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(21) CometColumnarExchange -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] -Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(22) ColumnarToRow [codegen id : 4] -Input [4]: [ss_list_price#18, sum#23, count#24, count#25] - -(23) HashAggregate [codegen id : 4] +(17) HashAggregate [codegen id : 4] Input [4]: [ss_list_price#18, sum#23, count#24, count#25] Keys [1]: [ss_list_price#18] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22] Results [4]: [ss_list_price#18, sum#23, count#24, count#25] -(24) HashAggregate [codegen id : 4] +(18) HashAggregate [codegen id : 4] Input [4]: [ss_list_price#18, sum#23, count#24, count#25] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] Results [4]: [sum#23, count#24, count#25, count#27] -(25) RowToColumnar +(19) Exchange Input [4]: [sum#23, count#24, count#25, count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] -(26) CometColumnarExchange -Input [4]: [sum#23, count#24, count#25, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(27) ColumnarToRow [codegen id : 5] -Input [4]: [sum#23, count#24, count#25, count#27] - -(28) HashAggregate [codegen id : 5] +(20) HashAggregate [codegen id : 5] Input [4]: [sum#23, count#24, count#25, count#27] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#21, count(ss_list_price#18)#22, count(ss_list_price#18)#26] Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#21 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#22 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] -(29) BroadcastExchange +(21) BroadcastExchange Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] Arguments: IdentityBroadcastMode, [plan_id=5] -(30) BroadcastNestedLoopJoin [codegen id : 18] +(22) BroadcastNestedLoopJoin [codegen id : 18] Join type: Inner Join condition: None -(31) Scan parquet spark_catalog.default.store_sales +(23) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] ReadSchema: struct -(32) CometFilter +(24) CometFilter Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) -(33) CometProject +(25) CometProject Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Arguments: [ss_list_price#33], [ss_list_price#33] -(34) ColumnarToRow [codegen id : 6] +(26) ColumnarToRow [codegen id : 6] Input [1]: [ss_list_price#33] -(35) HashAggregate [codegen id : 6] +(27) HashAggregate [codegen id : 6] Input [1]: [ss_list_price#33] Keys [1]: [ss_list_price#33] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] Results [4]: [ss_list_price#33, sum#38, count#39, count#40] -(36) RowToColumnar -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] - -(37) CometColumnarExchange +(28) Exchange Input [4]: [ss_list_price#33, sum#38, count#39, count#40] -Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(38) ColumnarToRow [codegen id : 7] -Input [4]: [ss_list_price#33, sum#38, count#39, count#40] - -(39) HashAggregate [codegen id : 7] +(29) HashAggregate [codegen id : 7] Input [4]: [ss_list_price#33, sum#38, count#39, count#40] Keys [1]: [ss_list_price#33] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37] Results [4]: [ss_list_price#33, sum#38, count#39, count#40] -(40) HashAggregate [codegen id : 7] +(30) HashAggregate [codegen id : 7] Input [4]: [ss_list_price#33, sum#38, count#39, count#40] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] Results [4]: [sum#38, count#39, count#40, count#42] -(41) RowToColumnar -Input [4]: [sum#38, count#39, count#40, count#42] - -(42) CometColumnarExchange +(31) Exchange Input [4]: [sum#38, count#39, count#40, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(43) ColumnarToRow [codegen id : 8] -Input [4]: [sum#38, count#39, count#40, count#42] - -(44) HashAggregate [codegen id : 8] +(32) HashAggregate [codegen id : 8] Input [4]: [sum#38, count#39, count#40, count#42] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#36, count(ss_list_price#33)#37, count(ss_list_price#33)#41] Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#36 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#37 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] -(45) BroadcastExchange +(33) BroadcastExchange Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] Arguments: IdentityBroadcastMode, [plan_id=8] -(46) BroadcastNestedLoopJoin [codegen id : 18] +(34) BroadcastNestedLoopJoin [codegen id : 18] Join type: Inner Join condition: None -(47) Scan parquet spark_catalog.default.store_sales +(35) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] ReadSchema: struct -(48) CometFilter +(36) CometFilter Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) -(49) CometProject +(37) CometProject Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Arguments: [ss_list_price#48], [ss_list_price#48] -(50) ColumnarToRow [codegen id : 9] +(38) ColumnarToRow [codegen id : 9] Input [1]: [ss_list_price#48] -(51) HashAggregate [codegen id : 9] +(39) HashAggregate [codegen id : 9] Input [1]: [ss_list_price#48] Keys [1]: [ss_list_price#48] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] Results [4]: [ss_list_price#48, sum#53, count#54, count#55] -(52) RowToColumnar +(40) Exchange Input [4]: [ss_list_price#48, sum#53, count#54, count#55] +Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(53) CometColumnarExchange -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] -Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(54) ColumnarToRow [codegen id : 10] -Input [4]: [ss_list_price#48, sum#53, count#54, count#55] - -(55) HashAggregate [codegen id : 10] +(41) HashAggregate [codegen id : 10] Input [4]: [ss_list_price#48, sum#53, count#54, count#55] Keys [1]: [ss_list_price#48] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52] Results [4]: [ss_list_price#48, sum#53, count#54, count#55] -(56) HashAggregate [codegen id : 10] +(42) HashAggregate [codegen id : 10] Input [4]: [ss_list_price#48, sum#53, count#54, count#55] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] Results [4]: [sum#53, count#54, count#55, count#57] -(57) RowToColumnar -Input [4]: [sum#53, count#54, count#55, count#57] - -(58) CometColumnarExchange -Input [4]: [sum#53, count#54, count#55, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] - -(59) ColumnarToRow [codegen id : 11] +(43) Exchange Input [4]: [sum#53, count#54, count#55, count#57] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] -(60) HashAggregate [codegen id : 11] +(44) HashAggregate [codegen id : 11] Input [4]: [sum#53, count#54, count#55, count#57] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#51, count(ss_list_price#48)#52, count(ss_list_price#48)#56] Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#51 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#52 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] -(61) BroadcastExchange +(45) BroadcastExchange Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] Arguments: IdentityBroadcastMode, [plan_id=11] -(62) BroadcastNestedLoopJoin [codegen id : 18] +(46) BroadcastNestedLoopJoin [codegen id : 18] Join type: Inner Join condition: None -(63) Scan parquet spark_catalog.default.store_sales +(47) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] ReadSchema: struct -(64) CometFilter +(48) CometFilter Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) -(65) CometProject +(49) CometProject Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Arguments: [ss_list_price#63], [ss_list_price#63] -(66) ColumnarToRow [codegen id : 12] +(50) ColumnarToRow [codegen id : 12] Input [1]: [ss_list_price#63] -(67) HashAggregate [codegen id : 12] +(51) HashAggregate [codegen id : 12] Input [1]: [ss_list_price#63] Keys [1]: [ss_list_price#63] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] Results [4]: [ss_list_price#63, sum#68, count#69, count#70] -(68) RowToColumnar -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] - -(69) CometColumnarExchange +(52) Exchange Input [4]: [ss_list_price#63, sum#68, count#69, count#70] -Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, [plan_id=12] -(70) ColumnarToRow [codegen id : 13] -Input [4]: [ss_list_price#63, sum#68, count#69, count#70] - -(71) HashAggregate [codegen id : 13] +(53) HashAggregate [codegen id : 13] Input [4]: [ss_list_price#63, sum#68, count#69, count#70] Keys [1]: [ss_list_price#63] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67] Results [4]: [ss_list_price#63, sum#68, count#69, count#70] -(72) HashAggregate [codegen id : 13] +(54) HashAggregate [codegen id : 13] Input [4]: [ss_list_price#63, sum#68, count#69, count#70] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] Results [4]: [sum#68, count#69, count#70, count#72] -(73) RowToColumnar -Input [4]: [sum#68, count#69, count#70, count#72] - -(74) CometColumnarExchange +(55) Exchange Input [4]: [sum#68, count#69, count#70, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] -(75) ColumnarToRow [codegen id : 14] -Input [4]: [sum#68, count#69, count#70, count#72] - -(76) HashAggregate [codegen id : 14] +(56) HashAggregate [codegen id : 14] Input [4]: [sum#68, count#69, count#70, count#72] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#66, count(ss_list_price#63)#67, count(ss_list_price#63)#71] Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#66 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#67 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] -(77) BroadcastExchange +(57) BroadcastExchange Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] Arguments: IdentityBroadcastMode, [plan_id=14] -(78) BroadcastNestedLoopJoin [codegen id : 18] +(58) BroadcastNestedLoopJoin [codegen id : 18] Join type: Inner Join condition: None -(79) Scan parquet spark_catalog.default.store_sales +(59) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] ReadSchema: struct -(80) CometFilter +(60) CometFilter Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) -(81) CometProject +(61) CometProject Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Arguments: [ss_list_price#78], [ss_list_price#78] -(82) ColumnarToRow [codegen id : 15] +(62) ColumnarToRow [codegen id : 15] Input [1]: [ss_list_price#78] -(83) HashAggregate [codegen id : 15] +(63) HashAggregate [codegen id : 15] Input [1]: [ss_list_price#78] Keys [1]: [ss_list_price#78] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] Results [4]: [ss_list_price#78, sum#83, count#84, count#85] -(84) RowToColumnar -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] - -(85) CometColumnarExchange -Input [4]: [ss_list_price#78, sum#83, count#84, count#85] -Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] - -(86) ColumnarToRow [codegen id : 16] +(64) Exchange Input [4]: [ss_list_price#78, sum#83, count#84, count#85] +Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, [plan_id=15] -(87) HashAggregate [codegen id : 16] +(65) HashAggregate [codegen id : 16] Input [4]: [ss_list_price#78, sum#83, count#84, count#85] Keys [1]: [ss_list_price#78] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82] Results [4]: [ss_list_price#78, sum#83, count#84, count#85] -(88) HashAggregate [codegen id : 16] +(66) HashAggregate [codegen id : 16] Input [4]: [ss_list_price#78, sum#83, count#84, count#85] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] Results [4]: [sum#83, count#84, count#85, count#87] -(89) RowToColumnar -Input [4]: [sum#83, count#84, count#85, count#87] - -(90) CometColumnarExchange -Input [4]: [sum#83, count#84, count#85, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] - -(91) ColumnarToRow [codegen id : 17] +(67) Exchange Input [4]: [sum#83, count#84, count#85, count#87] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] -(92) HashAggregate [codegen id : 17] +(68) HashAggregate [codegen id : 17] Input [4]: [sum#83, count#84, count#85, count#87] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#81, count(ss_list_price#78)#82, count(ss_list_price#78)#86] Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#81 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#82 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] -(93) BroadcastExchange +(69) BroadcastExchange Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] Arguments: IdentityBroadcastMode, [plan_id=17] -(94) BroadcastNestedLoopJoin [codegen id : 18] +(70) BroadcastNestedLoopJoin [codegen id : 18] Join type: Inner Join condition: None diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/simplified.txt index 7c4823efd..2136886fe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q28/simplified.txt @@ -5,131 +5,107 @@ WholeStageCodegen (18) BroadcastNestedLoopJoin BroadcastNestedLoopJoin HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + Exchange #1 + WholeStageCodegen (2) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + InputAdapter + Exchange [ss_list_price] #2 + WholeStageCodegen (1) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (5) HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #4 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #5 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + Exchange #4 + WholeStageCodegen (4) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + InputAdapter + Exchange [ss_list_price] #5 + WholeStageCodegen (3) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #6 WholeStageCodegen (8) HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #7 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #8 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + Exchange #7 + WholeStageCodegen (7) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + InputAdapter + Exchange [ss_list_price] #8 + WholeStageCodegen (6) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #9 WholeStageCodegen (11) HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #10 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #11 - RowToColumnar - WholeStageCodegen (9) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + Exchange #10 + WholeStageCodegen (10) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + InputAdapter + Exchange [ss_list_price] #11 + WholeStageCodegen (9) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #12 WholeStageCodegen (14) HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #13 - RowToColumnar - WholeStageCodegen (13) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #14 - RowToColumnar - WholeStageCodegen (12) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + Exchange #13 + WholeStageCodegen (13) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + InputAdapter + Exchange [ss_list_price] #14 + WholeStageCodegen (12) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #15 WholeStageCodegen (17) HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #16 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #17 - RowToColumnar - WholeStageCodegen (15) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + Exchange #16 + WholeStageCodegen (16) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + InputAdapter + Exchange [ss_list_price] #17 + WholeStageCodegen (15) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt index f6ab979fb..1bf9ac34d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/explain.txt @@ -1,51 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- * ColumnarToRow (45) - +- CometColumnarExchange (44) - +- RowToColumnar (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store_returns (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (8) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometProject (15) - : : : : +- CometFilter (14) - : : : : +- CometScan parquet spark_catalog.default.date_dim (13) - : : : +- CometBroadcastExchange (22) - : : : +- CometProject (21) - : : : +- CometFilter (20) - : : : +- CometScan parquet spark_catalog.default.date_dim (19) - : : +- CometBroadcastExchange (28) - : : +- CometProject (27) - : : +- CometFilter (26) - : : +- CometScan parquet spark_catalog.default.date_dim (25) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometScan parquet spark_catalog.default.store (31) - +- CometBroadcastExchange (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.item (36) +TakeOrderedAndProject (45) ++- * HashAggregate (44) + +- Exchange (43) + +- * HashAggregate (42) + +- * ColumnarToRow (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan parquet spark_catalog.default.store_returns (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (8) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometProject (15) + : : : : +- CometFilter (14) + : : : : +- CometScan parquet spark_catalog.default.date_dim (13) + : : : +- CometBroadcastExchange (22) + : : : +- CometProject (21) + : : : +- CometFilter (20) + : : : +- CometScan parquet spark_catalog.default.date_dim (19) + : : +- CometBroadcastExchange (28) + : : +- CometProject (27) + : : +- CometFilter (26) + : : +- CometScan parquet spark_catalog.default.date_dim (25) + : +- CometBroadcastExchange (33) + : +- CometFilter (32) + : +- CometScan parquet spark_catalog.default.store (31) + +- CometBroadcastExchange (38) + +- CometFilter (37) + +- CometScan parquet spark_catalog.default.item (36) (1) Scan parquet spark_catalog.default.store_sales @@ -252,116 +250,110 @@ Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#11), Aggregate Attributes [3]: [sum#33, sum#34, sum#35] Results [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, sum#36, sum#37, sum#38] -(43) RowToColumnar +(43) Exchange Input [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, sum#36, sum#37, sum#38] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(44) CometColumnarExchange -Input [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, sum#36, sum#37, sum#38] -Arguments: hashpartitioning(i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(45) ColumnarToRow [codegen id : 2] -Input [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, sum#36, sum#37, sum#38] - -(46) HashAggregate [codegen id : 2] +(44) HashAggregate [codegen id : 2] Input [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, sum#36, sum#37, sum#38] Keys [4]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29] Functions [3]: [sum(ss_quantity#5), sum(sr_return_quantity#11), sum(cs_quantity#16)] Aggregate Attributes [3]: [sum(ss_quantity#5)#39, sum(sr_return_quantity#11)#40, sum(cs_quantity#16)#41] Results [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, sum(ss_quantity#5)#39 AS store_sales_quantity#42, sum(sr_return_quantity#11)#40 AS store_returns_quantity#43, sum(cs_quantity#16)#41 AS catalog_sales_quantity#44] -(47) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, store_sales_quantity#42, store_returns_quantity#43, catalog_sales_quantity#44] Arguments: 100, [i_item_id#31 ASC NULLS FIRST, i_item_desc#32 ASC NULLS FIRST, s_store_id#28 ASC NULLS FIRST, s_store_name#29 ASC NULLS FIRST], [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, store_sales_quantity#42, store_returns_quantity#43, catalog_sales_quantity#44] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (52) -+- * ColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan parquet spark_catalog.default.date_dim (48) +BroadcastExchange (50) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(48) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#19, d_year#20, d_moy#21] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,9), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(49) CometFilter +(47) CometFilter Input [3]: [d_date_sk#19, d_year#20, d_moy#21] Condition : ((((isnotnull(d_moy#21) AND isnotnull(d_year#20)) AND (d_moy#21 = 9)) AND (d_year#20 = 1999)) AND isnotnull(d_date_sk#19)) -(50) CometProject +(48) CometProject Input [3]: [d_date_sk#19, d_year#20, d_moy#21] Arguments: [d_date_sk#19], [d_date_sk#19] -(51) ColumnarToRow [codegen id : 1] +(49) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#19] -(52) BroadcastExchange +(50) BroadcastExchange Input [1]: [d_date_sk#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] Subquery:2 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (57) -+- * ColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan parquet spark_catalog.default.date_dim (53) +BroadcastExchange (55) ++- * ColumnarToRow (54) + +- CometProject (53) + +- CometFilter (52) + +- CometScan parquet spark_catalog.default.date_dim (51) -(53) Scan parquet spark_catalog.default.date_dim +(51) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#22, d_year#23, d_moy#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), GreaterThanOrEqual(d_moy,9), LessThanOrEqual(d_moy,12), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(54) CometFilter +(52) CometFilter Input [3]: [d_date_sk#22, d_year#23, d_moy#24] Condition : (((((isnotnull(d_moy#24) AND isnotnull(d_year#23)) AND (d_moy#24 >= 9)) AND (d_moy#24 <= 12)) AND (d_year#23 = 1999)) AND isnotnull(d_date_sk#22)) -(55) CometProject +(53) CometProject Input [3]: [d_date_sk#22, d_year#23, d_moy#24] Arguments: [d_date_sk#22], [d_date_sk#22] -(56) ColumnarToRow [codegen id : 1] +(54) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#22] -(57) BroadcastExchange +(55) BroadcastExchange Input [1]: [d_date_sk#22] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] Subquery:3 Hosting operator id = 8 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#18 -BroadcastExchange (62) -+- * ColumnarToRow (61) - +- CometProject (60) - +- CometFilter (59) - +- CometScan parquet spark_catalog.default.date_dim (58) +BroadcastExchange (60) ++- * ColumnarToRow (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.date_dim (56) -(58) Scan parquet spark_catalog.default.date_dim +(56) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(59) CometFilter +(57) CometFilter Input [2]: [d_date_sk#25, d_year#26] Condition : (d_year#26 IN (1999,2000,2001) AND isnotnull(d_date_sk#25)) -(60) CometProject +(58) CometProject Input [2]: [d_date_sk#25, d_year#26] Arguments: [d_date_sk#25], [d_date_sk#25] -(61) ColumnarToRow [codegen id : 1] +(59) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#25] -(62) BroadcastExchange +(60) BroadcastExchange Input [1]: [d_date_sk#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt index e2c8ade5f..ee1f435d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q29/simplified.txt @@ -1,75 +1,73 @@ TakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] WholeStageCodegen (2) HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum] [sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity),store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_quantity,sr_return_quantity,cs_quantity] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] - CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_store_id,s_store_name] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 + InputAdapter + Exchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,ss_quantity,sr_return_quantity,cs_quantity] [sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] + CometProject [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_sk,s_store_id,s_store_name] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_return_quantity,sr_returned_date_sk,cs_quantity,cs_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk,cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_return_quantity,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_quantity,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] #3 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_return_quantity,sr_returned_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #4 WholeStageCodegen (1) ColumnarToRow InputAdapter CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #7 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #8 + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] #5 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #3 + BroadcastExchange #6 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk] #7 CometProject [d_date_sk] CometFilter [d_date_sk,d_year,d_moy] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #9 + CometBroadcastExchange [d_date_sk] #8 CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #10 - CometFilter [s_store_sk,s_store_id,s_store_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #11 - CometFilter [i_item_sk,i_item_id,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #9 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #10 + CometFilter [s_store_sk,s_store_id,s_store_name] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc] #11 + CometFilter [i_item_sk,i_item_id,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt index 8e804be1f..75546d3e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/explain.txt @@ -1,25 +1,23 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometColumnarExchange (18) - +- RowToColumnar (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.date_dim (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometScan parquet spark_catalog.default.store_sales (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometScan parquet spark_catalog.default.item (9) +TakeOrderedAndProject (19) ++- * HashAggregate (18) + +- Exchange (17) + +- * HashAggregate (16) + +- * ColumnarToRow (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan parquet spark_catalog.default.item (9) (1) Scan parquet spark_catalog.default.date_dim @@ -100,24 +98,18 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum#11] Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -(17) RowToColumnar +(17) Exchange Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(18) CometColumnarExchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(19) ColumnarToRow [codegen id : 2] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] - -(20) HashAggregate [codegen id : 2] +(18) HashAggregate [codegen id : 2] Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum_agg#16] -(21) TakeOrderedAndProject +(19) TakeOrderedAndProject Input [4]: [d_year#2, brand_id#14, brand#15, sum_agg#16] Arguments: 100, [d_year#2 ASC NULLS FIRST, sum_agg#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, sum_agg#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/simplified.txt index ffb864d3f..36bb17a16 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q3/simplified.txt @@ -1,25 +1,23 @@ TakeOrderedAndProject [d_year,sum_agg,brand_id,brand] WholeStageCodegen (2) HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,sum_agg,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,i_brand,i_brand_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] + InputAdapter + Exchange [d_year,i_brand,i_brand_id] #1 + WholeStageCodegen (1) + HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manufact_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt index 83890381c..004e04634 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/explain.txt @@ -1,61 +1,55 @@ == Physical Plan == -TakeOrderedAndProject (57) -+- * Project (56) - +- * BroadcastHashJoin Inner BuildRight (55) - :- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (43) - : : +- * BroadcastHashJoin Inner BuildRight (42) - : : :- * Filter (20) - : : : +- * HashAggregate (19) - : : : +- * ColumnarToRow (18) - : : : +- CometColumnarExchange (17) - : : : +- RowToColumnar (16) - : : : +- * HashAggregate (15) - : : : +- * ColumnarToRow (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_returns (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.customer_address (9) - : : +- BroadcastExchange (41) - : : +- * Filter (40) - : : +- * HashAggregate (39) - : : +- * ColumnarToRow (38) - : : +- CometColumnarExchange (37) - : : +- RowToColumnar (36) - : : +- * HashAggregate (35) - : : +- * HashAggregate (34) - : : +- * ColumnarToRow (33) - : : +- CometColumnarExchange (32) - : : +- RowToColumnar (31) - : : +- * HashAggregate (30) - : : +- * ColumnarToRow (29) - : : +- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) - : : : :- CometFilter (22) - : : : : +- CometScan parquet spark_catalog.default.web_returns (21) - : : : +- ReusedExchange (23) - : : +- ReusedExchange (26) - : +- BroadcastExchange (47) - : +- * ColumnarToRow (46) - : +- CometFilter (45) - : +- CometScan parquet spark_catalog.default.customer (44) - +- BroadcastExchange (54) - +- * ColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.customer_address (50) +TakeOrderedAndProject (51) ++- * Project (50) + +- * BroadcastHashJoin Inner BuildRight (49) + :- * Project (43) + : +- * BroadcastHashJoin Inner BuildRight (42) + : :- * Project (37) + : : +- * BroadcastHashJoin Inner BuildRight (36) + : : :- * Filter (18) + : : : +- * HashAggregate (17) + : : : +- Exchange (16) + : : : +- * HashAggregate (15) + : : : +- * ColumnarToRow (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.web_returns (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan parquet spark_catalog.default.customer_address (9) + : : +- BroadcastExchange (35) + : : +- * Filter (34) + : : +- * HashAggregate (33) + : : +- Exchange (32) + : : +- * HashAggregate (31) + : : +- * HashAggregate (30) + : : +- Exchange (29) + : : +- * HashAggregate (28) + : : +- * ColumnarToRow (27) + : : +- CometProject (26) + : : +- CometBroadcastHashJoin (25) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometFilter (20) + : : : : +- CometScan parquet spark_catalog.default.web_returns (19) + : : : +- ReusedExchange (21) + : : +- ReusedExchange (24) + : +- BroadcastExchange (41) + : +- * ColumnarToRow (40) + : +- CometFilter (39) + : +- CometScan parquet spark_catalog.default.customer (38) + +- BroadcastExchange (48) + +- * ColumnarToRow (47) + +- CometProject (46) + +- CometFilter (45) + +- CometScan parquet spark_catalog.default.customer_address (44) (1) Scan parquet spark_catalog.default.web_returns @@ -132,28 +126,22 @@ Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] Aggregate Attributes [1]: [sum#10] Results [3]: [wr_returning_customer_sk#1, ca_state#9, sum#11] -(16) RowToColumnar +(16) Exchange Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#11] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#11] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 7] -Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#11] - -(19) HashAggregate [codegen id : 7] +(17) HashAggregate [codegen id : 7] Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#11] Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(wr_return_amt#3))] Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#3))#12] Results [3]: [wr_returning_customer_sk#1 AS ctr_customer_sk#13, ca_state#9 AS ctr_state#14, MakeDecimal(sum(UnscaledValue(wr_return_amt#3))#12,17,2) AS ctr_total_return#15] -(20) Filter [codegen id : 7] +(18) Filter [codegen id : 7] Input [3]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15] Condition : isnotnull(ctr_total_return#15) -(21) Scan parquet spark_catalog.default.web_returns +(19) Scan parquet spark_catalog.default.web_returns Output [4]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18, wr_returned_date_sk#19] Batched: true Location: InMemoryFileIndex [] @@ -161,199 +149,187 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#19), dynamicpruningexpression(w PushedFilters: [IsNotNull(wr_returning_addr_sk)] ReadSchema: struct -(22) CometFilter +(20) CometFilter Input [4]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18, wr_returned_date_sk#19] Condition : isnotnull(wr_returning_addr_sk#17) -(23) ReusedExchange [Reuses operator id: 6] +(21) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#21] -(24) CometBroadcastHashJoin +(22) CometBroadcastHashJoin Left output [4]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18, wr_returned_date_sk#19] Right output [1]: [d_date_sk#21] Arguments: [wr_returned_date_sk#19], [d_date_sk#21], Inner, BuildRight -(25) CometProject +(23) CometProject Input [5]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18, wr_returned_date_sk#19, d_date_sk#21] Arguments: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18], [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18] -(26) ReusedExchange [Reuses operator id: 11] +(24) ReusedExchange [Reuses operator id: 11] Output [2]: [ca_address_sk#22, ca_state#23] -(27) CometBroadcastHashJoin +(25) CometBroadcastHashJoin Left output [3]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18] Right output [2]: [ca_address_sk#22, ca_state#23] Arguments: [wr_returning_addr_sk#17], [ca_address_sk#22], Inner, BuildRight -(28) CometProject +(26) CometProject Input [5]: [wr_returning_customer_sk#16, wr_returning_addr_sk#17, wr_return_amt#18, ca_address_sk#22, ca_state#23] Arguments: [wr_returning_customer_sk#16, wr_return_amt#18, ca_state#23], [wr_returning_customer_sk#16, wr_return_amt#18, ca_state#23] -(29) ColumnarToRow [codegen id : 2] +(27) ColumnarToRow [codegen id : 2] Input [3]: [wr_returning_customer_sk#16, wr_return_amt#18, ca_state#23] -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [3]: [wr_returning_customer_sk#16, wr_return_amt#18, ca_state#23] Keys [2]: [wr_returning_customer_sk#16, ca_state#23] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#18))] Aggregate Attributes [1]: [sum#24] Results [3]: [wr_returning_customer_sk#16, ca_state#23, sum#25] -(31) RowToColumnar -Input [3]: [wr_returning_customer_sk#16, ca_state#23, sum#25] - -(32) CometColumnarExchange -Input [3]: [wr_returning_customer_sk#16, ca_state#23, sum#25] -Arguments: hashpartitioning(wr_returning_customer_sk#16, ca_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(33) ColumnarToRow [codegen id : 3] +(29) Exchange Input [3]: [wr_returning_customer_sk#16, ca_state#23, sum#25] +Arguments: hashpartitioning(wr_returning_customer_sk#16, ca_state#23, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(34) HashAggregate [codegen id : 3] +(30) HashAggregate [codegen id : 3] Input [3]: [wr_returning_customer_sk#16, ca_state#23, sum#25] Keys [2]: [wr_returning_customer_sk#16, ca_state#23] Functions [1]: [sum(UnscaledValue(wr_return_amt#18))] Aggregate Attributes [1]: [sum(UnscaledValue(wr_return_amt#18))#12] Results [2]: [ca_state#23 AS ctr_state#26, MakeDecimal(sum(UnscaledValue(wr_return_amt#18))#12,17,2) AS ctr_total_return#27] -(35) HashAggregate [codegen id : 3] +(31) HashAggregate [codegen id : 3] Input [2]: [ctr_state#26, ctr_total_return#27] Keys [1]: [ctr_state#26] Functions [1]: [partial_avg(ctr_total_return#27)] Aggregate Attributes [2]: [sum#28, count#29] Results [3]: [ctr_state#26, sum#30, count#31] -(36) RowToColumnar -Input [3]: [ctr_state#26, sum#30, count#31] - -(37) CometColumnarExchange +(32) Exchange Input [3]: [ctr_state#26, sum#30, count#31] -Arguments: hashpartitioning(ctr_state#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ctr_state#26, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(38) ColumnarToRow [codegen id : 4] -Input [3]: [ctr_state#26, sum#30, count#31] - -(39) HashAggregate [codegen id : 4] +(33) HashAggregate [codegen id : 4] Input [3]: [ctr_state#26, sum#30, count#31] Keys [1]: [ctr_state#26] Functions [1]: [avg(ctr_total_return#27)] Aggregate Attributes [1]: [avg(ctr_total_return#27)#32] Results [2]: [(avg(ctr_total_return#27)#32 * 1.2) AS (avg(ctr_total_return) * 1.2)#33, ctr_state#26] -(40) Filter [codegen id : 4] +(34) Filter [codegen id : 4] Input [2]: [(avg(ctr_total_return) * 1.2)#33, ctr_state#26] Condition : isnotnull((avg(ctr_total_return) * 1.2)#33) -(41) BroadcastExchange +(35) BroadcastExchange Input [2]: [(avg(ctr_total_return) * 1.2)#33, ctr_state#26] Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] -(42) BroadcastHashJoin [codegen id : 7] +(36) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ctr_state#14] Right keys [1]: [ctr_state#26] Join type: Inner Join condition: (cast(ctr_total_return#15 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#33) -(43) Project [codegen id : 7] +(37) Project [codegen id : 7] Output [2]: [ctr_customer_sk#13, ctr_total_return#15] Input [5]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15, (avg(ctr_total_return) * 1.2)#33, ctr_state#26] -(44) Scan parquet spark_catalog.default.customer +(38) Scan parquet spark_catalog.default.customer Output [14]: [c_customer_sk#34, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_day#41, c_birth_month#42, c_birth_year#43, c_birth_country#44, c_login#45, c_email_address#46, c_last_review_date#47] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(45) CometFilter +(39) CometFilter Input [14]: [c_customer_sk#34, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_day#41, c_birth_month#42, c_birth_year#43, c_birth_country#44, c_login#45, c_email_address#46, c_last_review_date#47] Condition : (isnotnull(c_customer_sk#34) AND isnotnull(c_current_addr_sk#36)) -(46) ColumnarToRow [codegen id : 5] +(40) ColumnarToRow [codegen id : 5] Input [14]: [c_customer_sk#34, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_day#41, c_birth_month#42, c_birth_year#43, c_birth_country#44, c_login#45, c_email_address#46, c_last_review_date#47] -(47) BroadcastExchange +(41) BroadcastExchange Input [14]: [c_customer_sk#34, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_day#41, c_birth_month#42, c_birth_year#43, c_birth_country#44, c_login#45, c_email_address#46, c_last_review_date#47] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(48) BroadcastHashJoin [codegen id : 7] +(42) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ctr_customer_sk#13] Right keys [1]: [c_customer_sk#34] Join type: Inner Join condition: None -(49) Project [codegen id : 7] +(43) Project [codegen id : 7] Output [14]: [ctr_total_return#15, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_day#41, c_birth_month#42, c_birth_year#43, c_birth_country#44, c_login#45, c_email_address#46, c_last_review_date#47] Input [16]: [ctr_customer_sk#13, ctr_total_return#15, c_customer_sk#34, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_day#41, c_birth_month#42, c_birth_year#43, c_birth_country#44, c_login#45, c_email_address#46, c_last_review_date#47] -(50) Scan parquet spark_catalog.default.customer_address +(44) Scan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#48, ca_state#49] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] ReadSchema: struct -(51) CometFilter +(45) CometFilter Input [2]: [ca_address_sk#48, ca_state#49] Condition : ((isnotnull(ca_state#49) AND (ca_state#49 = GA)) AND isnotnull(ca_address_sk#48)) -(52) CometProject +(46) CometProject Input [2]: [ca_address_sk#48, ca_state#49] Arguments: [ca_address_sk#48], [ca_address_sk#48] -(53) ColumnarToRow [codegen id : 6] +(47) ColumnarToRow [codegen id : 6] Input [1]: [ca_address_sk#48] -(54) BroadcastExchange +(48) BroadcastExchange Input [1]: [ca_address_sk#48] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -(55) BroadcastHashJoin [codegen id : 7] +(49) BroadcastHashJoin [codegen id : 7] Left keys [1]: [c_current_addr_sk#36] Right keys [1]: [ca_address_sk#48] Join type: Inner Join condition: None -(56) Project [codegen id : 7] +(50) Project [codegen id : 7] Output [13]: [c_customer_id#35, c_salutation#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_day#41, c_birth_month#42, c_birth_year#43, c_birth_country#44, c_login#45, c_email_address#46, c_last_review_date#47, ctr_total_return#15] Input [15]: [ctr_total_return#15, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_day#41, c_birth_month#42, c_birth_year#43, c_birth_country#44, c_login#45, c_email_address#46, c_last_review_date#47, ca_address_sk#48] -(57) TakeOrderedAndProject +(51) TakeOrderedAndProject Input [13]: [c_customer_id#35, c_salutation#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_day#41, c_birth_month#42, c_birth_year#43, c_birth_country#44, c_login#45, c_email_address#46, c_last_review_date#47, ctr_total_return#15] Arguments: 100, [c_customer_id#35 ASC NULLS FIRST, c_salutation#37 ASC NULLS FIRST, c_first_name#38 ASC NULLS FIRST, c_last_name#39 ASC NULLS FIRST, c_preferred_cust_flag#40 ASC NULLS FIRST, c_birth_day#41 ASC NULLS FIRST, c_birth_month#42 ASC NULLS FIRST, c_birth_year#43 ASC NULLS FIRST, c_birth_country#44 ASC NULLS FIRST, c_login#45 ASC NULLS FIRST, c_email_address#46 ASC NULLS FIRST, c_last_review_date#47 ASC NULLS FIRST, ctr_total_return#15 ASC NULLS FIRST], [c_customer_id#35, c_salutation#37, c_first_name#38, c_last_name#39, c_preferred_cust_flag#40, c_birth_day#41, c_birth_month#42, c_birth_year#43, c_birth_country#44, c_login#45, c_email_address#46, c_last_review_date#47, ctr_total_return#15] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = wr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (62) -+- * ColumnarToRow (61) - +- CometProject (60) - +- CometFilter (59) - +- CometScan parquet spark_catalog.default.date_dim (58) +BroadcastExchange (56) ++- * ColumnarToRow (55) + +- CometProject (54) + +- CometFilter (53) + +- CometScan parquet spark_catalog.default.date_dim (52) -(58) Scan parquet spark_catalog.default.date_dim +(52) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(59) CometFilter +(53) CometFilter Input [2]: [d_date_sk#6, d_year#7] Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2002)) AND isnotnull(d_date_sk#6)) -(60) CometProject +(54) CometProject Input [2]: [d_date_sk#6, d_year#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(61) ColumnarToRow [codegen id : 1] +(55) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(62) BroadcastExchange +(56) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] -Subquery:2 Hosting operator id = 21 Hosting Expression = wr_returned_date_sk#19 IN dynamicpruning#5 +Subquery:2 Hosting operator id = 19 Hosting Expression = wr_returned_date_sk#19 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt index 0af6e3103..c4e59b49a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q30/simplified.txt @@ -8,64 +8,58 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,c_pre BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] Filter [ctr_total_return] HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [wr_returning_customer_sk,ca_state] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] - CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] - CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #4 - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + Exchange [wr_returning_customer_sk,ca_state] #1 + WholeStageCodegen (1) + HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] + CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] + CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) Filter [(avg(ctr_total_return) * 1.2)] HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ctr_state] #6 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [wr_returning_customer_sk,ca_state] #7 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] - CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] - CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] - CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk,ca_state] #4 + InputAdapter + Exchange [ctr_state] #6 + WholeStageCodegen (3) + HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] + HashAggregate [wr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(wr_return_amt)),ctr_state,ctr_total_return,sum] + InputAdapter + Exchange [wr_returning_customer_sk,ca_state] #7 + WholeStageCodegen (2) + HashAggregate [wr_returning_customer_sk,ca_state,wr_return_amt] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] + CometProject [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt] + CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk,d_date_sk] + CometFilter [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk,ca_state] #4 InputAdapter BroadcastExchange #8 WholeStageCodegen (5) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt index 64de6dbe0..059065f96 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/explain.txt @@ -1,113 +1,101 @@ == Physical Plan == -* ColumnarToRow (109) -+- CometSort (108) - +- CometColumnarExchange (107) - +- RowToColumnar (106) - +- * Project (105) - +- * BroadcastHashJoin Inner BuildRight (104) - :- * Project (88) - : +- * BroadcastHashJoin Inner BuildRight (87) - : :- * BroadcastHashJoin Inner BuildRight (71) - : : :- * Project (55) - : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : :- * BroadcastHashJoin Inner BuildRight (36) - : : : : :- * HashAggregate (18) - : : : : : +- * ColumnarToRow (17) - : : : : : +- CometColumnarExchange (16) - : : : : : +- RowToColumnar (15) - : : : : : +- * HashAggregate (14) - : : : : : +- * ColumnarToRow (13) - : : : : : +- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometScan parquet spark_catalog.default.customer_address (8) - : : : : +- BroadcastExchange (35) - : : : : +- * HashAggregate (34) - : : : : +- * ColumnarToRow (33) - : : : : +- CometColumnarExchange (32) - : : : : +- RowToColumnar (31) - : : : : +- * HashAggregate (30) - : : : : +- * ColumnarToRow (29) - : : : : +- CometProject (28) - : : : : +- CometBroadcastHashJoin (27) - : : : : :- CometProject (25) - : : : : : +- CometBroadcastHashJoin (24) - : : : : : :- CometFilter (20) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (19) - : : : : : +- CometBroadcastExchange (23) - : : : : : +- CometFilter (22) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (21) - : : : : +- ReusedExchange (26) - : : : +- BroadcastExchange (53) - : : : +- * HashAggregate (52) - : : : +- * ColumnarToRow (51) - : : : +- CometColumnarExchange (50) - : : : +- RowToColumnar (49) - : : : +- * HashAggregate (48) - : : : +- * ColumnarToRow (47) - : : : +- CometProject (46) - : : : +- CometBroadcastHashJoin (45) - : : : :- CometProject (43) - : : : : +- CometBroadcastHashJoin (42) - : : : : :- CometFilter (38) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (37) - : : : : +- CometBroadcastExchange (41) - : : : : +- CometFilter (40) - : : : : +- CometScan parquet spark_catalog.default.date_dim (39) - : : : +- ReusedExchange (44) - : : +- BroadcastExchange (70) - : : +- * HashAggregate (69) - : : +- * ColumnarToRow (68) - : : +- CometColumnarExchange (67) - : : +- RowToColumnar (66) - : : +- * HashAggregate (65) - : : +- * ColumnarToRow (64) - : : +- CometProject (63) - : : +- CometBroadcastHashJoin (62) - : : :- CometProject (60) - : : : +- CometBroadcastHashJoin (59) - : : : :- CometFilter (57) - : : : : +- CometScan parquet spark_catalog.default.web_sales (56) - : : : +- ReusedExchange (58) - : : +- ReusedExchange (61) - : +- BroadcastExchange (86) - : +- * HashAggregate (85) - : +- * ColumnarToRow (84) - : +- CometColumnarExchange (83) - : +- RowToColumnar (82) - : +- * HashAggregate (81) - : +- * ColumnarToRow (80) - : +- CometProject (79) - : +- CometBroadcastHashJoin (78) - : :- CometProject (76) - : : +- CometBroadcastHashJoin (75) - : : :- CometFilter (73) - : : : +- CometScan parquet spark_catalog.default.web_sales (72) - : : +- ReusedExchange (74) - : +- ReusedExchange (77) - +- BroadcastExchange (103) - +- * HashAggregate (102) - +- * ColumnarToRow (101) - +- CometColumnarExchange (100) - +- RowToColumnar (99) - +- * HashAggregate (98) - +- * ColumnarToRow (97) - +- CometProject (96) - +- CometBroadcastHashJoin (95) - :- CometProject (93) - : +- CometBroadcastHashJoin (92) - : :- CometFilter (90) - : : +- CometScan parquet spark_catalog.default.web_sales (89) - : +- ReusedExchange (91) - +- ReusedExchange (94) +* ColumnarToRow (97) ++- CometSort (96) + +- CometColumnarExchange (95) + +- RowToColumnar (94) + +- * Project (93) + +- * BroadcastHashJoin Inner BuildRight (92) + :- * Project (78) + : +- * BroadcastHashJoin Inner BuildRight (77) + : :- * BroadcastHashJoin Inner BuildRight (63) + : : :- * Project (49) + : : : +- * BroadcastHashJoin Inner BuildRight (48) + : : : :- * BroadcastHashJoin Inner BuildRight (32) + : : : : :- * HashAggregate (16) + : : : : : +- Exchange (15) + : : : : : +- * HashAggregate (14) + : : : : : +- * ColumnarToRow (13) + : : : : : +- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan parquet spark_catalog.default.customer_address (8) + : : : : +- BroadcastExchange (31) + : : : : +- * HashAggregate (30) + : : : : +- Exchange (29) + : : : : +- * HashAggregate (28) + : : : : +- * ColumnarToRow (27) + : : : : +- CometProject (26) + : : : : +- CometBroadcastHashJoin (25) + : : : : :- CometProject (23) + : : : : : +- CometBroadcastHashJoin (22) + : : : : : :- CometFilter (18) + : : : : : : +- CometScan parquet spark_catalog.default.store_sales (17) + : : : : : +- CometBroadcastExchange (21) + : : : : : +- CometFilter (20) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (19) + : : : : +- ReusedExchange (24) + : : : +- BroadcastExchange (47) + : : : +- * HashAggregate (46) + : : : +- Exchange (45) + : : : +- * HashAggregate (44) + : : : +- * ColumnarToRow (43) + : : : +- CometProject (42) + : : : +- CometBroadcastHashJoin (41) + : : : :- CometProject (39) + : : : : +- CometBroadcastHashJoin (38) + : : : : :- CometFilter (34) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (33) + : : : : +- CometBroadcastExchange (37) + : : : : +- CometFilter (36) + : : : : +- CometScan parquet spark_catalog.default.date_dim (35) + : : : +- ReusedExchange (40) + : : +- BroadcastExchange (62) + : : +- * HashAggregate (61) + : : +- Exchange (60) + : : +- * HashAggregate (59) + : : +- * ColumnarToRow (58) + : : +- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometFilter (51) + : : : : +- CometScan parquet spark_catalog.default.web_sales (50) + : : : +- ReusedExchange (52) + : : +- ReusedExchange (55) + : +- BroadcastExchange (76) + : +- * HashAggregate (75) + : +- Exchange (74) + : +- * HashAggregate (73) + : +- * ColumnarToRow (72) + : +- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometProject (68) + : : +- CometBroadcastHashJoin (67) + : : :- CometFilter (65) + : : : +- CometScan parquet spark_catalog.default.web_sales (64) + : : +- ReusedExchange (66) + : +- ReusedExchange (69) + +- BroadcastExchange (91) + +- * HashAggregate (90) + +- Exchange (89) + +- * HashAggregate (88) + +- * ColumnarToRow (87) + +- CometProject (86) + +- CometBroadcastHashJoin (85) + :- CometProject (83) + : +- CometBroadcastHashJoin (82) + : :- CometFilter (80) + : : +- CometScan parquet spark_catalog.default.web_sales (79) + : +- ReusedExchange (81) + +- ReusedExchange (84) (1) Scan parquet spark_catalog.default.store_sales @@ -180,24 +168,18 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] Aggregate Attributes [1]: [sum#10] Results [4]: [ca_county#9, d_qoy#7, d_year#6, sum#11] -(15) RowToColumnar +(15) Exchange Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#11] +Arguments: hashpartitioning(ca_county#9, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(16) CometColumnarExchange -Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#11] -Arguments: hashpartitioning(ca_county#9, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(17) ColumnarToRow [codegen id : 12] -Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#11] - -(18) HashAggregate [codegen id : 12] +(16) HashAggregate [codegen id : 12] Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#11] Keys [3]: [ca_county#9, d_qoy#7, d_year#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#12] Results [3]: [ca_county#9, d_year#6, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS store_sales#13] -(19) Scan parquet spark_catalog.default.store_sales +(17) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#14, ss_ext_sales_price#15, ss_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -205,84 +187,78 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#16), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(20) CometFilter +(18) CometFilter Input [3]: [ss_addr_sk#14, ss_ext_sales_price#15, ss_sold_date_sk#16] Condition : isnotnull(ss_addr_sk#14) -(21) Scan parquet spark_catalog.default.date_dim +(19) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#18, d_year#19, d_qoy#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(22) CometFilter +(20) CometFilter Input [3]: [d_date_sk#18, d_year#19, d_qoy#20] Condition : ((((isnotnull(d_qoy#20) AND isnotnull(d_year#19)) AND (d_qoy#20 = 2)) AND (d_year#19 = 2000)) AND isnotnull(d_date_sk#18)) -(23) CometBroadcastExchange +(21) CometBroadcastExchange Input [3]: [d_date_sk#18, d_year#19, d_qoy#20] Arguments: [d_date_sk#18, d_year#19, d_qoy#20] -(24) CometBroadcastHashJoin +(22) CometBroadcastHashJoin Left output [3]: [ss_addr_sk#14, ss_ext_sales_price#15, ss_sold_date_sk#16] Right output [3]: [d_date_sk#18, d_year#19, d_qoy#20] Arguments: [ss_sold_date_sk#16], [d_date_sk#18], Inner, BuildRight -(25) CometProject +(23) CometProject Input [6]: [ss_addr_sk#14, ss_ext_sales_price#15, ss_sold_date_sk#16, d_date_sk#18, d_year#19, d_qoy#20] Arguments: [ss_addr_sk#14, ss_ext_sales_price#15, d_year#19, d_qoy#20], [ss_addr_sk#14, ss_ext_sales_price#15, d_year#19, d_qoy#20] -(26) ReusedExchange [Reuses operator id: 10] +(24) ReusedExchange [Reuses operator id: 10] Output [2]: [ca_address_sk#21, ca_county#22] -(27) CometBroadcastHashJoin +(25) CometBroadcastHashJoin Left output [4]: [ss_addr_sk#14, ss_ext_sales_price#15, d_year#19, d_qoy#20] Right output [2]: [ca_address_sk#21, ca_county#22] Arguments: [ss_addr_sk#14], [ca_address_sk#21], Inner, BuildRight -(28) CometProject +(26) CometProject Input [6]: [ss_addr_sk#14, ss_ext_sales_price#15, d_year#19, d_qoy#20, ca_address_sk#21, ca_county#22] Arguments: [ss_ext_sales_price#15, d_year#19, d_qoy#20, ca_county#22], [ss_ext_sales_price#15, d_year#19, d_qoy#20, ca_county#22] -(29) ColumnarToRow [codegen id : 2] +(27) ColumnarToRow [codegen id : 2] Input [4]: [ss_ext_sales_price#15, d_year#19, d_qoy#20, ca_county#22] -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [4]: [ss_ext_sales_price#15, d_year#19, d_qoy#20, ca_county#22] Keys [3]: [ca_county#22, d_qoy#20, d_year#19] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#15))] Aggregate Attributes [1]: [sum#23] Results [4]: [ca_county#22, d_qoy#20, d_year#19, sum#24] -(31) RowToColumnar +(29) Exchange Input [4]: [ca_county#22, d_qoy#20, d_year#19, sum#24] +Arguments: hashpartitioning(ca_county#22, d_qoy#20, d_year#19, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(32) CometColumnarExchange -Input [4]: [ca_county#22, d_qoy#20, d_year#19, sum#24] -Arguments: hashpartitioning(ca_county#22, d_qoy#20, d_year#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(33) ColumnarToRow [codegen id : 3] -Input [4]: [ca_county#22, d_qoy#20, d_year#19, sum#24] - -(34) HashAggregate [codegen id : 3] +(30) HashAggregate [codegen id : 3] Input [4]: [ca_county#22, d_qoy#20, d_year#19, sum#24] Keys [3]: [ca_county#22, d_qoy#20, d_year#19] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#15))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#15))#12] Results [2]: [ca_county#22, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#15))#12,17,2) AS store_sales#25] -(35) BroadcastExchange +(31) BroadcastExchange Input [2]: [ca_county#22, store_sales#25] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(36) BroadcastHashJoin [codegen id : 12] +(32) BroadcastHashJoin [codegen id : 12] Left keys [1]: [ca_county#9] Right keys [1]: [ca_county#22] Join type: Inner Join condition: None -(37) Scan parquet spark_catalog.default.store_sales +(33) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_addr_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] Batched: true Location: InMemoryFileIndex [] @@ -290,88 +266,82 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#28), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_addr_sk)] ReadSchema: struct -(38) CometFilter +(34) CometFilter Input [3]: [ss_addr_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] Condition : isnotnull(ss_addr_sk#26) -(39) Scan parquet spark_catalog.default.date_dim +(35) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#30, d_year#31, d_qoy#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(40) CometFilter +(36) CometFilter Input [3]: [d_date_sk#30, d_year#31, d_qoy#32] Condition : ((((isnotnull(d_qoy#32) AND isnotnull(d_year#31)) AND (d_qoy#32 = 3)) AND (d_year#31 = 2000)) AND isnotnull(d_date_sk#30)) -(41) CometBroadcastExchange +(37) CometBroadcastExchange Input [3]: [d_date_sk#30, d_year#31, d_qoy#32] Arguments: [d_date_sk#30, d_year#31, d_qoy#32] -(42) CometBroadcastHashJoin +(38) CometBroadcastHashJoin Left output [3]: [ss_addr_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28] Right output [3]: [d_date_sk#30, d_year#31, d_qoy#32] Arguments: [ss_sold_date_sk#28], [d_date_sk#30], Inner, BuildRight -(43) CometProject +(39) CometProject Input [6]: [ss_addr_sk#26, ss_ext_sales_price#27, ss_sold_date_sk#28, d_date_sk#30, d_year#31, d_qoy#32] Arguments: [ss_addr_sk#26, ss_ext_sales_price#27, d_year#31, d_qoy#32], [ss_addr_sk#26, ss_ext_sales_price#27, d_year#31, d_qoy#32] -(44) ReusedExchange [Reuses operator id: 10] +(40) ReusedExchange [Reuses operator id: 10] Output [2]: [ca_address_sk#33, ca_county#34] -(45) CometBroadcastHashJoin +(41) CometBroadcastHashJoin Left output [4]: [ss_addr_sk#26, ss_ext_sales_price#27, d_year#31, d_qoy#32] Right output [2]: [ca_address_sk#33, ca_county#34] Arguments: [ss_addr_sk#26], [ca_address_sk#33], Inner, BuildRight -(46) CometProject +(42) CometProject Input [6]: [ss_addr_sk#26, ss_ext_sales_price#27, d_year#31, d_qoy#32, ca_address_sk#33, ca_county#34] Arguments: [ss_ext_sales_price#27, d_year#31, d_qoy#32, ca_county#34], [ss_ext_sales_price#27, d_year#31, d_qoy#32, ca_county#34] -(47) ColumnarToRow [codegen id : 4] +(43) ColumnarToRow [codegen id : 4] Input [4]: [ss_ext_sales_price#27, d_year#31, d_qoy#32, ca_county#34] -(48) HashAggregate [codegen id : 4] +(44) HashAggregate [codegen id : 4] Input [4]: [ss_ext_sales_price#27, d_year#31, d_qoy#32, ca_county#34] Keys [3]: [ca_county#34, d_qoy#32, d_year#31] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#27))] Aggregate Attributes [1]: [sum#35] Results [4]: [ca_county#34, d_qoy#32, d_year#31, sum#36] -(49) RowToColumnar -Input [4]: [ca_county#34, d_qoy#32, d_year#31, sum#36] - -(50) CometColumnarExchange -Input [4]: [ca_county#34, d_qoy#32, d_year#31, sum#36] -Arguments: hashpartitioning(ca_county#34, d_qoy#32, d_year#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(51) ColumnarToRow [codegen id : 5] +(45) Exchange Input [4]: [ca_county#34, d_qoy#32, d_year#31, sum#36] +Arguments: hashpartitioning(ca_county#34, d_qoy#32, d_year#31, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(52) HashAggregate [codegen id : 5] +(46) HashAggregate [codegen id : 5] Input [4]: [ca_county#34, d_qoy#32, d_year#31, sum#36] Keys [3]: [ca_county#34, d_qoy#32, d_year#31] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#27))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#27))#12] Results [2]: [ca_county#34, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#27))#12,17,2) AS store_sales#37] -(53) BroadcastExchange +(47) BroadcastExchange Input [2]: [ca_county#34, store_sales#37] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(54) BroadcastHashJoin [codegen id : 12] +(48) BroadcastHashJoin [codegen id : 12] Left keys [1]: [ca_county#22] Right keys [1]: [ca_county#34] Join type: Inner Join condition: None -(55) Project [codegen id : 12] +(49) Project [codegen id : 12] Output [5]: [ca_county#9, d_year#6, store_sales#13, store_sales#25, store_sales#37] Input [7]: [ca_county#9, d_year#6, store_sales#13, ca_county#22, store_sales#25, ca_county#34, store_sales#37] -(56) Scan parquet spark_catalog.default.web_sales +(50) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40] Batched: true Location: InMemoryFileIndex [] @@ -379,72 +349,66 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#40), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(57) CometFilter +(51) CometFilter Input [3]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40] Condition : isnotnull(ws_bill_addr_sk#38) -(58) ReusedExchange [Reuses operator id: 5] +(52) ReusedExchange [Reuses operator id: 5] Output [3]: [d_date_sk#42, d_year#43, d_qoy#44] -(59) CometBroadcastHashJoin +(53) CometBroadcastHashJoin Left output [3]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40] Right output [3]: [d_date_sk#42, d_year#43, d_qoy#44] Arguments: [ws_sold_date_sk#40], [d_date_sk#42], Inner, BuildRight -(60) CometProject +(54) CometProject Input [6]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, ws_sold_date_sk#40, d_date_sk#42, d_year#43, d_qoy#44] Arguments: [ws_bill_addr_sk#38, ws_ext_sales_price#39, d_year#43, d_qoy#44], [ws_bill_addr_sk#38, ws_ext_sales_price#39, d_year#43, d_qoy#44] -(61) ReusedExchange [Reuses operator id: 10] +(55) ReusedExchange [Reuses operator id: 10] Output [2]: [ca_address_sk#45, ca_county#46] -(62) CometBroadcastHashJoin +(56) CometBroadcastHashJoin Left output [4]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, d_year#43, d_qoy#44] Right output [2]: [ca_address_sk#45, ca_county#46] Arguments: [ws_bill_addr_sk#38], [ca_address_sk#45], Inner, BuildRight -(63) CometProject +(57) CometProject Input [6]: [ws_bill_addr_sk#38, ws_ext_sales_price#39, d_year#43, d_qoy#44, ca_address_sk#45, ca_county#46] Arguments: [ws_ext_sales_price#39, d_year#43, d_qoy#44, ca_county#46], [ws_ext_sales_price#39, d_year#43, d_qoy#44, ca_county#46] -(64) ColumnarToRow [codegen id : 6] +(58) ColumnarToRow [codegen id : 6] Input [4]: [ws_ext_sales_price#39, d_year#43, d_qoy#44, ca_county#46] -(65) HashAggregate [codegen id : 6] +(59) HashAggregate [codegen id : 6] Input [4]: [ws_ext_sales_price#39, d_year#43, d_qoy#44, ca_county#46] Keys [3]: [ca_county#46, d_qoy#44, d_year#43] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#39))] Aggregate Attributes [1]: [sum#47] Results [4]: [ca_county#46, d_qoy#44, d_year#43, sum#48] -(66) RowToColumnar -Input [4]: [ca_county#46, d_qoy#44, d_year#43, sum#48] - -(67) CometColumnarExchange +(60) Exchange Input [4]: [ca_county#46, d_qoy#44, d_year#43, sum#48] -Arguments: hashpartitioning(ca_county#46, d_qoy#44, d_year#43, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ca_county#46, d_qoy#44, d_year#43, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(68) ColumnarToRow [codegen id : 7] -Input [4]: [ca_county#46, d_qoy#44, d_year#43, sum#48] - -(69) HashAggregate [codegen id : 7] +(61) HashAggregate [codegen id : 7] Input [4]: [ca_county#46, d_qoy#44, d_year#43, sum#48] Keys [3]: [ca_county#46, d_qoy#44, d_year#43] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#39))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#39))#49] Results [2]: [ca_county#46, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#39))#49,17,2) AS web_sales#50] -(70) BroadcastExchange +(62) BroadcastExchange Input [2]: [ca_county#46, web_sales#50] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(71) BroadcastHashJoin [codegen id : 12] +(63) BroadcastHashJoin [codegen id : 12] Left keys [1]: [ca_county#9] Right keys [1]: [ca_county#46] Join type: Inner Join condition: None -(72) Scan parquet spark_catalog.default.web_sales +(64) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] @@ -452,76 +416,70 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#53), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(73) CometFilter +(65) CometFilter Input [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] Condition : isnotnull(ws_bill_addr_sk#51) -(74) ReusedExchange [Reuses operator id: 23] +(66) ReusedExchange [Reuses operator id: 21] Output [3]: [d_date_sk#55, d_year#56, d_qoy#57] -(75) CometBroadcastHashJoin +(67) CometBroadcastHashJoin Left output [3]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53] Right output [3]: [d_date_sk#55, d_year#56, d_qoy#57] Arguments: [ws_sold_date_sk#53], [d_date_sk#55], Inner, BuildRight -(76) CometProject +(68) CometProject Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, ws_sold_date_sk#53, d_date_sk#55, d_year#56, d_qoy#57] Arguments: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#56, d_qoy#57], [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#56, d_qoy#57] -(77) ReusedExchange [Reuses operator id: 10] +(69) ReusedExchange [Reuses operator id: 10] Output [2]: [ca_address_sk#58, ca_county#59] -(78) CometBroadcastHashJoin +(70) CometBroadcastHashJoin Left output [4]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#56, d_qoy#57] Right output [2]: [ca_address_sk#58, ca_county#59] Arguments: [ws_bill_addr_sk#51], [ca_address_sk#58], Inner, BuildRight -(79) CometProject +(71) CometProject Input [6]: [ws_bill_addr_sk#51, ws_ext_sales_price#52, d_year#56, d_qoy#57, ca_address_sk#58, ca_county#59] Arguments: [ws_ext_sales_price#52, d_year#56, d_qoy#57, ca_county#59], [ws_ext_sales_price#52, d_year#56, d_qoy#57, ca_county#59] -(80) ColumnarToRow [codegen id : 8] +(72) ColumnarToRow [codegen id : 8] Input [4]: [ws_ext_sales_price#52, d_year#56, d_qoy#57, ca_county#59] -(81) HashAggregate [codegen id : 8] +(73) HashAggregate [codegen id : 8] Input [4]: [ws_ext_sales_price#52, d_year#56, d_qoy#57, ca_county#59] Keys [3]: [ca_county#59, d_qoy#57, d_year#56] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#52))] Aggregate Attributes [1]: [sum#60] Results [4]: [ca_county#59, d_qoy#57, d_year#56, sum#61] -(82) RowToColumnar -Input [4]: [ca_county#59, d_qoy#57, d_year#56, sum#61] - -(83) CometColumnarExchange +(74) Exchange Input [4]: [ca_county#59, d_qoy#57, d_year#56, sum#61] -Arguments: hashpartitioning(ca_county#59, d_qoy#57, d_year#56, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(ca_county#59, d_qoy#57, d_year#56, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(84) ColumnarToRow [codegen id : 9] -Input [4]: [ca_county#59, d_qoy#57, d_year#56, sum#61] - -(85) HashAggregate [codegen id : 9] +(75) HashAggregate [codegen id : 9] Input [4]: [ca_county#59, d_qoy#57, d_year#56, sum#61] Keys [3]: [ca_county#59, d_qoy#57, d_year#56] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#52))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#52))#49] Results [2]: [ca_county#59, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#52))#49,17,2) AS web_sales#62] -(86) BroadcastExchange +(76) BroadcastExchange Input [2]: [ca_county#59, web_sales#62] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] -(87) BroadcastHashJoin [codegen id : 12] +(77) BroadcastHashJoin [codegen id : 12] Left keys [1]: [ca_county#46] Right keys [1]: [ca_county#59] Join type: Inner Join condition: (CASE WHEN (web_sales#50 > 0.00) THEN (web_sales#62 / web_sales#50) END > CASE WHEN (store_sales#13 > 0.00) THEN (store_sales#25 / store_sales#13) END) -(88) Project [codegen id : 12] +(78) Project [codegen id : 12] Output [8]: [ca_county#9, d_year#6, store_sales#13, store_sales#25, store_sales#37, ca_county#46, web_sales#50, web_sales#62] Input [9]: [ca_county#9, d_year#6, store_sales#13, store_sales#25, store_sales#37, ca_county#46, web_sales#50, ca_county#59, web_sales#62] -(89) Scan parquet spark_catalog.default.web_sales +(79) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65] Batched: true Location: InMemoryFileIndex [] @@ -529,170 +487,164 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#65), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk)] ReadSchema: struct -(90) CometFilter +(80) CometFilter Input [3]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65] Condition : isnotnull(ws_bill_addr_sk#63) -(91) ReusedExchange [Reuses operator id: 41] +(81) ReusedExchange [Reuses operator id: 37] Output [3]: [d_date_sk#67, d_year#68, d_qoy#69] -(92) CometBroadcastHashJoin +(82) CometBroadcastHashJoin Left output [3]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65] Right output [3]: [d_date_sk#67, d_year#68, d_qoy#69] Arguments: [ws_sold_date_sk#65], [d_date_sk#67], Inner, BuildRight -(93) CometProject +(83) CometProject Input [6]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, ws_sold_date_sk#65, d_date_sk#67, d_year#68, d_qoy#69] Arguments: [ws_bill_addr_sk#63, ws_ext_sales_price#64, d_year#68, d_qoy#69], [ws_bill_addr_sk#63, ws_ext_sales_price#64, d_year#68, d_qoy#69] -(94) ReusedExchange [Reuses operator id: 10] +(84) ReusedExchange [Reuses operator id: 10] Output [2]: [ca_address_sk#70, ca_county#71] -(95) CometBroadcastHashJoin +(85) CometBroadcastHashJoin Left output [4]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, d_year#68, d_qoy#69] Right output [2]: [ca_address_sk#70, ca_county#71] Arguments: [ws_bill_addr_sk#63], [ca_address_sk#70], Inner, BuildRight -(96) CometProject +(86) CometProject Input [6]: [ws_bill_addr_sk#63, ws_ext_sales_price#64, d_year#68, d_qoy#69, ca_address_sk#70, ca_county#71] Arguments: [ws_ext_sales_price#64, d_year#68, d_qoy#69, ca_county#71], [ws_ext_sales_price#64, d_year#68, d_qoy#69, ca_county#71] -(97) ColumnarToRow [codegen id : 10] +(87) ColumnarToRow [codegen id : 10] Input [4]: [ws_ext_sales_price#64, d_year#68, d_qoy#69, ca_county#71] -(98) HashAggregate [codegen id : 10] +(88) HashAggregate [codegen id : 10] Input [4]: [ws_ext_sales_price#64, d_year#68, d_qoy#69, ca_county#71] Keys [3]: [ca_county#71, d_qoy#69, d_year#68] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#64))] Aggregate Attributes [1]: [sum#72] Results [4]: [ca_county#71, d_qoy#69, d_year#68, sum#73] -(99) RowToColumnar -Input [4]: [ca_county#71, d_qoy#69, d_year#68, sum#73] - -(100) CometColumnarExchange -Input [4]: [ca_county#71, d_qoy#69, d_year#68, sum#73] -Arguments: hashpartitioning(ca_county#71, d_qoy#69, d_year#68, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] - -(101) ColumnarToRow [codegen id : 11] +(89) Exchange Input [4]: [ca_county#71, d_qoy#69, d_year#68, sum#73] +Arguments: hashpartitioning(ca_county#71, d_qoy#69, d_year#68, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(102) HashAggregate [codegen id : 11] +(90) HashAggregate [codegen id : 11] Input [4]: [ca_county#71, d_qoy#69, d_year#68, sum#73] Keys [3]: [ca_county#71, d_qoy#69, d_year#68] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#64))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#64))#49] Results [2]: [ca_county#71, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#64))#49,17,2) AS web_sales#74] -(103) BroadcastExchange +(91) BroadcastExchange Input [2]: [ca_county#71, web_sales#74] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] -(104) BroadcastHashJoin [codegen id : 12] +(92) BroadcastHashJoin [codegen id : 12] Left keys [1]: [ca_county#46] Right keys [1]: [ca_county#71] Join type: Inner Join condition: (CASE WHEN (web_sales#62 > 0.00) THEN (web_sales#74 / web_sales#62) END > CASE WHEN (store_sales#25 > 0.00) THEN (store_sales#37 / store_sales#25) END) -(105) Project [codegen id : 12] +(93) Project [codegen id : 12] Output [6]: [ca_county#9, d_year#6, (web_sales#62 / web_sales#50) AS web_q1_q2_increase#75, (store_sales#25 / store_sales#13) AS store_q1_q2_increase#76, (web_sales#74 / web_sales#62) AS web_q2_q3_increase#77, (store_sales#37 / store_sales#25) AS store_q2_q3_increase#78] Input [10]: [ca_county#9, d_year#6, store_sales#13, store_sales#25, store_sales#37, ca_county#46, web_sales#50, web_sales#62, ca_county#71, web_sales#74] -(106) RowToColumnar +(94) RowToColumnar Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#75, store_q1_q2_increase#76, web_q2_q3_increase#77, store_q2_q3_increase#78] -(107) CometColumnarExchange +(95) CometColumnarExchange Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#75, store_q1_q2_increase#76, web_q2_q3_increase#77, store_q2_q3_increase#78] Arguments: rangepartitioning(ca_county#9 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] -(108) CometSort +(96) CometSort Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#75, store_q1_q2_increase#76, web_q2_q3_increase#77, store_q2_q3_increase#78] Arguments: [ca_county#9, d_year#6, web_q1_q2_increase#75, store_q1_q2_increase#76, web_q2_q3_increase#77, store_q2_q3_increase#78], [ca_county#9 ASC NULLS FIRST] -(109) ColumnarToRow [codegen id : 13] +(97) ColumnarToRow [codegen id : 13] Input [6]: [ca_county#9, d_year#6, web_q1_q2_increase#75, store_q1_q2_increase#76, web_q2_q3_increase#77, store_q2_q3_increase#78] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (113) -+- * ColumnarToRow (112) - +- CometFilter (111) - +- CometScan parquet spark_catalog.default.date_dim (110) +BroadcastExchange (101) ++- * ColumnarToRow (100) + +- CometFilter (99) + +- CometScan parquet spark_catalog.default.date_dim (98) -(110) Scan parquet spark_catalog.default.date_dim +(98) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,1), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(111) CometFilter +(99) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 1)) AND (d_year#6 = 2000)) AND isnotnull(d_date_sk#5)) -(112) ColumnarToRow [codegen id : 1] +(100) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] -(113) BroadcastExchange +(101) BroadcastExchange Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] -Subquery:2 Hosting operator id = 19 Hosting Expression = ss_sold_date_sk#16 IN dynamicpruning#17 -BroadcastExchange (117) -+- * ColumnarToRow (116) - +- CometFilter (115) - +- CometScan parquet spark_catalog.default.date_dim (114) +Subquery:2 Hosting operator id = 17 Hosting Expression = ss_sold_date_sk#16 IN dynamicpruning#17 +BroadcastExchange (105) ++- * ColumnarToRow (104) + +- CometFilter (103) + +- CometScan parquet spark_catalog.default.date_dim (102) -(114) Scan parquet spark_catalog.default.date_dim +(102) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#18, d_year#19, d_qoy#20] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(115) CometFilter +(103) CometFilter Input [3]: [d_date_sk#18, d_year#19, d_qoy#20] Condition : ((((isnotnull(d_qoy#20) AND isnotnull(d_year#19)) AND (d_qoy#20 = 2)) AND (d_year#19 = 2000)) AND isnotnull(d_date_sk#18)) -(116) ColumnarToRow [codegen id : 1] +(104) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#18, d_year#19, d_qoy#20] -(117) BroadcastExchange +(105) BroadcastExchange Input [3]: [d_date_sk#18, d_year#19, d_qoy#20] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=14] -Subquery:3 Hosting operator id = 37 Hosting Expression = ss_sold_date_sk#28 IN dynamicpruning#29 -BroadcastExchange (121) -+- * ColumnarToRow (120) - +- CometFilter (119) - +- CometScan parquet spark_catalog.default.date_dim (118) +Subquery:3 Hosting operator id = 33 Hosting Expression = ss_sold_date_sk#28 IN dynamicpruning#29 +BroadcastExchange (109) ++- * ColumnarToRow (108) + +- CometFilter (107) + +- CometScan parquet spark_catalog.default.date_dim (106) -(118) Scan parquet spark_catalog.default.date_dim +(106) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#30, d_year#31, d_qoy#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,3), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(119) CometFilter +(107) CometFilter Input [3]: [d_date_sk#30, d_year#31, d_qoy#32] Condition : ((((isnotnull(d_qoy#32) AND isnotnull(d_year#31)) AND (d_qoy#32 = 3)) AND (d_year#31 = 2000)) AND isnotnull(d_date_sk#30)) -(120) ColumnarToRow [codegen id : 1] +(108) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#30, d_year#31, d_qoy#32] -(121) BroadcastExchange +(109) BroadcastExchange Input [3]: [d_date_sk#30, d_year#31, d_qoy#32] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=15] -Subquery:4 Hosting operator id = 56 Hosting Expression = ws_sold_date_sk#40 IN dynamicpruning#4 +Subquery:4 Hosting operator id = 50 Hosting Expression = ws_sold_date_sk#40 IN dynamicpruning#4 -Subquery:5 Hosting operator id = 72 Hosting Expression = ws_sold_date_sk#53 IN dynamicpruning#17 +Subquery:5 Hosting operator id = 64 Hosting Expression = ws_sold_date_sk#53 IN dynamicpruning#17 -Subquery:6 Hosting operator id = 89 Hosting Expression = ws_sold_date_sk#65 IN dynamicpruning#29 +Subquery:6 Hosting operator id = 79 Hosting Expression = ws_sold_date_sk#65 IN dynamicpruning#29 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt index 4567f8e73..95b4d3a9a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q31/simplified.txt @@ -14,11 +14,65 @@ WholeStageCodegen (13) BroadcastHashJoin [ca_county,ca_county] BroadcastHashJoin [ca_county,ca_county] HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_county,d_qoy,d_year] #2 - RowToColumnar - WholeStageCodegen (1) + InputAdapter + Exchange [ca_county,d_qoy,d_year] #2 + WholeStageCodegen (1) + HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #4 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [ca_address_sk,ca_county] #5 + CometFilter [ca_address_sk,ca_county] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] + InputAdapter + Exchange [ca_county,d_qoy,d_year] #7 + WholeStageCodegen (2) + HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #8 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #9 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + ReusedExchange [ca_address_sk,ca_county] #5 + InputAdapter + BroadcastExchange #10 + WholeStageCodegen (5) + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] + InputAdapter + Exchange [ca_county,d_qoy,d_year] #11 + WholeStageCodegen (4) HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] ColumnarToRow InputAdapter @@ -28,128 +82,43 @@ WholeStageCodegen (13) CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 + SubqueryBroadcast [d_date_sk] #3 + BroadcastExchange #12 WholeStageCodegen (1) ColumnarToRow InputAdapter CometFilter [d_date_sk,d_year,d_qoy] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #4 + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #13 CometFilter [d_date_sk,d_year,d_qoy] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [ca_address_sk,ca_county] #5 - CometFilter [ca_address_sk,ca_county] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_county,d_qoy,d_year] #7 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #9 - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #5 - InputAdapter - BroadcastExchange #10 - WholeStageCodegen (5) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ss_ext_sales_price)),store_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_county,d_qoy,d_year] #11 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [ca_county,d_qoy,d_year,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #12 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #13 - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - ReusedExchange [ca_address_sk,ca_county] #5 + ReusedExchange [ca_address_sk,ca_county] #5 InputAdapter BroadcastExchange #14 WholeStageCodegen (7) HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_county,d_qoy,d_year] #15 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year,d_qoy] #4 - ReusedExchange [ca_address_sk,ca_county] #5 + InputAdapter + Exchange [ca_county,d_qoy,d_year] #15 + WholeStageCodegen (6) + HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_year,d_qoy] #4 + ReusedExchange [ca_address_sk,ca_county] #5 InputAdapter BroadcastExchange #16 WholeStageCodegen (9) HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_county,d_qoy,d_year] #17 - RowToColumnar - WholeStageCodegen (8) - HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] - CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] - CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] - CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] - CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #9 - ReusedExchange [ca_address_sk,ca_county] #5 - InputAdapter - BroadcastExchange #18 - WholeStageCodegen (11) - HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_county,d_qoy,d_year] #19 - RowToColumnar - WholeStageCodegen (10) + InputAdapter + Exchange [ca_county,d_qoy,d_year] #17 + WholeStageCodegen (8) HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] ColumnarToRow InputAdapter @@ -159,6 +128,25 @@ WholeStageCodegen (13) CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk,d_year,d_qoy] #13 + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #9 ReusedExchange [ca_address_sk,ca_county] #5 + InputAdapter + BroadcastExchange #18 + WholeStageCodegen (11) + HashAggregate [ca_county,d_qoy,d_year,sum] [sum(UnscaledValue(ws_ext_sales_price)),web_sales,sum] + InputAdapter + Exchange [ca_county,d_qoy,d_year] #19 + WholeStageCodegen (10) + HashAggregate [ca_county,d_qoy,d_year,ws_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] + CometProject [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy] + CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk,d_year,d_qoy] + CometFilter [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + ReusedExchange [d_date_sk,d_year,d_qoy] #13 + ReusedExchange [ca_address_sk,ca_county] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt index 6a4fcb1d8..fe9e7d49a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/explain.txt @@ -1,39 +1,35 @@ == Physical Plan == -* HashAggregate (35) -+- * ColumnarToRow (34) - +- CometColumnarExchange (33) - +- RowToColumnar (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.item (3) - : +- BroadcastExchange (25) - : +- * Filter (24) - : +- * HashAggregate (23) - : +- * ColumnarToRow (22) - : +- CometColumnarExchange (21) - : +- RowToColumnar (20) - : +- * HashAggregate (19) - : +- * ColumnarToRow (18) - : +- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometFilter (11) - : : +- CometScan parquet spark_catalog.default.catalog_sales (10) - : +- CometBroadcastExchange (15) - : +- CometProject (14) - : +- CometFilter (13) - : +- CometScan parquet spark_catalog.default.date_dim (12) - +- ReusedExchange (28) +* HashAggregate (31) ++- Exchange (30) + +- * HashAggregate (29) + +- * Project (28) + +- * BroadcastHashJoin Inner BuildRight (27) + :- * Project (25) + : +- * BroadcastHashJoin Inner BuildRight (24) + : :- * ColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.item (3) + : +- BroadcastExchange (23) + : +- * Filter (22) + : +- * HashAggregate (21) + : +- Exchange (20) + : +- * HashAggregate (19) + : +- * ColumnarToRow (18) + : +- CometProject (17) + : +- CometBroadcastHashJoin (16) + : :- CometFilter (11) + : : +- CometScan parquet spark_catalog.default.catalog_sales (10) + : +- CometBroadcastExchange (15) + : +- CometProject (14) + : +- CometFilter (13) + : +- CometScan parquet spark_catalog.default.date_dim (12) + +- ReusedExchange (26) (1) Scan parquet spark_catalog.default.catalog_sales @@ -129,72 +125,60 @@ Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#8))] Aggregate Attributes [2]: [sum#13, count#14] Results [3]: [cs_item_sk#7, sum#15, count#16] -(20) RowToColumnar +(20) Exchange Input [3]: [cs_item_sk#7, sum#15, count#16] +Arguments: hashpartitioning(cs_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(21) CometColumnarExchange -Input [3]: [cs_item_sk#7, sum#15, count#16] -Arguments: hashpartitioning(cs_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(22) ColumnarToRow [codegen id : 2] -Input [3]: [cs_item_sk#7, sum#15, count#16] - -(23) HashAggregate [codegen id : 2] +(21) HashAggregate [codegen id : 2] Input [3]: [cs_item_sk#7, sum#15, count#16] Keys [1]: [cs_item_sk#7] Functions [1]: [avg(UnscaledValue(cs_ext_discount_amt#8))] Aggregate Attributes [1]: [avg(UnscaledValue(cs_ext_discount_amt#8))#17] Results [2]: [(1.3 * cast((avg(UnscaledValue(cs_ext_discount_amt#8))#17 / 100.0) as decimal(11,6))) AS (1.3 * avg(cs_ext_discount_amt))#18, cs_item_sk#7] -(24) Filter [codegen id : 2] +(22) Filter [codegen id : 2] Input [2]: [(1.3 * avg(cs_ext_discount_amt))#18, cs_item_sk#7] Condition : isnotnull((1.3 * avg(cs_ext_discount_amt))#18) -(25) BroadcastExchange +(23) BroadcastExchange Input [2]: [(1.3 * avg(cs_ext_discount_amt))#18, cs_item_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=2] -(26) BroadcastHashJoin [codegen id : 4] +(24) BroadcastHashJoin [codegen id : 4] Left keys [1]: [i_item_sk#5] Right keys [1]: [cs_item_sk#7] Join type: Inner Join condition: (cast(cs_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(cs_ext_discount_amt))#18) -(27) Project [codegen id : 4] +(25) Project [codegen id : 4] Output [2]: [cs_ext_discount_amt#2, cs_sold_date_sk#3] Input [5]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, i_item_sk#5, (1.3 * avg(cs_ext_discount_amt))#18, cs_item_sk#7] -(28) ReusedExchange [Reuses operator id: 40] +(26) ReusedExchange [Reuses operator id: 36] Output [1]: [d_date_sk#19] -(29) BroadcastHashJoin [codegen id : 4] +(27) BroadcastHashJoin [codegen id : 4] Left keys [1]: [cs_sold_date_sk#3] Right keys [1]: [d_date_sk#19] Join type: Inner Join condition: None -(30) Project [codegen id : 4] +(28) Project [codegen id : 4] Output [1]: [cs_ext_discount_amt#2] Input [3]: [cs_ext_discount_amt#2, cs_sold_date_sk#3, d_date_sk#19] -(31) HashAggregate [codegen id : 4] +(29) HashAggregate [codegen id : 4] Input [1]: [cs_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] Aggregate Attributes [1]: [sum#20] Results [1]: [sum#21] -(32) RowToColumnar -Input [1]: [sum#21] - -(33) CometColumnarExchange -Input [1]: [sum#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(34) ColumnarToRow [codegen id : 5] +(30) Exchange Input [1]: [sum#21] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] -(35) HashAggregate [codegen id : 5] +(31) HashAggregate [codegen id : 5] Input [1]: [sum#21] Keys: [] Functions [1]: [sum(UnscaledValue(cs_ext_discount_amt#2))] @@ -204,32 +188,32 @@ Results [1]: [MakeDecimal(sum(UnscaledValue(cs_ext_discount_amt#2))#22,17,2) AS ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (40) -+- * ColumnarToRow (39) - +- CometProject (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.date_dim (36) +BroadcastExchange (36) ++- * ColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan parquet spark_catalog.default.date_dim (32) -(36) Scan parquet spark_catalog.default.date_dim +(32) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_date#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct -(37) CometFilter +(33) CometFilter Input [2]: [d_date_sk#19, d_date#24] Condition : (((isnotnull(d_date#24) AND (d_date#24 >= 2000-01-27)) AND (d_date#24 <= 2000-04-26)) AND isnotnull(d_date_sk#19)) -(38) CometProject +(34) CometProject Input [2]: [d_date_sk#19, d_date#24] Arguments: [d_date_sk#19], [d_date_sk#19] -(39) ColumnarToRow [codegen id : 1] +(35) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#19] -(40) BroadcastExchange +(36) BroadcastExchange Input [1]: [d_date_sk#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt index d415c4b58..b6ba1fbab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q32/simplified.txt @@ -1,54 +1,50 @@ WholeStageCodegen (5) HashAggregate [sum] [sum(UnscaledValue(cs_ext_discount_amt)),excess discount amount,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [cs_ext_discount_amt] [sum,sum] - Project [cs_ext_discount_amt] - BroadcastHashJoin [cs_sold_date_sk,d_date_sk] - Project [cs_ext_discount_amt,cs_sold_date_sk] - BroadcastHashJoin [i_item_sk,cs_item_sk,cs_ext_discount_amt,(1.3 * avg(cs_ext_discount_amt))] - ColumnarToRow - InputAdapter - CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] - CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk] #3 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [(1.3 * avg(cs_ext_discount_amt))] - HashAggregate [cs_item_sk,sum,count] [avg(UnscaledValue(cs_ext_discount_amt)),(1.3 * avg(cs_ext_discount_amt)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [cs_item_sk] #5 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [cs_item_sk,cs_ext_discount_amt] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_item_sk,cs_ext_discount_amt] - CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] - CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (4) + HashAggregate [cs_ext_discount_amt] [sum,sum] + Project [cs_ext_discount_amt] + BroadcastHashJoin [cs_sold_date_sk,d_date_sk] + Project [cs_ext_discount_amt,cs_sold_date_sk] + BroadcastHashJoin [i_item_sk,cs_item_sk,cs_ext_discount_amt,(1.3 * avg(cs_ext_discount_amt))] + ColumnarToRow + InputAdapter + CometProject [cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] + CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,i_item_sk] + CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [i_item_sk] #3 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_manufact_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] InputAdapter - ReusedExchange [d_date_sk] #2 + BroadcastExchange #4 + WholeStageCodegen (2) + Filter [(1.3 * avg(cs_ext_discount_amt))] + HashAggregate [cs_item_sk,sum,count] [avg(UnscaledValue(cs_ext_discount_amt)),(1.3 * avg(cs_ext_discount_amt)),sum,count] + InputAdapter + Exchange [cs_item_sk] #5 + WholeStageCodegen (1) + HashAggregate [cs_item_sk,cs_ext_discount_amt] [sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [cs_item_sk,cs_ext_discount_amt] + CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] + CometFilter [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + ReusedExchange [d_date_sk] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt index d4e67403e..4c00423c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/explain.txt @@ -1,75 +1,67 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * HashAggregate (70) - +- * ColumnarToRow (69) - +- CometColumnarExchange (68) - +- RowToColumnar (67) - +- * HashAggregate (66) - +- Union (65) - :- * HashAggregate (30) - : +- * ColumnarToRow (29) - : +- CometColumnarExchange (28) - : +- RowToColumnar (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer_address (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.item (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometScan parquet spark_catalog.default.item (17) - :- * HashAggregate (47) - : +- * ColumnarToRow (46) - : +- CometColumnarExchange (45) - : +- RowToColumnar (44) - : +- * HashAggregate (43) - : +- * ColumnarToRow (42) - : +- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (38) - : : +- CometBroadcastHashJoin (37) - : : :- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometFilter (32) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (31) - : : : +- ReusedExchange (33) - : : +- ReusedExchange (36) - : +- ReusedExchange (39) - +- * HashAggregate (64) - +- * ColumnarToRow (63) - +- CometColumnarExchange (62) - +- RowToColumnar (61) - +- * HashAggregate (60) - +- * ColumnarToRow (59) - +- CometProject (58) - +- CometBroadcastHashJoin (57) - :- CometProject (55) - : +- CometBroadcastHashJoin (54) - : :- CometProject (52) - : : +- CometBroadcastHashJoin (51) - : : :- CometFilter (49) - : : : +- CometScan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (50) - : +- ReusedExchange (53) - +- ReusedExchange (56) +TakeOrderedAndProject (63) ++- * HashAggregate (62) + +- Exchange (61) + +- * HashAggregate (60) + +- Union (59) + :- * HashAggregate (28) + : +- Exchange (27) + : +- * HashAggregate (26) + : +- * ColumnarToRow (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan parquet spark_catalog.default.item (17) + :- * HashAggregate (43) + : +- Exchange (42) + : +- * HashAggregate (41) + : +- * ColumnarToRow (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- * HashAggregate (58) + +- Exchange (57) + +- * HashAggregate (56) + +- * ColumnarToRow (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometProject (48) + : : +- CometBroadcastHashJoin (47) + : : :- CometFilter (45) + : : : +- CometScan parquet spark_catalog.default.web_sales (44) + : : +- ReusedExchange (46) + : +- ReusedExchange (49) + +- ReusedExchange (52) (1) Scan parquet spark_catalog.default.store_sales @@ -198,24 +190,18 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum#15] Results [2]: [i_manufact_id#12, sum#16] -(27) RowToColumnar +(27) Exchange Input [2]: [i_manufact_id#12, sum#16] +Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(28) CometColumnarExchange -Input [2]: [i_manufact_id#12, sum#16] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(29) ColumnarToRow [codegen id : 2] -Input [2]: [i_manufact_id#12, sum#16] - -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [2]: [i_manufact_id#12, sum#16] Keys [1]: [i_manufact_id#12] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#17] Results [2]: [i_manufact_id#12, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#17,17,2) AS total_sales#18] -(31) Scan parquet spark_catalog.default.catalog_sales +(29) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22] Batched: true Location: InMemoryFileIndex [] @@ -223,74 +209,68 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#22), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(32) CometFilter +(30) CometFilter Input [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22] Condition : (isnotnull(cs_bill_addr_sk#19) AND isnotnull(cs_item_sk#20)) -(33) ReusedExchange [Reuses operator id: 6] +(31) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#24] -(34) CometBroadcastHashJoin +(32) CometBroadcastHashJoin Left output [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22] Right output [1]: [d_date_sk#24] Arguments: [cs_sold_date_sk#22], [d_date_sk#24], Inner, BuildRight -(35) CometProject +(33) CometProject Input [5]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22, d_date_sk#24] Arguments: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21], [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21] -(36) ReusedExchange [Reuses operator id: 12] +(34) ReusedExchange [Reuses operator id: 12] Output [1]: [ca_address_sk#25] -(37) CometBroadcastHashJoin +(35) CometBroadcastHashJoin Left output [3]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21] Right output [1]: [ca_address_sk#25] Arguments: [cs_bill_addr_sk#19], [ca_address_sk#25], Inner, BuildRight -(38) CometProject +(36) CometProject Input [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, ca_address_sk#25] Arguments: [cs_item_sk#20, cs_ext_sales_price#21], [cs_item_sk#20, cs_ext_sales_price#21] -(39) ReusedExchange [Reuses operator id: 22] +(37) ReusedExchange [Reuses operator id: 22] Output [2]: [i_item_sk#26, i_manufact_id#27] -(40) CometBroadcastHashJoin +(38) CometBroadcastHashJoin Left output [2]: [cs_item_sk#20, cs_ext_sales_price#21] Right output [2]: [i_item_sk#26, i_manufact_id#27] Arguments: [cs_item_sk#20], [i_item_sk#26], Inner, BuildRight -(41) CometProject +(39) CometProject Input [4]: [cs_item_sk#20, cs_ext_sales_price#21, i_item_sk#26, i_manufact_id#27] Arguments: [cs_ext_sales_price#21, i_manufact_id#27], [cs_ext_sales_price#21, i_manufact_id#27] -(42) ColumnarToRow [codegen id : 3] +(40) ColumnarToRow [codegen id : 3] Input [2]: [cs_ext_sales_price#21, i_manufact_id#27] -(43) HashAggregate [codegen id : 3] +(41) HashAggregate [codegen id : 3] Input [2]: [cs_ext_sales_price#21, i_manufact_id#27] Keys [1]: [i_manufact_id#27] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#21))] Aggregate Attributes [1]: [sum#28] Results [2]: [i_manufact_id#27, sum#29] -(44) RowToColumnar -Input [2]: [i_manufact_id#27, sum#29] - -(45) CometColumnarExchange -Input [2]: [i_manufact_id#27, sum#29] -Arguments: hashpartitioning(i_manufact_id#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(46) ColumnarToRow [codegen id : 4] +(42) Exchange Input [2]: [i_manufact_id#27, sum#29] +Arguments: hashpartitioning(i_manufact_id#27, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(47) HashAggregate [codegen id : 4] +(43) HashAggregate [codegen id : 4] Input [2]: [i_manufact_id#27, sum#29] Keys [1]: [i_manufact_id#27] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#21))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#21))#30] Results [2]: [i_manufact_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#21))#30,17,2) AS total_sales#31] -(48) Scan parquet spark_catalog.default.web_sales +(44) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35] Batched: true Location: InMemoryFileIndex [] @@ -298,137 +278,125 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#35), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(49) CometFilter +(45) CometFilter Input [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35] Condition : (isnotnull(ws_bill_addr_sk#33) AND isnotnull(ws_item_sk#32)) -(50) ReusedExchange [Reuses operator id: 6] +(46) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#37] -(51) CometBroadcastHashJoin +(47) CometBroadcastHashJoin Left output [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35] Right output [1]: [d_date_sk#37] Arguments: [ws_sold_date_sk#35], [d_date_sk#37], Inner, BuildRight -(52) CometProject +(48) CometProject Input [5]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35, d_date_sk#37] Arguments: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34], [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34] -(53) ReusedExchange [Reuses operator id: 12] +(49) ReusedExchange [Reuses operator id: 12] Output [1]: [ca_address_sk#38] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [3]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34] Right output [1]: [ca_address_sk#38] Arguments: [ws_bill_addr_sk#33], [ca_address_sk#38], Inner, BuildRight -(55) CometProject +(51) CometProject Input [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ca_address_sk#38] Arguments: [ws_item_sk#32, ws_ext_sales_price#34], [ws_item_sk#32, ws_ext_sales_price#34] -(56) ReusedExchange [Reuses operator id: 22] +(52) ReusedExchange [Reuses operator id: 22] Output [2]: [i_item_sk#39, i_manufact_id#40] -(57) CometBroadcastHashJoin +(53) CometBroadcastHashJoin Left output [2]: [ws_item_sk#32, ws_ext_sales_price#34] Right output [2]: [i_item_sk#39, i_manufact_id#40] Arguments: [ws_item_sk#32], [i_item_sk#39], Inner, BuildRight -(58) CometProject +(54) CometProject Input [4]: [ws_item_sk#32, ws_ext_sales_price#34, i_item_sk#39, i_manufact_id#40] Arguments: [ws_ext_sales_price#34, i_manufact_id#40], [ws_ext_sales_price#34, i_manufact_id#40] -(59) ColumnarToRow [codegen id : 5] +(55) ColumnarToRow [codegen id : 5] Input [2]: [ws_ext_sales_price#34, i_manufact_id#40] -(60) HashAggregate [codegen id : 5] +(56) HashAggregate [codegen id : 5] Input [2]: [ws_ext_sales_price#34, i_manufact_id#40] Keys [1]: [i_manufact_id#40] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#34))] Aggregate Attributes [1]: [sum#41] Results [2]: [i_manufact_id#40, sum#42] -(61) RowToColumnar +(57) Exchange Input [2]: [i_manufact_id#40, sum#42] +Arguments: hashpartitioning(i_manufact_id#40, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(62) CometColumnarExchange -Input [2]: [i_manufact_id#40, sum#42] -Arguments: hashpartitioning(i_manufact_id#40, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(63) ColumnarToRow [codegen id : 6] -Input [2]: [i_manufact_id#40, sum#42] - -(64) HashAggregate [codegen id : 6] +(58) HashAggregate [codegen id : 6] Input [2]: [i_manufact_id#40, sum#42] Keys [1]: [i_manufact_id#40] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#34))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#34))#43] Results [2]: [i_manufact_id#40, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#34))#43,17,2) AS total_sales#44] -(65) Union +(59) Union -(66) HashAggregate [codegen id : 7] +(60) HashAggregate [codegen id : 7] Input [2]: [i_manufact_id#12, total_sales#18] Keys [1]: [i_manufact_id#12] Functions [1]: [partial_sum(total_sales#18)] Aggregate Attributes [2]: [sum#45, isEmpty#46] Results [3]: [i_manufact_id#12, sum#47, isEmpty#48] -(67) RowToColumnar -Input [3]: [i_manufact_id#12, sum#47, isEmpty#48] - -(68) CometColumnarExchange -Input [3]: [i_manufact_id#12, sum#47, isEmpty#48] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(69) ColumnarToRow [codegen id : 8] +(61) Exchange Input [3]: [i_manufact_id#12, sum#47, isEmpty#48] +Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(70) HashAggregate [codegen id : 8] +(62) HashAggregate [codegen id : 8] Input [3]: [i_manufact_id#12, sum#47, isEmpty#48] Keys [1]: [i_manufact_id#12] Functions [1]: [sum(total_sales#18)] Aggregate Attributes [1]: [sum(total_sales#18)#49] Results [2]: [i_manufact_id#12, sum(total_sales#18)#49 AS total_sales#50] -(71) TakeOrderedAndProject +(63) TakeOrderedAndProject Input [2]: [i_manufact_id#12, total_sales#50] Arguments: 100, [total_sales#50 ASC NULLS FIRST], [i_manufact_id#12, total_sales#50] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (76) -+- * ColumnarToRow (75) - +- CometProject (74) - +- CometFilter (73) - +- CometScan parquet spark_catalog.default.date_dim (72) +BroadcastExchange (68) ++- * ColumnarToRow (67) + +- CometProject (66) + +- CometFilter (65) + +- CometScan parquet spark_catalog.default.date_dim (64) -(72) Scan parquet spark_catalog.default.date_dim +(64) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,5), IsNotNull(d_date_sk)] ReadSchema: struct -(73) CometFilter +(65) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_moy#8] Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 5)) AND isnotnull(d_date_sk#6)) -(74) CometProject +(66) CometProject Input [3]: [d_date_sk#6, d_year#7, d_moy#8] Arguments: [d_date_sk#6], [d_date_sk#6] -(75) ColumnarToRow [codegen id : 1] +(67) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(76) BroadcastExchange +(68) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -Subquery:2 Hosting operator id = 31 Hosting Expression = cs_sold_date_sk#22 IN dynamicpruning#5 +Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#22 IN dynamicpruning#5 -Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#35 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#35 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt index 7680cab6c..bf0ef7ae4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q33/simplified.txt @@ -1,97 +1,89 @@ TakeOrderedAndProject [total_sales,i_manufact_id] WholeStageCodegen (8) HashAggregate [i_manufact_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_manufact_id] #1 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_manufact_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_manufact_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_manufact_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_manufact_id] #6 - CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] - CometFilter [i_item_sk,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - CometBroadcastExchange [i_manufact_id] #7 - CometProject [i_manufact_id] - CometFilter [i_category,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_category,i_manufact_id] - WholeStageCodegen (4) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_manufact_id] #8 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [i_manufact_id,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_manufact_id] #6 - WholeStageCodegen (6) - HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_manufact_id] #9 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [i_manufact_id,ws_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,i_manufact_id] - CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] - CometProject [ws_item_sk,ws_ext_sales_price] - CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] - CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_manufact_id] #6 + InputAdapter + Exchange [i_manufact_id] #1 + WholeStageCodegen (7) + HashAggregate [i_manufact_id,total_sales] [sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (2) + HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] + InputAdapter + Exchange [i_manufact_id] #2 + WholeStageCodegen (1) + HashAggregate [i_manufact_id,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #5 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_manufact_id] #6 + CometBroadcastHashJoin [i_item_sk,i_manufact_id,i_manufact_id] + CometFilter [i_item_sk,i_manufact_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] + CometBroadcastExchange [i_manufact_id] #7 + CometProject [i_manufact_id] + CometFilter [i_category,i_manufact_id] + CometScan parquet spark_catalog.default.item [i_category,i_manufact_id] + WholeStageCodegen (4) + HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] + InputAdapter + Exchange [i_manufact_id] #8 + WholeStageCodegen (3) + HashAggregate [i_manufact_id,cs_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [cs_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [ca_address_sk] #5 + ReusedExchange [i_item_sk,i_manufact_id] #6 + WholeStageCodegen (6) + HashAggregate [i_manufact_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] + InputAdapter + Exchange [i_manufact_id] #9 + WholeStageCodegen (5) + HashAggregate [i_manufact_id,ws_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_ext_sales_price,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [ca_address_sk] #5 + ReusedExchange [i_item_sk,i_manufact_id] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt index c33735fa4..dccad5750 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/explain.txt @@ -6,7 +6,7 @@ +- CometBroadcastHashJoin (28) :- CometFilter (24) : +- CometHashAggregate (23) - : +- CometColumnarExchange (22) + : +- CometExchange (22) : +- CometHashAggregate (21) : +- CometProject (20) : +- CometBroadcastHashJoin (19) @@ -134,9 +134,9 @@ Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt index c131bd9c0..779d4cbdd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q34/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometColumnarExchange [ss_ticket_number,ss_customer_sk] #2 + CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt index b35338d18..c06c1dd16 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/explain.txt @@ -1,50 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- Exchange (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * ColumnarToRow (31) + : +- CometFilter (30) + : +- CometScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometFilter (36) + +- CometScan parquet spark_catalog.default.customer_demographics (35) (1) Scan parquet spark_catalog.default.customer @@ -240,56 +238,50 @@ Functions [10]: [partial_count(1), partial_min(cd_dep_count#25), partial_max(cd_ Aggregate Attributes [13]: [count#28, min#29, max#30, sum#31, count#32, min#33, max#34, sum#35, count#36, min#37, max#38, sum#39, count#40] Results [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] -(42) RowToColumnar +(42) Exchange Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) CometColumnarExchange -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] -Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 6] -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] - -(45) HashAggregate [codegen id : 6] +(43) HashAggregate [codegen id : 6] Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] Keys [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [10]: [count(1), min(cd_dep_count#25), max(cd_dep_count#25), avg(cd_dep_count#25), min(cd_dep_employed_count#26), max(cd_dep_employed_count#26), avg(cd_dep_employed_count#26), min(cd_dep_college_count#27), max(cd_dep_college_count#27), avg(cd_dep_college_count#27)] Aggregate Attributes [10]: [count(1)#54, min(cd_dep_count#25)#55, max(cd_dep_count#25)#56, avg(cd_dep_count#25)#57, min(cd_dep_employed_count#26)#58, max(cd_dep_employed_count#26)#59, avg(cd_dep_employed_count#26)#60, min(cd_dep_college_count#27)#61, max(cd_dep_college_count#27)#62, avg(cd_dep_college_count#27)#63] Results [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, count(1)#54 AS cnt1#64, min(cd_dep_count#25)#55 AS min(cd_dep_count)#65, max(cd_dep_count#25)#56 AS max(cd_dep_count)#66, avg(cd_dep_count#25)#57 AS avg(cd_dep_count)#67, cd_dep_employed_count#26, count(1)#54 AS cnt2#68, min(cd_dep_employed_count#26)#58 AS min(cd_dep_employed_count)#69, max(cd_dep_employed_count#26)#59 AS max(cd_dep_employed_count)#70, avg(cd_dep_employed_count#26)#60 AS avg(cd_dep_employed_count)#71, cd_dep_college_count#27, count(1)#54 AS cnt3#72, min(cd_dep_college_count#27)#61 AS min(cd_dep_college_count)#73, max(cd_dep_college_count#27)#62 AS max(cd_dep_college_count)#74, avg(cd_dep_college_count#27)#63 AS avg(cd_dep_college_count)#75, cd_dep_count#25] -(46) TakeOrderedAndProject +(44) TakeOrderedAndProject Input [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75, cd_dep_count#25] Arguments: 100, [ca_state#21 ASC NULLS FIRST, cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [ca_state#21, cd_gender#23, cd_marital_status#24, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (51) -+- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.date_dim (47) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan parquet spark_catalog.default.date_dim (45) -(47) Scan parquet spark_catalog.default.date_dim +(45) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(48) CometFilter +(46) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) -(49) CometProject +(47) CometProject Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Arguments: [d_date_sk#9], [d_date_sk#9] -(50) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(51) BroadcastExchange +(49) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt index f9ef660d5..9deb51342 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q35/simplified.txt @@ -1,72 +1,70 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + InputAdapter + Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt index ca46904b3..1fdba8689 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/explain.txt @@ -1,37 +1,33 @@ == Physical Plan == -TakeOrderedAndProject (33) -+- * Project (32) - +- Window (31) - +- * ColumnarToRow (30) - +- CometSort (29) - +- CometColumnarExchange (28) - +- RowToColumnar (27) - +- * HashAggregate (26) - +- * ColumnarToRow (25) - +- CometColumnarExchange (24) - +- RowToColumnar (23) - +- * HashAggregate (22) - +- * ColumnarToRow (21) - +- CometExpand (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.item (9) - +- CometBroadcastExchange (17) - +- CometProject (16) - +- CometFilter (15) - +- CometScan parquet spark_catalog.default.store (14) +TakeOrderedAndProject (29) ++- * Project (28) + +- Window (27) + +- * Sort (26) + +- Exchange (25) + +- * HashAggregate (24) + +- Exchange (23) + +- * HashAggregate (22) + +- * ColumnarToRow (21) + +- CometExpand (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.item (9) + +- CometBroadcastExchange (17) + +- CometProject (16) + +- CometFilter (15) + +- CometScan parquet spark_catalog.default.store (14) (1) Scan parquet spark_catalog.default.store_sales @@ -140,78 +136,66 @@ Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(Unscale Aggregate Attributes [2]: [sum#17, sum#18] Results [5]: [i_category#14, i_class#15, spark_grouping_id#16, sum#19, sum#20] -(23) RowToColumnar +(23) Exchange Input [5]: [i_category#14, i_class#15, spark_grouping_id#16, sum#19, sum#20] +Arguments: hashpartitioning(i_category#14, i_class#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(24) CometColumnarExchange -Input [5]: [i_category#14, i_class#15, spark_grouping_id#16, sum#19, sum#20] -Arguments: hashpartitioning(i_category#14, i_class#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(25) ColumnarToRow [codegen id : 2] -Input [5]: [i_category#14, i_class#15, spark_grouping_id#16, sum#19, sum#20] - -(26) HashAggregate [codegen id : 2] +(24) HashAggregate [codegen id : 2] Input [5]: [i_category#14, i_class#15, spark_grouping_id#16, sum#19, sum#20] Keys [3]: [i_category#14, i_class#15, spark_grouping_id#16] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#21, sum(UnscaledValue(ss_ext_sales_price#3))#22] Results [7]: [(MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#21,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#22,17,2)) AS gross_margin#23, i_category#14, i_class#15, (cast((shiftright(spark_grouping_id#16, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#16, 0) & 1) as tinyint)) AS lochierarchy#24, (MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#21,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#22,17,2)) AS _w0#25, (cast((shiftright(spark_grouping_id#16, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#16, 0) & 1) as tinyint)) AS _w1#26, CASE WHEN (cast((shiftright(spark_grouping_id#16, 0) & 1) as tinyint) = 0) THEN i_category#14 END AS _w2#27] -(27) RowToColumnar -Input [7]: [gross_margin#23, i_category#14, i_class#15, lochierarchy#24, _w0#25, _w1#26, _w2#27] - -(28) CometColumnarExchange -Input [7]: [gross_margin#23, i_category#14, i_class#15, lochierarchy#24, _w0#25, _w1#26, _w2#27] -Arguments: hashpartitioning(_w1#26, _w2#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(29) CometSort +(25) Exchange Input [7]: [gross_margin#23, i_category#14, i_class#15, lochierarchy#24, _w0#25, _w1#26, _w2#27] -Arguments: [gross_margin#23, i_category#14, i_class#15, lochierarchy#24, _w0#25, _w1#26, _w2#27], [_w1#26 ASC NULLS FIRST, _w2#27 ASC NULLS FIRST, _w0#25 ASC NULLS FIRST] +Arguments: hashpartitioning(_w1#26, _w2#27, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(30) ColumnarToRow [codegen id : 3] +(26) Sort [codegen id : 3] Input [7]: [gross_margin#23, i_category#14, i_class#15, lochierarchy#24, _w0#25, _w1#26, _w2#27] +Arguments: [_w1#26 ASC NULLS FIRST, _w2#27 ASC NULLS FIRST, _w0#25 ASC NULLS FIRST], false, 0 -(31) Window +(27) Window Input [7]: [gross_margin#23, i_category#14, i_class#15, lochierarchy#24, _w0#25, _w1#26, _w2#27] Arguments: [rank(_w0#25) windowspecdefinition(_w1#26, _w2#27, _w0#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#28], [_w1#26, _w2#27], [_w0#25 ASC NULLS FIRST] -(32) Project [codegen id : 4] +(28) Project [codegen id : 4] Output [5]: [gross_margin#23, i_category#14, i_class#15, lochierarchy#24, rank_within_parent#28] Input [8]: [gross_margin#23, i_category#14, i_class#15, lochierarchy#24, _w0#25, _w1#26, _w2#27, rank_within_parent#28] -(33) TakeOrderedAndProject +(29) TakeOrderedAndProject Input [5]: [gross_margin#23, i_category#14, i_class#15, lochierarchy#24, rank_within_parent#28] Arguments: 100, [lochierarchy#24 DESC NULLS LAST, CASE WHEN (lochierarchy#24 = 0) THEN i_category#14 END ASC NULLS FIRST, rank_within_parent#28 ASC NULLS FIRST], [gross_margin#23, i_category#14, i_class#15, lochierarchy#24, rank_within_parent#28] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * ColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan parquet spark_catalog.default.date_dim (34) +BroadcastExchange (34) ++- * ColumnarToRow (33) + +- CometProject (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.date_dim (30) -(34) Scan parquet spark_catalog.default.date_dim +(30) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#7, d_year#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(35) CometFilter +(31) CometFilter Input [2]: [d_date_sk#7, d_year#8] Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) -(36) CometProject +(32) CometProject Input [2]: [d_date_sk#7, d_year#8] Arguments: [d_date_sk#7], [d_date_sk#7] -(37) ColumnarToRow [codegen id : 1] +(33) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#7] -(38) BroadcastExchange +(34) BroadcastExchange Input [1]: [d_date_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt index f3febe692..58d617430 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q36/simplified.txt @@ -4,46 +4,42 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (3) - ColumnarToRow + Sort [_w1,_w2,_w0] InputAdapter - CometSort [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2] - CometColumnarExchange [_w1,_w2] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w0,_w1,_w2,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class,spark_grouping_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_class,spark_grouping_id,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] - CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] - CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #5 - CometFilter [i_item_sk,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #6 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + Exchange [_w1,_w2] #1 + WholeStageCodegen (2) + HashAggregate [i_category,i_class,spark_grouping_id,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,lochierarchy,_w0,_w1,_w2,sum,sum] + InputAdapter + Exchange [i_category,i_class,spark_grouping_id] #2 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,spark_grouping_id,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] + CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] + CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometFilter [i_item_sk,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [s_store_sk] #6 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt index 42dab0b38..bd5ca5ef6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometBroadcastHashJoin (20) @@ -132,9 +132,9 @@ Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Functions: [] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt index 39edfb519..85e86ca8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q37/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometColumnarExchange [i_item_id,i_item_desc,i_current_price] #1 + CometExchange [i_item_id,i_item_desc,i_current_price] #1 CometHashAggregate [i_item_id,i_item_desc,i_current_price] CometProject [i_item_id,i_item_desc,i_current_price] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,cs_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt index 091883bd6..598dccaf0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == * ColumnarToRow (47) +- CometHashAggregate (46) - +- CometColumnarExchange (45) + +- CometExchange (45) +- CometHashAggregate (44) +- CometProject (43) +- CometBroadcastHashJoin (42) :- CometBroadcastHashJoin (29) : :- CometHashAggregate (16) - : : +- CometColumnarExchange (15) + : : +- CometExchange (15) : : +- CometHashAggregate (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) @@ -24,7 +24,7 @@ : : +- CometScan parquet spark_catalog.default.customer (9) : +- CometBroadcastExchange (28) : +- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -36,7 +36,7 @@ : +- ReusedExchange (22) +- CometBroadcastExchange (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -117,9 +117,9 @@ Input [3]: [c_last_name#9, c_first_name#8, d_date#5] Keys [3]: [c_last_name#9, c_first_name#8, d_date#5] Functions: [] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [c_last_name#9, c_first_name#8, d_date#5] -Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [c_last_name#9, c_first_name#8, d_date#5] @@ -167,9 +167,9 @@ Input [3]: [c_last_name#17, c_first_name#16, d_date#14] Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(26) CometColumnarExchange +(26) CometExchange Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate Input [3]: [c_last_name#17, c_first_name#16, d_date#14] @@ -226,9 +226,9 @@ Input [3]: [c_last_name#25, c_first_name#24, d_date#22] Keys [3]: [c_last_name#25, c_first_name#24, d_date#22] Functions: [] -(39) CometColumnarExchange +(39) CometExchange Input [3]: [c_last_name#25, c_first_name#24, d_date#22] -Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (40) CometHashAggregate Input [3]: [c_last_name#25, c_first_name#24, d_date#22] @@ -252,9 +252,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(45) CometColumnarExchange +(45) CometExchange Input [1]: [count#26] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (46) CometHashAggregate Input [1]: [count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt index 77970e8d3..8272260f5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q38/simplified.txt @@ -2,13 +2,13 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [count(1),count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #2 + CometExchange [c_last_name,c_first_name,d_date] #2 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -33,7 +33,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #7 + CometExchange [c_last_name,c_first_name,d_date] #7 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -46,7 +46,7 @@ WholeStageCodegen (1) ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 CometBroadcastExchange [c_last_name,c_first_name,d_date] #8 CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #9 + CometExchange [c_last_name,c_first_name,d_date] #9 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt index e266e3e1c..18064ce97 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/explain.txt @@ -1,58 +1,54 @@ == Physical Plan == -* ColumnarToRow (54) -+- CometSort (53) - +- CometColumnarExchange (52) - +- RowToColumnar (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (26) - : +- * Filter (25) - : +- * HashAggregate (24) - : +- * ColumnarToRow (23) - : +- CometColumnarExchange (22) - : +- RowToColumnar (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.inventory (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.warehouse (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.date_dim (13) - +- BroadcastExchange (49) - +- * Project (48) - +- * Filter (47) - +- * HashAggregate (46) - +- * ColumnarToRow (45) - +- CometColumnarExchange (44) - +- RowToColumnar (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometFilter (28) - : : : +- CometScan parquet spark_catalog.default.inventory (27) - : : +- ReusedExchange (29) - : +- ReusedExchange (32) - +- CometBroadcastExchange (38) - +- CometProject (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.date_dim (35) +* ColumnarToRow (50) ++- CometSort (49) + +- CometColumnarExchange (48) + +- RowToColumnar (47) + +- * BroadcastHashJoin Inner BuildRight (46) + :- * Project (24) + : +- * Filter (23) + : +- * HashAggregate (22) + : +- Exchange (21) + : +- * HashAggregate (20) + : +- * ColumnarToRow (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.warehouse (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometScan parquet spark_catalog.default.date_dim (13) + +- BroadcastExchange (45) + +- * Project (44) + +- * Filter (43) + +- * HashAggregate (42) + +- Exchange (41) + +- * HashAggregate (40) + +- * ColumnarToRow (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (32) + : +- CometBroadcastHashJoin (31) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometFilter (26) + : : : +- CometScan parquet spark_catalog.default.inventory (25) + : : +- ReusedExchange (27) + : +- ReusedExchange (30) + +- CometBroadcastExchange (36) + +- CometProject (35) + +- CometFilter (34) + +- CometScan parquet spark_catalog.default.date_dim (33) (1) Scan parquet spark_catalog.default.inventory @@ -153,32 +149,26 @@ Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), par Aggregate Attributes [5]: [n#12, avg#13, m2#14, sum#15, count#16] Results [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#17, avg#18, m2#19, sum#20, count#21] -(21) RowToColumnar +(21) Exchange Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#17, avg#18, m2#19, sum#20, count#21] +Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(22) CometColumnarExchange -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#17, avg#18, m2#19, sum#20, count#21] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(23) ColumnarToRow [codegen id : 4] -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#17, avg#18, m2#19, sum#20, count#21] - -(24) HashAggregate [codegen id : 4] +(22) HashAggregate [codegen id : 4] Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#17, avg#18, m2#19, sum#20, count#21] Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#22, avg(inv_quantity_on_hand#3)#23] Results [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stddev_samp(cast(inv_quantity_on_hand#3 as double))#22 AS stdev#24, avg(inv_quantity_on_hand#3)#23 AS mean#25] -(25) Filter [codegen id : 4] +(23) Filter [codegen id : 4] Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#24, mean#25] Condition : CASE WHEN (mean#25 = 0.0) THEN false ELSE ((stdev#24 / mean#25) > 1.0) END -(26) Project [codegen id : 4] +(24) Project [codegen id : 4] Output [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, CASE WHEN (mean#25 = 0.0) THEN null ELSE (stdev#24 / mean#25) END AS cov#26] Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#24, mean#25] -(27) Scan parquet spark_catalog.default.inventory +(25) Scan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#27, inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30] Batched: true Location: InMemoryFileIndex [] @@ -186,180 +176,174 @@ PartitionFilters: [isnotnull(inv_date_sk#30), dynamicpruningexpression(inv_date_ PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [4]: [inv_item_sk#27, inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30] Condition : (isnotnull(inv_item_sk#27) AND isnotnull(inv_warehouse_sk#28)) -(29) ReusedExchange [Reuses operator id: 5] +(27) ReusedExchange [Reuses operator id: 5] Output [1]: [i_item_sk#32] -(30) CometBroadcastHashJoin +(28) CometBroadcastHashJoin Left output [4]: [inv_item_sk#27, inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30] Right output [1]: [i_item_sk#32] Arguments: [inv_item_sk#27], [i_item_sk#32], Inner, BuildRight -(31) CometProject +(29) CometProject Input [5]: [inv_item_sk#27, inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32] Arguments: [inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32], [inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32] -(32) ReusedExchange [Reuses operator id: 10] +(30) ReusedExchange [Reuses operator id: 10] Output [2]: [w_warehouse_sk#33, w_warehouse_name#34] -(33) CometBroadcastHashJoin +(31) CometBroadcastHashJoin Left output [4]: [inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32] Right output [2]: [w_warehouse_sk#33, w_warehouse_name#34] Arguments: [inv_warehouse_sk#28], [w_warehouse_sk#33], Inner, BuildRight -(34) CometProject +(32) CometProject Input [6]: [inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34] Arguments: [inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34], [inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34] -(35) Scan parquet spark_catalog.default.date_dim +(33) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#35, d_year#36, d_moy#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(36) CometFilter +(34) CometFilter Input [3]: [d_date_sk#35, d_year#36, d_moy#37] Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2001)) AND (d_moy#37 = 2)) AND isnotnull(d_date_sk#35)) -(37) CometProject +(35) CometProject Input [3]: [d_date_sk#35, d_year#36, d_moy#37] Arguments: [d_date_sk#35, d_moy#37], [d_date_sk#35, d_moy#37] -(38) CometBroadcastExchange +(36) CometBroadcastExchange Input [2]: [d_date_sk#35, d_moy#37] Arguments: [d_date_sk#35, d_moy#37] -(39) CometBroadcastHashJoin +(37) CometBroadcastHashJoin Left output [5]: [inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34] Right output [2]: [d_date_sk#35, d_moy#37] Arguments: [inv_date_sk#30], [d_date_sk#35], Inner, BuildRight -(40) CometProject +(38) CometProject Input [7]: [inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34, d_date_sk#35, d_moy#37] Arguments: [inv_quantity_on_hand#29, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34, d_moy#37], [inv_quantity_on_hand#29, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34, d_moy#37] -(41) ColumnarToRow [codegen id : 2] +(39) ColumnarToRow [codegen id : 2] Input [5]: [inv_quantity_on_hand#29, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34, d_moy#37] -(42) HashAggregate [codegen id : 2] +(40) HashAggregate [codegen id : 2] Input [5]: [inv_quantity_on_hand#29, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34, d_moy#37] Keys [4]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#29 as double)), partial_avg(inv_quantity_on_hand#29)] Aggregate Attributes [5]: [n#38, avg#39, m2#40, sum#41, count#42] Results [9]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, n#43, avg#44, m2#45, sum#46, count#47] -(43) RowToColumnar -Input [9]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, n#43, avg#44, m2#45, sum#46, count#47] - -(44) CometColumnarExchange -Input [9]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, n#43, avg#44, m2#45, sum#46, count#47] -Arguments: hashpartitioning(w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(45) ColumnarToRow [codegen id : 3] +(41) Exchange Input [9]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, n#43, avg#44, m2#45, sum#46, count#47] +Arguments: hashpartitioning(w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(46) HashAggregate [codegen id : 3] +(42) HashAggregate [codegen id : 3] Input [9]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, n#43, avg#44, m2#45, sum#46, count#47] Keys [4]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#29 as double)), avg(inv_quantity_on_hand#29)] Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#29 as double))#22, avg(inv_quantity_on_hand#29)#23] Results [5]: [w_warehouse_sk#33, i_item_sk#32, d_moy#37, stddev_samp(cast(inv_quantity_on_hand#29 as double))#22 AS stdev#48, avg(inv_quantity_on_hand#29)#23 AS mean#49] -(47) Filter [codegen id : 3] +(43) Filter [codegen id : 3] Input [5]: [w_warehouse_sk#33, i_item_sk#32, d_moy#37, stdev#48, mean#49] Condition : CASE WHEN (mean#49 = 0.0) THEN false ELSE ((stdev#48 / mean#49) > 1.0) END -(48) Project [codegen id : 3] +(44) Project [codegen id : 3] Output [5]: [w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, CASE WHEN (mean#49 = 0.0) THEN null ELSE (stdev#48 / mean#49) END AS cov#50] Input [5]: [w_warehouse_sk#33, i_item_sk#32, d_moy#37, stdev#48, mean#49] -(49) BroadcastExchange +(45) BroadcastExchange Input [5]: [w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50] Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=3] -(50) BroadcastHashJoin [codegen id : 4] +(46) BroadcastHashJoin [codegen id : 4] Left keys [2]: [i_item_sk#6, w_warehouse_sk#7] Right keys [2]: [i_item_sk#32, w_warehouse_sk#33] Join type: Inner Join condition: None -(51) RowToColumnar +(47) RowToColumnar Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, cov#26, w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50] -(52) CometColumnarExchange +(48) CometColumnarExchange Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, cov#26, w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50] Arguments: rangepartitioning(w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#25 ASC NULLS FIRST, cov#26 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, mean#49 ASC NULLS FIRST, cov#50 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(53) CometSort +(49) CometSort Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, cov#26, w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50] Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, cov#26, w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50], [w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#25 ASC NULLS FIRST, cov#26 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, mean#49 ASC NULLS FIRST, cov#50 ASC NULLS FIRST] -(54) ColumnarToRow [codegen id : 5] +(50) ColumnarToRow [codegen id : 5] Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, cov#26, w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (59) -+- * ColumnarToRow (58) - +- CometProject (57) - +- CometFilter (56) - +- CometScan parquet spark_catalog.default.date_dim (55) +BroadcastExchange (55) ++- * ColumnarToRow (54) + +- CometProject (53) + +- CometFilter (52) + +- CometScan parquet spark_catalog.default.date_dim (51) -(55) Scan parquet spark_catalog.default.date_dim +(51) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct -(56) CometFilter +(52) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) -(57) CometProject +(53) CometProject Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] -(58) ColumnarToRow [codegen id : 1] +(54) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#9, d_moy#11] -(59) BroadcastExchange +(55) BroadcastExchange Input [2]: [d_date_sk#9, d_moy#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -Subquery:2 Hosting operator id = 27 Hosting Expression = inv_date_sk#30 IN dynamicpruning#31 -BroadcastExchange (64) -+- * ColumnarToRow (63) - +- CometProject (62) - +- CometFilter (61) - +- CometScan parquet spark_catalog.default.date_dim (60) +Subquery:2 Hosting operator id = 25 Hosting Expression = inv_date_sk#30 IN dynamicpruning#31 +BroadcastExchange (60) ++- * ColumnarToRow (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.date_dim (56) -(60) Scan parquet spark_catalog.default.date_dim +(56) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#35, d_year#36, d_moy#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(61) CometFilter +(57) CometFilter Input [3]: [d_date_sk#35, d_year#36, d_moy#37] Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2001)) AND (d_moy#37 = 2)) AND isnotnull(d_date_sk#35)) -(62) CometProject +(58) CometProject Input [3]: [d_date_sk#35, d_year#36, d_moy#37] Arguments: [d_date_sk#35, d_moy#37], [d_date_sk#35, d_moy#37] -(63) ColumnarToRow [codegen id : 1] +(59) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#35, d_moy#37] -(64) BroadcastExchange +(60) BroadcastExchange Input [2]: [d_date_sk#35, d_moy#37] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt index 48c1cc8b9..2a2e04bae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39a/simplified.txt @@ -9,73 +9,69 @@ WholeStageCodegen (5) Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] Filter [mean,stdev] HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [i_item_sk] #4 - CometFilter [i_item_sk] - CometScan parquet spark_catalog.default.item [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #6 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + WholeStageCodegen (1) + HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] + ColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [i_item_sk] #4 + CometFilter [i_item_sk] + CometScan parquet spark_catalog.default.item [i_item_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [d_date_sk,d_moy] #6 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #7 WholeStageCodegen (3) Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] Filter [mean,stdev] HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [i_item_sk] #4 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #5 - CometBroadcastExchange [d_date_sk,d_moy] #10 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + WholeStageCodegen (2) + HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] + ColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #9 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + ReusedExchange [i_item_sk] #4 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [d_date_sk,d_moy] #10 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt index 295647e5d..279efee7b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/explain.txt @@ -1,58 +1,54 @@ == Physical Plan == -* ColumnarToRow (54) -+- CometSort (53) - +- CometColumnarExchange (52) - +- RowToColumnar (51) - +- * BroadcastHashJoin Inner BuildRight (50) - :- * Project (26) - : +- * Filter (25) - : +- * HashAggregate (24) - : +- * ColumnarToRow (23) - : +- CometColumnarExchange (22) - : +- RowToColumnar (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.inventory (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.warehouse (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.date_dim (13) - +- BroadcastExchange (49) - +- * Project (48) - +- * Filter (47) - +- * HashAggregate (46) - +- * ColumnarToRow (45) - +- CometColumnarExchange (44) - +- RowToColumnar (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometFilter (28) - : : : +- CometScan parquet spark_catalog.default.inventory (27) - : : +- ReusedExchange (29) - : +- ReusedExchange (32) - +- CometBroadcastExchange (38) - +- CometProject (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.date_dim (35) +* ColumnarToRow (50) ++- CometSort (49) + +- CometColumnarExchange (48) + +- RowToColumnar (47) + +- * BroadcastHashJoin Inner BuildRight (46) + :- * Project (24) + : +- * Filter (23) + : +- * HashAggregate (22) + : +- Exchange (21) + : +- * HashAggregate (20) + : +- * ColumnarToRow (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.warehouse (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometScan parquet spark_catalog.default.date_dim (13) + +- BroadcastExchange (45) + +- * Project (44) + +- * Filter (43) + +- * HashAggregate (42) + +- Exchange (41) + +- * HashAggregate (40) + +- * ColumnarToRow (39) + +- CometProject (38) + +- CometBroadcastHashJoin (37) + :- CometProject (32) + : +- CometBroadcastHashJoin (31) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometFilter (26) + : : : +- CometScan parquet spark_catalog.default.inventory (25) + : : +- ReusedExchange (27) + : +- ReusedExchange (30) + +- CometBroadcastExchange (36) + +- CometProject (35) + +- CometFilter (34) + +- CometScan parquet spark_catalog.default.date_dim (33) (1) Scan parquet spark_catalog.default.inventory @@ -153,32 +149,26 @@ Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), par Aggregate Attributes [5]: [n#12, avg#13, m2#14, sum#15, count#16] Results [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#17, avg#18, m2#19, sum#20, count#21] -(21) RowToColumnar +(21) Exchange Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#17, avg#18, m2#19, sum#20, count#21] +Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(22) CometColumnarExchange -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#17, avg#18, m2#19, sum#20, count#21] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(23) ColumnarToRow [codegen id : 4] -Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#17, avg#18, m2#19, sum#20, count#21] - -(24) HashAggregate [codegen id : 4] +(22) HashAggregate [codegen id : 4] Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#17, avg#18, m2#19, sum#20, count#21] Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double)), avg(inv_quantity_on_hand#3)] Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#3 as double))#22, avg(inv_quantity_on_hand#3)#23] Results [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stddev_samp(cast(inv_quantity_on_hand#3 as double))#22 AS stdev#24, avg(inv_quantity_on_hand#3)#23 AS mean#25] -(25) Filter [codegen id : 4] +(23) Filter [codegen id : 4] Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#24, mean#25] Condition : (CASE WHEN (mean#25 = 0.0) THEN false ELSE ((stdev#24 / mean#25) > 1.0) END AND CASE WHEN (mean#25 = 0.0) THEN false ELSE ((stdev#24 / mean#25) > 1.5) END) -(26) Project [codegen id : 4] +(24) Project [codegen id : 4] Output [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, CASE WHEN (mean#25 = 0.0) THEN null ELSE (stdev#24 / mean#25) END AS cov#26] Input [5]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, stdev#24, mean#25] -(27) Scan parquet spark_catalog.default.inventory +(25) Scan parquet spark_catalog.default.inventory Output [4]: [inv_item_sk#27, inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30] Batched: true Location: InMemoryFileIndex [] @@ -186,180 +176,174 @@ PartitionFilters: [isnotnull(inv_date_sk#30), dynamicpruningexpression(inv_date_ PushedFilters: [IsNotNull(inv_item_sk), IsNotNull(inv_warehouse_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [4]: [inv_item_sk#27, inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30] Condition : (isnotnull(inv_item_sk#27) AND isnotnull(inv_warehouse_sk#28)) -(29) ReusedExchange [Reuses operator id: 5] +(27) ReusedExchange [Reuses operator id: 5] Output [1]: [i_item_sk#32] -(30) CometBroadcastHashJoin +(28) CometBroadcastHashJoin Left output [4]: [inv_item_sk#27, inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30] Right output [1]: [i_item_sk#32] Arguments: [inv_item_sk#27], [i_item_sk#32], Inner, BuildRight -(31) CometProject +(29) CometProject Input [5]: [inv_item_sk#27, inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32] Arguments: [inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32], [inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32] -(32) ReusedExchange [Reuses operator id: 10] +(30) ReusedExchange [Reuses operator id: 10] Output [2]: [w_warehouse_sk#33, w_warehouse_name#34] -(33) CometBroadcastHashJoin +(31) CometBroadcastHashJoin Left output [4]: [inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32] Right output [2]: [w_warehouse_sk#33, w_warehouse_name#34] Arguments: [inv_warehouse_sk#28], [w_warehouse_sk#33], Inner, BuildRight -(34) CometProject +(32) CometProject Input [6]: [inv_warehouse_sk#28, inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34] Arguments: [inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34], [inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34] -(35) Scan parquet spark_catalog.default.date_dim +(33) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#35, d_year#36, d_moy#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(36) CometFilter +(34) CometFilter Input [3]: [d_date_sk#35, d_year#36, d_moy#37] Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2001)) AND (d_moy#37 = 2)) AND isnotnull(d_date_sk#35)) -(37) CometProject +(35) CometProject Input [3]: [d_date_sk#35, d_year#36, d_moy#37] Arguments: [d_date_sk#35, d_moy#37], [d_date_sk#35, d_moy#37] -(38) CometBroadcastExchange +(36) CometBroadcastExchange Input [2]: [d_date_sk#35, d_moy#37] Arguments: [d_date_sk#35, d_moy#37] -(39) CometBroadcastHashJoin +(37) CometBroadcastHashJoin Left output [5]: [inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34] Right output [2]: [d_date_sk#35, d_moy#37] Arguments: [inv_date_sk#30], [d_date_sk#35], Inner, BuildRight -(40) CometProject +(38) CometProject Input [7]: [inv_quantity_on_hand#29, inv_date_sk#30, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34, d_date_sk#35, d_moy#37] Arguments: [inv_quantity_on_hand#29, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34, d_moy#37], [inv_quantity_on_hand#29, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34, d_moy#37] -(41) ColumnarToRow [codegen id : 2] +(39) ColumnarToRow [codegen id : 2] Input [5]: [inv_quantity_on_hand#29, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34, d_moy#37] -(42) HashAggregate [codegen id : 2] +(40) HashAggregate [codegen id : 2] Input [5]: [inv_quantity_on_hand#29, i_item_sk#32, w_warehouse_sk#33, w_warehouse_name#34, d_moy#37] Keys [4]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#29 as double)), partial_avg(inv_quantity_on_hand#29)] Aggregate Attributes [5]: [n#38, avg#39, m2#40, sum#41, count#42] Results [9]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, n#43, avg#44, m2#45, sum#46, count#47] -(43) RowToColumnar -Input [9]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, n#43, avg#44, m2#45, sum#46, count#47] - -(44) CometColumnarExchange -Input [9]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, n#43, avg#44, m2#45, sum#46, count#47] -Arguments: hashpartitioning(w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(45) ColumnarToRow [codegen id : 3] +(41) Exchange Input [9]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, n#43, avg#44, m2#45, sum#46, count#47] +Arguments: hashpartitioning(w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(46) HashAggregate [codegen id : 3] +(42) HashAggregate [codegen id : 3] Input [9]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37, n#43, avg#44, m2#45, sum#46, count#47] Keys [4]: [w_warehouse_name#34, w_warehouse_sk#33, i_item_sk#32, d_moy#37] Functions [2]: [stddev_samp(cast(inv_quantity_on_hand#29 as double)), avg(inv_quantity_on_hand#29)] Aggregate Attributes [2]: [stddev_samp(cast(inv_quantity_on_hand#29 as double))#22, avg(inv_quantity_on_hand#29)#23] Results [5]: [w_warehouse_sk#33, i_item_sk#32, d_moy#37, stddev_samp(cast(inv_quantity_on_hand#29 as double))#22 AS stdev#48, avg(inv_quantity_on_hand#29)#23 AS mean#49] -(47) Filter [codegen id : 3] +(43) Filter [codegen id : 3] Input [5]: [w_warehouse_sk#33, i_item_sk#32, d_moy#37, stdev#48, mean#49] Condition : CASE WHEN (mean#49 = 0.0) THEN false ELSE ((stdev#48 / mean#49) > 1.0) END -(48) Project [codegen id : 3] +(44) Project [codegen id : 3] Output [5]: [w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, CASE WHEN (mean#49 = 0.0) THEN null ELSE (stdev#48 / mean#49) END AS cov#50] Input [5]: [w_warehouse_sk#33, i_item_sk#32, d_moy#37, stdev#48, mean#49] -(49) BroadcastExchange +(45) BroadcastExchange Input [5]: [w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50] Arguments: HashedRelationBroadcastMode(List((shiftleft(cast(input[1, int, true] as bigint), 32) | (cast(input[0, int, true] as bigint) & 4294967295))),false), [plan_id=3] -(50) BroadcastHashJoin [codegen id : 4] +(46) BroadcastHashJoin [codegen id : 4] Left keys [2]: [i_item_sk#6, w_warehouse_sk#7] Right keys [2]: [i_item_sk#32, w_warehouse_sk#33] Join type: Inner Join condition: None -(51) RowToColumnar +(47) RowToColumnar Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, cov#26, w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50] -(52) CometColumnarExchange +(48) CometColumnarExchange Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, cov#26, w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50] Arguments: rangepartitioning(w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#25 ASC NULLS FIRST, cov#26 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, mean#49 ASC NULLS FIRST, cov#50 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] -(53) CometSort +(49) CometSort Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, cov#26, w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50] Arguments: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, cov#26, w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50], [w_warehouse_sk#7 ASC NULLS FIRST, i_item_sk#6 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, mean#25 ASC NULLS FIRST, cov#26 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, mean#49 ASC NULLS FIRST, cov#50 ASC NULLS FIRST] -(54) ColumnarToRow [codegen id : 5] +(50) ColumnarToRow [codegen id : 5] Input [10]: [w_warehouse_sk#7, i_item_sk#6, d_moy#11, mean#25, cov#26, w_warehouse_sk#33, i_item_sk#32, d_moy#37, mean#49, cov#50] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (59) -+- * ColumnarToRow (58) - +- CometProject (57) - +- CometFilter (56) - +- CometScan parquet spark_catalog.default.date_dim (55) +BroadcastExchange (55) ++- * ColumnarToRow (54) + +- CometProject (53) + +- CometFilter (52) + +- CometScan parquet spark_catalog.default.date_dim (51) -(55) Scan parquet spark_catalog.default.date_dim +(51) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,1), IsNotNull(d_date_sk)] ReadSchema: struct -(56) CometFilter +(52) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Condition : ((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2001)) AND (d_moy#11 = 1)) AND isnotnull(d_date_sk#9)) -(57) CometProject +(53) CometProject Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Arguments: [d_date_sk#9, d_moy#11], [d_date_sk#9, d_moy#11] -(58) ColumnarToRow [codegen id : 1] +(54) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#9, d_moy#11] -(59) BroadcastExchange +(55) BroadcastExchange Input [2]: [d_date_sk#9, d_moy#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -Subquery:2 Hosting operator id = 27 Hosting Expression = inv_date_sk#30 IN dynamicpruning#31 -BroadcastExchange (64) -+- * ColumnarToRow (63) - +- CometProject (62) - +- CometFilter (61) - +- CometScan parquet spark_catalog.default.date_dim (60) +Subquery:2 Hosting operator id = 25 Hosting Expression = inv_date_sk#30 IN dynamicpruning#31 +BroadcastExchange (60) ++- * ColumnarToRow (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.date_dim (56) -(60) Scan parquet spark_catalog.default.date_dim +(56) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#35, d_year#36, d_moy#37] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(61) CometFilter +(57) CometFilter Input [3]: [d_date_sk#35, d_year#36, d_moy#37] Condition : ((((isnotnull(d_year#36) AND isnotnull(d_moy#37)) AND (d_year#36 = 2001)) AND (d_moy#37 = 2)) AND isnotnull(d_date_sk#35)) -(62) CometProject +(58) CometProject Input [3]: [d_date_sk#35, d_year#36, d_moy#37] Arguments: [d_date_sk#35, d_moy#37], [d_date_sk#35, d_moy#37] -(63) ColumnarToRow [codegen id : 1] +(59) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#35, d_moy#37] -(64) BroadcastExchange +(60) BroadcastExchange Input [2]: [d_date_sk#35, d_moy#37] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt index 48c1cc8b9..2a2e04bae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q39b/simplified.txt @@ -9,73 +9,69 @@ WholeStageCodegen (5) Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] Filter [mean,stdev] HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [i_item_sk] #4 - CometFilter [i_item_sk] - CometScan parquet spark_catalog.default.item [i_item_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [d_date_sk,d_moy] #6 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + WholeStageCodegen (1) + HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] + ColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [i_item_sk] #4 + CometFilter [i_item_sk] + CometScan parquet spark_catalog.default.item [i_item_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [d_date_sk,d_moy] #6 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] InputAdapter BroadcastExchange #7 WholeStageCodegen (3) Project [w_warehouse_sk,i_item_sk,d_moy,mean,stdev] Filter [mean,stdev] HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count] [stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand),stdev,mean,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] - CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] - CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - ReusedExchange [i_item_sk] #4 - ReusedExchange [w_warehouse_sk,w_warehouse_name] #5 - CometBroadcastExchange [d_date_sk,d_moy] #10 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + Exchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + WholeStageCodegen (2) + HashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,inv_quantity_on_hand] [n,avg,m2,sum,count,n,avg,m2,sum,count] + ColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] + CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] + CometProject [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,i_item_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #9 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + ReusedExchange [i_item_sk] #4 + ReusedExchange [w_warehouse_sk,w_warehouse_name] #5 + CometBroadcastExchange [d_date_sk,d_moy] #10 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt index 5bbd0d6a4..d25660dba 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/explain.txt @@ -1,122 +1,110 @@ == Physical Plan == -TakeOrderedAndProject (118) -+- * Project (117) - +- * BroadcastHashJoin Inner BuildRight (116) - :- * Project (98) - : +- * BroadcastHashJoin Inner BuildRight (97) - : :- * Project (78) - : : +- * BroadcastHashJoin Inner BuildRight (77) - : : :- * Project (59) - : : : +- * BroadcastHashJoin Inner BuildRight (58) - : : : :- * BroadcastHashJoin Inner BuildRight (39) - : : : : :- * Filter (19) - : : : : : +- * HashAggregate (18) - : : : : : +- * ColumnarToRow (17) - : : : : : +- CometColumnarExchange (16) - : : : : : +- RowToColumnar (15) - : : : : : +- * HashAggregate (14) - : : : : : +- * ColumnarToRow (13) - : : : : : +- CometProject (12) - : : : : : +- CometBroadcastHashJoin (11) - : : : : : :- CometProject (7) - : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : : : +- CometBroadcastExchange (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : : : +- CometBroadcastExchange (10) - : : : : : +- CometFilter (9) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : : : +- BroadcastExchange (38) - : : : : +- * HashAggregate (37) - : : : : +- * ColumnarToRow (36) - : : : : +- CometColumnarExchange (35) - : : : : +- RowToColumnar (34) - : : : : +- * HashAggregate (33) - : : : : +- * ColumnarToRow (32) - : : : : +- CometProject (31) - : : : : +- CometBroadcastHashJoin (30) - : : : : :- CometProject (26) - : : : : : +- CometBroadcastHashJoin (25) - : : : : : :- CometFilter (21) - : : : : : : +- CometScan parquet spark_catalog.default.customer (20) - : : : : : +- CometBroadcastExchange (24) - : : : : : +- CometFilter (23) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (22) - : : : : +- CometBroadcastExchange (29) - : : : : +- CometFilter (28) - : : : : +- CometScan parquet spark_catalog.default.date_dim (27) - : : : +- BroadcastExchange (57) - : : : +- * Filter (56) - : : : +- * HashAggregate (55) - : : : +- * ColumnarToRow (54) - : : : +- CometColumnarExchange (53) - : : : +- RowToColumnar (52) - : : : +- * HashAggregate (51) - : : : +- * ColumnarToRow (50) - : : : +- CometProject (49) - : : : +- CometBroadcastHashJoin (48) - : : : :- CometProject (46) - : : : : +- CometBroadcastHashJoin (45) - : : : : :- CometFilter (41) - : : : : : +- CometScan parquet spark_catalog.default.customer (40) - : : : : +- CometBroadcastExchange (44) - : : : : +- CometFilter (43) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (42) - : : : +- ReusedExchange (47) - : : +- BroadcastExchange (76) - : : +- * HashAggregate (75) - : : +- * ColumnarToRow (74) - : : +- CometColumnarExchange (73) - : : +- RowToColumnar (72) - : : +- * HashAggregate (71) - : : +- * ColumnarToRow (70) - : : +- CometProject (69) - : : +- CometBroadcastHashJoin (68) - : : :- CometProject (66) - : : : +- CometBroadcastHashJoin (65) - : : : :- CometFilter (61) - : : : : +- CometScan parquet spark_catalog.default.customer (60) - : : : +- CometBroadcastExchange (64) - : : : +- CometFilter (63) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (62) - : : +- ReusedExchange (67) - : +- BroadcastExchange (96) - : +- * Filter (95) - : +- * HashAggregate (94) - : +- * ColumnarToRow (93) - : +- CometColumnarExchange (92) - : +- RowToColumnar (91) - : +- * HashAggregate (90) - : +- * ColumnarToRow (89) - : +- CometProject (88) - : +- CometBroadcastHashJoin (87) - : :- CometProject (85) - : : +- CometBroadcastHashJoin (84) - : : :- CometFilter (80) - : : : +- CometScan parquet spark_catalog.default.customer (79) - : : +- CometBroadcastExchange (83) - : : +- CometFilter (82) - : : +- CometScan parquet spark_catalog.default.web_sales (81) - : +- ReusedExchange (86) - +- BroadcastExchange (115) - +- * HashAggregate (114) - +- * ColumnarToRow (113) - +- CometColumnarExchange (112) - +- RowToColumnar (111) - +- * HashAggregate (110) - +- * ColumnarToRow (109) - +- CometProject (108) - +- CometBroadcastHashJoin (107) - :- CometProject (105) - : +- CometBroadcastHashJoin (104) - : :- CometFilter (100) - : : +- CometScan parquet spark_catalog.default.customer (99) - : +- CometBroadcastExchange (103) - : +- CometFilter (102) - : +- CometScan parquet spark_catalog.default.web_sales (101) - +- ReusedExchange (106) +TakeOrderedAndProject (106) ++- * Project (105) + +- * BroadcastHashJoin Inner BuildRight (104) + :- * Project (88) + : +- * BroadcastHashJoin Inner BuildRight (87) + : :- * Project (70) + : : +- * BroadcastHashJoin Inner BuildRight (69) + : : :- * Project (53) + : : : +- * BroadcastHashJoin Inner BuildRight (52) + : : : :- * BroadcastHashJoin Inner BuildRight (35) + : : : : :- * Filter (17) + : : : : : +- * HashAggregate (16) + : : : : : +- Exchange (15) + : : : : : +- * HashAggregate (14) + : : : : : +- * ColumnarToRow (13) + : : : : : +- CometProject (12) + : : : : : +- CometBroadcastHashJoin (11) + : : : : : :- CometProject (7) + : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : : : +- CometBroadcastExchange (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan parquet spark_catalog.default.store_sales (3) + : : : : : +- CometBroadcastExchange (10) + : : : : : +- CometFilter (9) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : : : +- BroadcastExchange (34) + : : : : +- * HashAggregate (33) + : : : : +- Exchange (32) + : : : : +- * HashAggregate (31) + : : : : +- * ColumnarToRow (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (19) + : : : : : : +- CometScan parquet spark_catalog.default.customer (18) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometFilter (21) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (20) + : : : : +- CometBroadcastExchange (27) + : : : : +- CometFilter (26) + : : : : +- CometScan parquet spark_catalog.default.date_dim (25) + : : : +- BroadcastExchange (51) + : : : +- * Filter (50) + : : : +- * HashAggregate (49) + : : : +- Exchange (48) + : : : +- * HashAggregate (47) + : : : +- * ColumnarToRow (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometProject (42) + : : : : +- CometBroadcastHashJoin (41) + : : : : :- CometFilter (37) + : : : : : +- CometScan parquet spark_catalog.default.customer (36) + : : : : +- CometBroadcastExchange (40) + : : : : +- CometFilter (39) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (38) + : : : +- ReusedExchange (43) + : : +- BroadcastExchange (68) + : : +- * HashAggregate (67) + : : +- Exchange (66) + : : +- * HashAggregate (65) + : : +- * ColumnarToRow (64) + : : +- CometProject (63) + : : +- CometBroadcastHashJoin (62) + : : :- CometProject (60) + : : : +- CometBroadcastHashJoin (59) + : : : :- CometFilter (55) + : : : : +- CometScan parquet spark_catalog.default.customer (54) + : : : +- CometBroadcastExchange (58) + : : : +- CometFilter (57) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (56) + : : +- ReusedExchange (61) + : +- BroadcastExchange (86) + : +- * Filter (85) + : +- * HashAggregate (84) + : +- Exchange (83) + : +- * HashAggregate (82) + : +- * ColumnarToRow (81) + : +- CometProject (80) + : +- CometBroadcastHashJoin (79) + : :- CometProject (77) + : : +- CometBroadcastHashJoin (76) + : : :- CometFilter (72) + : : : +- CometScan parquet spark_catalog.default.customer (71) + : : +- CometBroadcastExchange (75) + : : +- CometFilter (74) + : : +- CometScan parquet spark_catalog.default.web_sales (73) + : +- ReusedExchange (78) + +- BroadcastExchange (103) + +- * HashAggregate (102) + +- Exchange (101) + +- * HashAggregate (100) + +- * ColumnarToRow (99) + +- CometProject (98) + +- CometBroadcastHashJoin (97) + :- CometProject (95) + : +- CometBroadcastHashJoin (94) + : :- CometFilter (90) + : : +- CometScan parquet spark_catalog.default.customer (89) + : +- CometBroadcastExchange (93) + : +- CometFilter (92) + : +- CometScan parquet spark_catalog.default.web_sales (91) + +- ReusedExchange (96) (1) Scan parquet spark_catalog.default.customer @@ -189,39 +177,33 @@ Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) Aggregate Attributes [2]: [sum#18, isEmpty#19] Results [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, sum#20, isEmpty#21] -(15) RowToColumnar +(15) Exchange Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, sum#20, isEmpty#21] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(16) CometColumnarExchange -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, sum#20, isEmpty#21] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(17) ColumnarToRow [codegen id : 12] -Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, sum#20, isEmpty#21] - -(18) HashAggregate [codegen id : 12] +(16) HashAggregate [codegen id : 12] Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, sum#20, isEmpty#21] Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17] Functions [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] Aggregate Attributes [1]: [sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#22] Results [2]: [c_customer_id#2 AS customer_id#23, sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))#22 AS year_total#24] -(19) Filter [codegen id : 12] +(17) Filter [codegen id : 12] Input [2]: [customer_id#23, year_total#24] Condition : (isnotnull(year_total#24) AND (year_total#24 > 0.000000)) -(20) Scan parquet spark_catalog.default.customer +(18) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(21) CometFilter +(19) CometFilter Input [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Condition : (isnotnull(c_customer_sk#25) AND isnotnull(c_customer_id#26)) -(22) Scan parquet spark_catalog.default.store_sales +(20) Scan parquet spark_catalog.default.store_sales Output [6]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] Batched: true Location: InMemoryFileIndex [] @@ -229,96 +211,90 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#38), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(23) CometFilter +(21) CometFilter Input [6]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] Condition : isnotnull(ss_customer_sk#33) -(24) CometBroadcastExchange +(22) CometBroadcastExchange Input [6]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] Arguments: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] -(25) CometBroadcastHashJoin +(23) CometBroadcastHashJoin Left output [8]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32] Right output [6]: [ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] Arguments: [c_customer_sk#25], [ss_customer_sk#33], Inner, BuildRight -(26) CometProject +(24) CometProject Input [14]: [c_customer_sk#25, c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, ss_customer_sk#33, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] Arguments: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38], [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] -(27) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#40, d_year#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [2]: [d_date_sk#40, d_year#41] Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2002)) AND isnotnull(d_date_sk#40)) -(29) CometBroadcastExchange +(27) CometBroadcastExchange Input [2]: [d_date_sk#40, d_year#41] Arguments: [d_date_sk#40, d_year#41] -(30) CometBroadcastHashJoin +(28) CometBroadcastHashJoin Left output [12]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38] Right output [2]: [d_date_sk#40, d_year#41] Arguments: [ss_sold_date_sk#38], [d_date_sk#40], Inner, BuildRight -(31) CometProject +(29) CometProject Input [14]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, ss_sold_date_sk#38, d_date_sk#40, d_year#41] Arguments: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, d_year#41], [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, d_year#41] -(32) ColumnarToRow [codegen id : 2] +(30) ColumnarToRow [codegen id : 2] Input [12]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, d_year#41] -(33) HashAggregate [codegen id : 2] +(31) HashAggregate [codegen id : 2] Input [12]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, ss_ext_discount_amt#34, ss_ext_sales_price#35, ss_ext_wholesale_cost#36, ss_ext_list_price#37, d_year#41] Keys [8]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, d_year#41] Functions [1]: [partial_sum(((((ss_ext_list_price#37 - ss_ext_wholesale_cost#36) - ss_ext_discount_amt#34) + ss_ext_sales_price#35) / 2))] Aggregate Attributes [2]: [sum#42, isEmpty#43] Results [10]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, d_year#41, sum#44, isEmpty#45] -(34) RowToColumnar +(32) Exchange Input [10]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, d_year#41, sum#44, isEmpty#45] +Arguments: hashpartitioning(c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, d_year#41, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(35) CometColumnarExchange -Input [10]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, d_year#41, sum#44, isEmpty#45] -Arguments: hashpartitioning(c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, d_year#41, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(36) ColumnarToRow [codegen id : 3] -Input [10]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, d_year#41, sum#44, isEmpty#45] - -(37) HashAggregate [codegen id : 3] +(33) HashAggregate [codegen id : 3] Input [10]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, d_year#41, sum#44, isEmpty#45] Keys [8]: [c_customer_id#26, c_first_name#27, c_last_name#28, c_preferred_cust_flag#29, c_birth_country#30, c_login#31, c_email_address#32, d_year#41] Functions [1]: [sum(((((ss_ext_list_price#37 - ss_ext_wholesale_cost#36) - ss_ext_discount_amt#34) + ss_ext_sales_price#35) / 2))] Aggregate Attributes [1]: [sum(((((ss_ext_list_price#37 - ss_ext_wholesale_cost#36) - ss_ext_discount_amt#34) + ss_ext_sales_price#35) / 2))#22] Results [8]: [c_customer_id#26 AS customer_id#46, c_first_name#27 AS customer_first_name#47, c_last_name#28 AS customer_last_name#48, c_preferred_cust_flag#29 AS customer_preferred_cust_flag#49, c_birth_country#30 AS customer_birth_country#50, c_login#31 AS customer_login#51, c_email_address#32 AS customer_email_address#52, sum(((((ss_ext_list_price#37 - ss_ext_wholesale_cost#36) - ss_ext_discount_amt#34) + ss_ext_sales_price#35) / 2))#22 AS year_total#53] -(38) BroadcastExchange +(34) BroadcastExchange Input [8]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52, year_total#53] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(39) BroadcastHashJoin [codegen id : 12] +(35) BroadcastHashJoin [codegen id : 12] Left keys [1]: [customer_id#23] Right keys [1]: [customer_id#46] Join type: Inner Join condition: None -(40) Scan parquet spark_catalog.default.customer +(36) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#54, c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(41) CometFilter +(37) CometFilter Input [8]: [c_customer_sk#54, c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61] Condition : (isnotnull(c_customer_sk#54) AND isnotnull(c_customer_id#55)) -(42) Scan parquet spark_catalog.default.catalog_sales +(38) Scan parquet spark_catalog.default.catalog_sales Output [6]: [cs_bill_customer_sk#62, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] Batched: true Location: InMemoryFileIndex [] @@ -326,92 +302,86 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#67), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(43) CometFilter +(39) CometFilter Input [6]: [cs_bill_customer_sk#62, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] Condition : isnotnull(cs_bill_customer_sk#62) -(44) CometBroadcastExchange +(40) CometBroadcastExchange Input [6]: [cs_bill_customer_sk#62, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] Arguments: [cs_bill_customer_sk#62, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] -(45) CometBroadcastHashJoin +(41) CometBroadcastHashJoin Left output [8]: [c_customer_sk#54, c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61] Right output [6]: [cs_bill_customer_sk#62, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] Arguments: [c_customer_sk#54], [cs_bill_customer_sk#62], Inner, BuildRight -(46) CometProject +(42) CometProject Input [14]: [c_customer_sk#54, c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, cs_bill_customer_sk#62, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] Arguments: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67], [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] -(47) ReusedExchange [Reuses operator id: 10] +(43) ReusedExchange [Reuses operator id: 10] Output [2]: [d_date_sk#69, d_year#70] -(48) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [12]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67] Right output [2]: [d_date_sk#69, d_year#70] Arguments: [cs_sold_date_sk#67], [d_date_sk#69], Inner, BuildRight -(49) CometProject +(45) CometProject Input [14]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, cs_sold_date_sk#67, d_date_sk#69, d_year#70] Arguments: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, d_year#70], [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, d_year#70] -(50) ColumnarToRow [codegen id : 4] +(46) ColumnarToRow [codegen id : 4] Input [12]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, d_year#70] -(51) HashAggregate [codegen id : 4] +(47) HashAggregate [codegen id : 4] Input [12]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, cs_ext_discount_amt#63, cs_ext_sales_price#64, cs_ext_wholesale_cost#65, cs_ext_list_price#66, d_year#70] Keys [8]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, d_year#70] Functions [1]: [partial_sum(((((cs_ext_list_price#66 - cs_ext_wholesale_cost#65) - cs_ext_discount_amt#63) + cs_ext_sales_price#64) / 2))] Aggregate Attributes [2]: [sum#71, isEmpty#72] Results [10]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, d_year#70, sum#73, isEmpty#74] -(52) RowToColumnar -Input [10]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, d_year#70, sum#73, isEmpty#74] - -(53) CometColumnarExchange -Input [10]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, d_year#70, sum#73, isEmpty#74] -Arguments: hashpartitioning(c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, d_year#70, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(54) ColumnarToRow [codegen id : 5] +(48) Exchange Input [10]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, d_year#70, sum#73, isEmpty#74] +Arguments: hashpartitioning(c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, d_year#70, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(55) HashAggregate [codegen id : 5] +(49) HashAggregate [codegen id : 5] Input [10]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, d_year#70, sum#73, isEmpty#74] Keys [8]: [c_customer_id#55, c_first_name#56, c_last_name#57, c_preferred_cust_flag#58, c_birth_country#59, c_login#60, c_email_address#61, d_year#70] Functions [1]: [sum(((((cs_ext_list_price#66 - cs_ext_wholesale_cost#65) - cs_ext_discount_amt#63) + cs_ext_sales_price#64) / 2))] Aggregate Attributes [1]: [sum(((((cs_ext_list_price#66 - cs_ext_wholesale_cost#65) - cs_ext_discount_amt#63) + cs_ext_sales_price#64) / 2))#75] Results [2]: [c_customer_id#55 AS customer_id#76, sum(((((cs_ext_list_price#66 - cs_ext_wholesale_cost#65) - cs_ext_discount_amt#63) + cs_ext_sales_price#64) / 2))#75 AS year_total#77] -(56) Filter [codegen id : 5] +(50) Filter [codegen id : 5] Input [2]: [customer_id#76, year_total#77] Condition : (isnotnull(year_total#77) AND (year_total#77 > 0.000000)) -(57) BroadcastExchange +(51) BroadcastExchange Input [2]: [customer_id#76, year_total#77] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(58) BroadcastHashJoin [codegen id : 12] +(52) BroadcastHashJoin [codegen id : 12] Left keys [1]: [customer_id#23] Right keys [1]: [customer_id#76] Join type: Inner Join condition: None -(59) Project [codegen id : 12] +(53) Project [codegen id : 12] Output [11]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52, year_total#53, year_total#77] Input [12]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52, year_total#53, customer_id#76, year_total#77] -(60) Scan parquet spark_catalog.default.customer +(54) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#78, c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(61) CometFilter +(55) CometFilter Input [8]: [c_customer_sk#78, c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85] Condition : (isnotnull(c_customer_sk#78) AND isnotnull(c_customer_id#79)) -(62) Scan parquet spark_catalog.default.catalog_sales +(56) Scan parquet spark_catalog.default.catalog_sales Output [6]: [cs_bill_customer_sk#86, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] Batched: true Location: InMemoryFileIndex [] @@ -419,88 +389,82 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#91), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(63) CometFilter +(57) CometFilter Input [6]: [cs_bill_customer_sk#86, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] Condition : isnotnull(cs_bill_customer_sk#86) -(64) CometBroadcastExchange +(58) CometBroadcastExchange Input [6]: [cs_bill_customer_sk#86, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] Arguments: [cs_bill_customer_sk#86, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] -(65) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [8]: [c_customer_sk#78, c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85] Right output [6]: [cs_bill_customer_sk#86, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] Arguments: [c_customer_sk#78], [cs_bill_customer_sk#86], Inner, BuildRight -(66) CometProject +(60) CometProject Input [14]: [c_customer_sk#78, c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, cs_bill_customer_sk#86, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] Arguments: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91], [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] -(67) ReusedExchange [Reuses operator id: 29] +(61) ReusedExchange [Reuses operator id: 27] Output [2]: [d_date_sk#93, d_year#94] -(68) CometBroadcastHashJoin +(62) CometBroadcastHashJoin Left output [12]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91] Right output [2]: [d_date_sk#93, d_year#94] Arguments: [cs_sold_date_sk#91], [d_date_sk#93], Inner, BuildRight -(69) CometProject +(63) CometProject Input [14]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, cs_sold_date_sk#91, d_date_sk#93, d_year#94] Arguments: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, d_year#94], [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, d_year#94] -(70) ColumnarToRow [codegen id : 6] +(64) ColumnarToRow [codegen id : 6] Input [12]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, d_year#94] -(71) HashAggregate [codegen id : 6] +(65) HashAggregate [codegen id : 6] Input [12]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, cs_ext_discount_amt#87, cs_ext_sales_price#88, cs_ext_wholesale_cost#89, cs_ext_list_price#90, d_year#94] Keys [8]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, d_year#94] Functions [1]: [partial_sum(((((cs_ext_list_price#90 - cs_ext_wholesale_cost#89) - cs_ext_discount_amt#87) + cs_ext_sales_price#88) / 2))] Aggregate Attributes [2]: [sum#95, isEmpty#96] Results [10]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, d_year#94, sum#97, isEmpty#98] -(72) RowToColumnar -Input [10]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, d_year#94, sum#97, isEmpty#98] - -(73) CometColumnarExchange +(66) Exchange Input [10]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, d_year#94, sum#97, isEmpty#98] -Arguments: hashpartitioning(c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, d_year#94, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, d_year#94, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(74) ColumnarToRow [codegen id : 7] -Input [10]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, d_year#94, sum#97, isEmpty#98] - -(75) HashAggregate [codegen id : 7] +(67) HashAggregate [codegen id : 7] Input [10]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, d_year#94, sum#97, isEmpty#98] Keys [8]: [c_customer_id#79, c_first_name#80, c_last_name#81, c_preferred_cust_flag#82, c_birth_country#83, c_login#84, c_email_address#85, d_year#94] Functions [1]: [sum(((((cs_ext_list_price#90 - cs_ext_wholesale_cost#89) - cs_ext_discount_amt#87) + cs_ext_sales_price#88) / 2))] Aggregate Attributes [1]: [sum(((((cs_ext_list_price#90 - cs_ext_wholesale_cost#89) - cs_ext_discount_amt#87) + cs_ext_sales_price#88) / 2))#75] Results [2]: [c_customer_id#79 AS customer_id#99, sum(((((cs_ext_list_price#90 - cs_ext_wholesale_cost#89) - cs_ext_discount_amt#87) + cs_ext_sales_price#88) / 2))#75 AS year_total#100] -(76) BroadcastExchange +(68) BroadcastExchange Input [2]: [customer_id#99, year_total#100] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(77) BroadcastHashJoin [codegen id : 12] +(69) BroadcastHashJoin [codegen id : 12] Left keys [1]: [customer_id#23] Right keys [1]: [customer_id#99] Join type: Inner Join condition: (CASE WHEN (year_total#77 > 0.000000) THEN (year_total#100 / year_total#77) END > CASE WHEN (year_total#24 > 0.000000) THEN (year_total#53 / year_total#24) END) -(78) Project [codegen id : 12] +(70) Project [codegen id : 12] Output [10]: [customer_id#23, customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52, year_total#77, year_total#100] Input [13]: [customer_id#23, year_total#24, customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52, year_total#53, year_total#77, customer_id#99, year_total#100] -(79) Scan parquet spark_catalog.default.customer +(71) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#101, c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(80) CometFilter +(72) CometFilter Input [8]: [c_customer_sk#101, c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108] Condition : (isnotnull(c_customer_sk#101) AND isnotnull(c_customer_id#102)) -(81) Scan parquet spark_catalog.default.web_sales +(73) Scan parquet spark_catalog.default.web_sales Output [6]: [ws_bill_customer_sk#109, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] Batched: true Location: InMemoryFileIndex [] @@ -508,92 +472,86 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#114), dynamicpruningexpression(ws_s PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(82) CometFilter +(74) CometFilter Input [6]: [ws_bill_customer_sk#109, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] Condition : isnotnull(ws_bill_customer_sk#109) -(83) CometBroadcastExchange +(75) CometBroadcastExchange Input [6]: [ws_bill_customer_sk#109, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] Arguments: [ws_bill_customer_sk#109, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] -(84) CometBroadcastHashJoin +(76) CometBroadcastHashJoin Left output [8]: [c_customer_sk#101, c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108] Right output [6]: [ws_bill_customer_sk#109, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] Arguments: [c_customer_sk#101], [ws_bill_customer_sk#109], Inner, BuildRight -(85) CometProject +(77) CometProject Input [14]: [c_customer_sk#101, c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, ws_bill_customer_sk#109, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] Arguments: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114], [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] -(86) ReusedExchange [Reuses operator id: 10] +(78) ReusedExchange [Reuses operator id: 10] Output [2]: [d_date_sk#116, d_year#117] -(87) CometBroadcastHashJoin +(79) CometBroadcastHashJoin Left output [12]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114] Right output [2]: [d_date_sk#116, d_year#117] Arguments: [ws_sold_date_sk#114], [d_date_sk#116], Inner, BuildRight -(88) CometProject +(80) CometProject Input [14]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, ws_sold_date_sk#114, d_date_sk#116, d_year#117] Arguments: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, d_year#117], [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, d_year#117] -(89) ColumnarToRow [codegen id : 8] +(81) ColumnarToRow [codegen id : 8] Input [12]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, d_year#117] -(90) HashAggregate [codegen id : 8] +(82) HashAggregate [codegen id : 8] Input [12]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, ws_ext_discount_amt#110, ws_ext_sales_price#111, ws_ext_wholesale_cost#112, ws_ext_list_price#113, d_year#117] Keys [8]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#117] Functions [1]: [partial_sum(((((ws_ext_list_price#113 - ws_ext_wholesale_cost#112) - ws_ext_discount_amt#110) + ws_ext_sales_price#111) / 2))] Aggregate Attributes [2]: [sum#118, isEmpty#119] Results [10]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#117, sum#120, isEmpty#121] -(91) RowToColumnar -Input [10]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#117, sum#120, isEmpty#121] - -(92) CometColumnarExchange +(83) Exchange Input [10]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#117, sum#120, isEmpty#121] -Arguments: hashpartitioning(c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#117, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#117, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(93) ColumnarToRow [codegen id : 9] -Input [10]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#117, sum#120, isEmpty#121] - -(94) HashAggregate [codegen id : 9] +(84) HashAggregate [codegen id : 9] Input [10]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#117, sum#120, isEmpty#121] Keys [8]: [c_customer_id#102, c_first_name#103, c_last_name#104, c_preferred_cust_flag#105, c_birth_country#106, c_login#107, c_email_address#108, d_year#117] Functions [1]: [sum(((((ws_ext_list_price#113 - ws_ext_wholesale_cost#112) - ws_ext_discount_amt#110) + ws_ext_sales_price#111) / 2))] Aggregate Attributes [1]: [sum(((((ws_ext_list_price#113 - ws_ext_wholesale_cost#112) - ws_ext_discount_amt#110) + ws_ext_sales_price#111) / 2))#122] Results [2]: [c_customer_id#102 AS customer_id#123, sum(((((ws_ext_list_price#113 - ws_ext_wholesale_cost#112) - ws_ext_discount_amt#110) + ws_ext_sales_price#111) / 2))#122 AS year_total#124] -(95) Filter [codegen id : 9] +(85) Filter [codegen id : 9] Input [2]: [customer_id#123, year_total#124] Condition : (isnotnull(year_total#124) AND (year_total#124 > 0.000000)) -(96) BroadcastExchange +(86) BroadcastExchange Input [2]: [customer_id#123, year_total#124] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=9] -(97) BroadcastHashJoin [codegen id : 12] +(87) BroadcastHashJoin [codegen id : 12] Left keys [1]: [customer_id#23] Right keys [1]: [customer_id#123] Join type: Inner Join condition: None -(98) Project [codegen id : 12] +(88) Project [codegen id : 12] Output [11]: [customer_id#23, customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52, year_total#77, year_total#100, year_total#124] Input [12]: [customer_id#23, customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52, year_total#77, year_total#100, customer_id#123, year_total#124] -(99) Scan parquet spark_catalog.default.customer +(89) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(100) CometFilter +(90) CometFilter Input [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] Condition : (isnotnull(c_customer_sk#125) AND isnotnull(c_customer_id#126)) -(101) Scan parquet spark_catalog.default.web_sales +(91) Scan parquet spark_catalog.default.web_sales Output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] Batched: true Location: InMemoryFileIndex [] @@ -601,138 +559,132 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#138), dynamicpruningexpression(ws_s PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(102) CometFilter +(92) CometFilter Input [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] Condition : isnotnull(ws_bill_customer_sk#133) -(103) CometBroadcastExchange +(93) CometBroadcastExchange Input [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] Arguments: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] -(104) CometBroadcastHashJoin +(94) CometBroadcastHashJoin Left output [8]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132] Right output [6]: [ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] Arguments: [c_customer_sk#125], [ws_bill_customer_sk#133], Inner, BuildRight -(105) CometProject +(95) CometProject Input [14]: [c_customer_sk#125, c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, ws_bill_customer_sk#133, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] Arguments: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138], [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] -(106) ReusedExchange [Reuses operator id: 29] +(96) ReusedExchange [Reuses operator id: 27] Output [2]: [d_date_sk#140, d_year#141] -(107) CometBroadcastHashJoin +(97) CometBroadcastHashJoin Left output [12]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138] Right output [2]: [d_date_sk#140, d_year#141] Arguments: [ws_sold_date_sk#138], [d_date_sk#140], Inner, BuildRight -(108) CometProject +(98) CometProject Input [14]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, ws_sold_date_sk#138, d_date_sk#140, d_year#141] Arguments: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#141], [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#141] -(109) ColumnarToRow [codegen id : 10] +(99) ColumnarToRow [codegen id : 10] Input [12]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#141] -(110) HashAggregate [codegen id : 10] +(100) HashAggregate [codegen id : 10] Input [12]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, ws_ext_discount_amt#134, ws_ext_sales_price#135, ws_ext_wholesale_cost#136, ws_ext_list_price#137, d_year#141] Keys [8]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, d_year#141] Functions [1]: [partial_sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] Aggregate Attributes [2]: [sum#142, isEmpty#143] Results [10]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, d_year#141, sum#144, isEmpty#145] -(111) RowToColumnar -Input [10]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, d_year#141, sum#144, isEmpty#145] - -(112) CometColumnarExchange -Input [10]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, d_year#141, sum#144, isEmpty#145] -Arguments: hashpartitioning(c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, d_year#141, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] - -(113) ColumnarToRow [codegen id : 11] +(101) Exchange Input [10]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, d_year#141, sum#144, isEmpty#145] +Arguments: hashpartitioning(c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, d_year#141, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(114) HashAggregate [codegen id : 11] +(102) HashAggregate [codegen id : 11] Input [10]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, d_year#141, sum#144, isEmpty#145] Keys [8]: [c_customer_id#126, c_first_name#127, c_last_name#128, c_preferred_cust_flag#129, c_birth_country#130, c_login#131, c_email_address#132, d_year#141] Functions [1]: [sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))] Aggregate Attributes [1]: [sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))#122] Results [2]: [c_customer_id#126 AS customer_id#146, sum(((((ws_ext_list_price#137 - ws_ext_wholesale_cost#136) - ws_ext_discount_amt#134) + ws_ext_sales_price#135) / 2))#122 AS year_total#147] -(115) BroadcastExchange +(103) BroadcastExchange Input [2]: [customer_id#146, year_total#147] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=11] -(116) BroadcastHashJoin [codegen id : 12] +(104) BroadcastHashJoin [codegen id : 12] Left keys [1]: [customer_id#23] Right keys [1]: [customer_id#146] Join type: Inner Join condition: (CASE WHEN (year_total#77 > 0.000000) THEN (year_total#100 / year_total#77) END > CASE WHEN (year_total#124 > 0.000000) THEN (year_total#147 / year_total#124) END) -(117) Project [codegen id : 12] +(105) Project [codegen id : 12] Output [7]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52] Input [13]: [customer_id#23, customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52, year_total#77, year_total#100, year_total#124, customer_id#146, year_total#147] -(118) TakeOrderedAndProject +(106) TakeOrderedAndProject Input [7]: [customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52] Arguments: 100, [customer_id#46 ASC NULLS FIRST, customer_first_name#47 ASC NULLS FIRST, customer_last_name#48 ASC NULLS FIRST, customer_preferred_cust_flag#49 ASC NULLS FIRST, customer_birth_country#50 ASC NULLS FIRST, customer_login#51 ASC NULLS FIRST, customer_email_address#52 ASC NULLS FIRST], [customer_id#46, customer_first_name#47, customer_last_name#48, customer_preferred_cust_flag#49, customer_birth_country#50, customer_login#51, customer_email_address#52] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#15 -BroadcastExchange (122) -+- * ColumnarToRow (121) - +- CometFilter (120) - +- CometScan parquet spark_catalog.default.date_dim (119) +BroadcastExchange (110) ++- * ColumnarToRow (109) + +- CometFilter (108) + +- CometScan parquet spark_catalog.default.date_dim (107) -(119) Scan parquet spark_catalog.default.date_dim +(107) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_year#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(120) CometFilter +(108) CometFilter Input [2]: [d_date_sk#16, d_year#17] Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(121) ColumnarToRow [codegen id : 1] +(109) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#16, d_year#17] -(122) BroadcastExchange +(110) BroadcastExchange Input [2]: [d_date_sk#16, d_year#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] -Subquery:2 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#38 IN dynamicpruning#39 -BroadcastExchange (126) -+- * ColumnarToRow (125) - +- CometFilter (124) - +- CometScan parquet spark_catalog.default.date_dim (123) +Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#38 IN dynamicpruning#39 +BroadcastExchange (114) ++- * ColumnarToRow (113) + +- CometFilter (112) + +- CometScan parquet spark_catalog.default.date_dim (111) -(123) Scan parquet spark_catalog.default.date_dim +(111) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#40, d_year#41] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(124) CometFilter +(112) CometFilter Input [2]: [d_date_sk#40, d_year#41] Condition : ((isnotnull(d_year#41) AND (d_year#41 = 2002)) AND isnotnull(d_date_sk#40)) -(125) ColumnarToRow [codegen id : 1] +(113) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#40, d_year#41] -(126) BroadcastExchange +(114) BroadcastExchange Input [2]: [d_date_sk#40, d_year#41] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=13] -Subquery:3 Hosting operator id = 42 Hosting Expression = cs_sold_date_sk#67 IN dynamicpruning#15 +Subquery:3 Hosting operator id = 38 Hosting Expression = cs_sold_date_sk#67 IN dynamicpruning#15 -Subquery:4 Hosting operator id = 62 Hosting Expression = cs_sold_date_sk#91 IN dynamicpruning#39 +Subquery:4 Hosting operator id = 56 Hosting Expression = cs_sold_date_sk#91 IN dynamicpruning#39 -Subquery:5 Hosting operator id = 81 Hosting Expression = ws_sold_date_sk#114 IN dynamicpruning#15 +Subquery:5 Hosting operator id = 73 Hosting Expression = ws_sold_date_sk#114 IN dynamicpruning#15 -Subquery:6 Hosting operator id = 101 Hosting Expression = ws_sold_date_sk#138 IN dynamicpruning#39 +Subquery:6 Hosting operator id = 91 Hosting Expression = ws_sold_date_sk#138 IN dynamicpruning#39 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt index 9f8d70dcb..a412a54d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q4/simplified.txt @@ -11,11 +11,38 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [customer_id,customer_id] Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 - RowToColumnar - WholeStageCodegen (1) + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,sum,isEmpty] + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 + WholeStageCodegen (2) HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] ColumnarToRow InputAdapter @@ -25,141 +52,102 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #8 WholeStageCodegen (1) ColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2)),customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #10 WholeStageCodegen (5) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #12 - CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + WholeStageCodegen (4) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #12 + CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_year] #4 InputAdapter BroadcastExchange #13 WholeStageCodegen (7) HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #15 - CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + WholeStageCodegen (6) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] #15 + CometFilter [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk,d_year] #9 InputAdapter BroadcastExchange #16 WholeStageCodegen (9) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #17 - RowToColumnar - WholeStageCodegen (8) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #18 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #17 + WholeStageCodegen (8) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #18 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_year] #4 InputAdapter BroadcastExchange #19 WholeStageCodegen (11) HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty] [sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2)),customer_id,year_total,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #20 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #21 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #20 + WholeStageCodegen (10) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] #21 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt index b962258c5..0f6374e8f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/explain.txt @@ -1,38 +1,36 @@ == Physical Plan == -TakeOrderedAndProject (34) -+- * HashAggregate (33) - +- * ColumnarToRow (32) - +- CometColumnarExchange (31) - +- RowToColumnar (30) - +- * HashAggregate (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometBroadcastHashJoin (26) - :- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (16) - : : +- CometBroadcastHashJoin (15) - : : :- CometProject (11) - : : : +- CometSortMergeJoin (10) - : : : :- CometSort (4) - : : : : +- CometColumnarExchange (3) - : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : +- CometSort (9) - : : : +- CometColumnarExchange (8) - : : : +- CometProject (7) - : : : +- CometFilter (6) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (5) - : : +- CometBroadcastExchange (14) - : : +- CometFilter (13) - : : +- CometScan parquet spark_catalog.default.warehouse (12) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometScan parquet spark_catalog.default.item (17) - +- CometBroadcastExchange (25) - +- CometFilter (24) - +- CometScan parquet spark_catalog.default.date_dim (23) +TakeOrderedAndProject (32) ++- * HashAggregate (31) + +- Exchange (30) + +- * HashAggregate (29) + +- * ColumnarToRow (28) + +- CometProject (27) + +- CometBroadcastHashJoin (26) + :- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometProject (16) + : : +- CometBroadcastHashJoin (15) + : : :- CometProject (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometScan parquet spark_catalog.default.catalog_returns (5) + : : +- CometBroadcastExchange (14) + : : +- CometFilter (13) + : : +- CometScan parquet spark_catalog.default.warehouse (12) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan parquet spark_catalog.default.item (17) + +- CometBroadcastExchange (25) + +- CometFilter (24) + +- CometScan parquet spark_catalog.default.date_dim (23) (1) Scan parquet spark_catalog.default.catalog_sales @@ -47,9 +45,9 @@ ReadSchema: struct= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)] Aggregate Attributes [2]: [sum(CASE WHEN (d_date#17 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26, sum(CASE WHEN (d_date#17 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)#27] Results [4]: [w_state#12, i_item_id#14, sum(CASE WHEN (d_date#17 < 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)#26 AS sales_before#28, sum(CASE WHEN (d_date#17 >= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)#27 AS sales_after#29] -(34) TakeOrderedAndProject +(32) TakeOrderedAndProject Input [4]: [w_state#12, i_item_id#14, sales_before#28, sales_after#29] Arguments: 100, [w_state#12 ASC NULLS FIRST, i_item_id#14 ASC NULLS FIRST], [w_state#12, i_item_id#14, sales_before#28, sales_after#29] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (38) -+- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.date_dim (35) +BroadcastExchange (36) ++- * ColumnarToRow (35) + +- CometFilter (34) + +- CometScan parquet spark_catalog.default.date_dim (33) -(35) Scan parquet spark_catalog.default.date_dim +(33) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_date#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-02-10), LessThanOrEqual(d_date,2000-04-10), IsNotNull(d_date_sk)] ReadSchema: struct -(36) CometFilter +(34) CometFilter Input [2]: [d_date_sk#16, d_date#17] Condition : (((isnotnull(d_date#17) AND (d_date#17 >= 2000-02-10)) AND (d_date#17 <= 2000-04-10)) AND isnotnull(d_date_sk#16)) -(37) ColumnarToRow [codegen id : 1] +(35) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#16, d_date#17] -(38) BroadcastExchange +(36) BroadcastExchange Input [2]: [d_date_sk#16, d_date#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt index d5ce3bfe8..25a80286d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q40/simplified.txt @@ -1,45 +1,43 @@ TakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] WholeStageCodegen (2) HashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sales_before,sales_after,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [w_state,i_item_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [w_state,i_item_id,d_date,cs_sales_price,cr_refunded_cash] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] - CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] - CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] - CometBroadcastHashJoin [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_warehouse_sk,w_state] - CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] - CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] - CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometColumnarExchange [cs_order_number,cs_item_sk] #2 - CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] - CometColumnarExchange [cr_order_number,cr_item_sk] #4 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] - CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_state] #5 - CometFilter [w_warehouse_sk,w_state] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 - CometProject [i_item_sk,i_item_id] - CometFilter [i_item_sk,i_item_id,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] - CometBroadcastExchange [d_date_sk,d_date] #7 - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + Exchange [w_state,i_item_id] #1 + WholeStageCodegen (1) + HashAggregate [w_state,i_item_id,d_date,cs_sales_price,cr_refunded_cash] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] + CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] + CometProject [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state] + CometBroadcastHashJoin [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_warehouse_sk,w_state] + CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] + CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] + CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #2 + CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] + CometExchange [cr_order_number,cr_item_sk] #4 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_state] #5 + CometFilter [w_warehouse_sk,w_state] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_state] + CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometProject [i_item_sk,i_item_id] + CometFilter [i_item_sk,i_item_id,i_current_price] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_current_price] + CometBroadcastExchange [d_date_sk,d_date] #7 + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt index 5c2b28b9d..c4059e5f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -13,7 +13,7 @@ +- CometProject (11) +- CometFilter (10) +- CometHashAggregate (9) - +- CometColumnarExchange (8) + +- CometExchange (8) +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) @@ -55,9 +55,9 @@ Input [1]: [i_manufact#5] Keys [1]: [i_manufact#5] Functions [1]: [partial_count(1)] -(8) CometColumnarExchange +(8) CometExchange Input [2]: [i_manufact#5, count#9] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (9) CometHashAggregate Input [2]: [i_manufact#5, count#9] @@ -90,9 +90,9 @@ Input [1]: [i_product_name#3] Keys [1]: [i_product_name#3] Functions: [] -(16) CometColumnarExchange +(16) CometExchange Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (17) CometHashAggregate Input [1]: [i_product_name#3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt index 732e8679b..007103499 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q41/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_product_name] CometHashAggregate [i_product_name] - CometColumnarExchange [i_product_name] #1 + CometExchange [i_product_name] #1 CometHashAggregate [i_product_name] CometProject [i_product_name] CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] @@ -14,7 +14,7 @@ WholeStageCodegen (1) CometProject [i_manufact] CometFilter [item_cnt,i_manufact] CometHashAggregate [item_cnt,i_manufact,count,count(1)] - CometColumnarExchange [i_manufact] #3 + CometExchange [i_manufact] #3 CometHashAggregate [i_manufact,count] CometProject [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt index 121d38dcd..a43b8716f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/explain.txt @@ -1,25 +1,23 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometColumnarExchange (18) - +- RowToColumnar (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.date_dim (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometScan parquet spark_catalog.default.store_sales (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometScan parquet spark_catalog.default.item (9) +TakeOrderedAndProject (19) ++- * HashAggregate (18) + +- Exchange (17) + +- * HashAggregate (16) + +- * ColumnarToRow (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan parquet spark_catalog.default.item (9) (1) Scan parquet spark_catalog.default.date_dim @@ -100,24 +98,18 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum#11] Results [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -(17) RowToColumnar +(17) Exchange Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] +Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(18) CometColumnarExchange -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(19) ColumnarToRow [codegen id : 2] -Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] - -(20) HashAggregate [codegen id : 2] +(18) HashAggregate [codegen id : 2] Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#12] Keys [3]: [d_year#2, i_category_id#8, i_category#9] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] Results [4]: [d_year#2, i_category_id#8, i_category#9, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS sum(ss_ext_sales_price)#14] -(21) TakeOrderedAndProject +(19) TakeOrderedAndProject Input [4]: [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] Arguments: 100, [sum(ss_ext_sales_price)#14 DESC NULLS LAST, d_year#2 ASC NULLS FIRST, i_category_id#8 ASC NULLS FIRST, i_category#9 ASC NULLS FIRST], [d_year#2, i_category_id#8, i_category#9, sum(ss_ext_sales_price)#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt index edc1cc2f5..d6e3733ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q42/simplified.txt @@ -1,25 +1,23 @@ TakeOrderedAndProject [sum(ss_ext_sales_price),d_year,i_category_id,i_category] WholeStageCodegen (2) HashAggregate [d_year,i_category_id,i_category,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(ss_ext_sales_price),sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,i_category_id,i_category] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 - CometProject [i_item_sk,i_category_id,i_category] - CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] + InputAdapter + Exchange [d_year,i_category_id,i_category] #1 + WholeStageCodegen (1) + HashAggregate [d_year,i_category_id,i_category,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_category_id,i_category] #3 + CometProject [i_item_sk,i_category_id,i_category] + CometFilter [i_item_sk,i_category_id,i_category,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_category_id,i_category,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt index a72d6cc7d..abe134d26 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/explain.txt @@ -1,25 +1,23 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometColumnarExchange (18) - +- RowToColumnar (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.date_dim (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometScan parquet spark_catalog.default.store_sales (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometScan parquet spark_catalog.default.store (9) +TakeOrderedAndProject (19) ++- * HashAggregate (18) + +- Exchange (17) + +- * HashAggregate (16) + +- * ColumnarToRow (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan parquet spark_catalog.default.store (9) (1) Scan parquet spark_catalog.default.date_dim @@ -100,24 +98,18 @@ Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) T Aggregate Attributes [7]: [sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] Results [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -(17) RowToColumnar +(17) Exchange Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] +Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(18) CometColumnarExchange -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(19) ColumnarToRow [codegen id : 2] -Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] - -(20) HashAggregate [codegen id : 2] +(18) HashAggregate [codegen id : 2] Input [9]: [s_store_name#9, s_store_id#8, sum#18, sum#19, sum#20, sum#21, sum#22, sum#23, sum#24] Keys [2]: [s_store_name#9, s_store_id#8] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31] Results [9]: [s_store_name#9, s_store_id#8, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END))#25,17,2) AS sun_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END))#26,17,2) AS mon_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END))#27,17,2) AS tue_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END))#28,17,2) AS wed_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END))#29,17,2) AS thu_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END))#30,17,2) AS fri_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))#31,17,2) AS sat_sales#38] -(21) TakeOrderedAndProject +(19) TakeOrderedAndProject Input [9]: [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] Arguments: 100, [s_store_name#9 ASC NULLS FIRST, s_store_id#8 ASC NULLS FIRST, sun_sales#32 ASC NULLS FIRST, mon_sales#33 ASC NULLS FIRST, tue_sales#34 ASC NULLS FIRST, wed_sales#35 ASC NULLS FIRST, thu_sales#36 ASC NULLS FIRST, fri_sales#37 ASC NULLS FIRST, sat_sales#38 ASC NULLS FIRST], [s_store_name#9, s_store_id#8, sun_sales#32, mon_sales#33, tue_sales#34, wed_sales#35, thu_sales#36, fri_sales#37, sat_sales#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt index 72d437599..f21d846c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q43/simplified.txt @@ -1,25 +1,23 @@ TakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] WholeStageCodegen (2) HashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_name,s_store_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] - CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] - CometProject [d_day_name,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_day_name] - CometFilter [d_date_sk,d_year,d_day_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] - CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 - CometProject [s_store_sk,s_store_id,s_store_name] - CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + InputAdapter + Exchange [s_store_name,s_store_id] #1 + WholeStageCodegen (1) + HashAggregate [s_store_name,s_store_id,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] + CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] + CometProject [d_day_name,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [d_date_sk,d_day_name,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_day_name] + CometFilter [d_date_sk,d_year,d_day_name] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_day_name] + CometBroadcastExchange [ss_store_sk,ss_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [s_store_sk,s_store_id,s_store_name] #3 + CometProject [s_store_sk,s_store_id,s_store_name] + CometFilter [s_store_sk,s_store_id,s_store_name,s_gmt_offset] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id,s_store_name,s_gmt_offset] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt index de02788f4..812f9f391 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/explain.txt @@ -1,56 +1,47 @@ == Physical Plan == -* ColumnarToRow (52) -+- CometTakeOrderedAndProject (51) - +- CometProject (50) - +- CometBroadcastHashJoin (49) - :- CometProject (47) - : +- CometBroadcastHashJoin (46) - : :- CometProject (42) - : : +- CometSortMergeJoin (41) - : : :- CometSort (23) - : : : +- CometColumnarExchange (22) - : : : +- RowToColumnar (21) - : : : +- * Project (20) - : : : +- * Filter (19) - : : : +- Window (18) - : : : +- WindowGroupLimit (17) - : : : +- * ColumnarToRow (16) - : : : +- CometSort (15) - : : : +- CometColumnarExchange (14) - : : : +- RowToColumnar (13) - : : : +- WindowGroupLimit (12) - : : : +- * Sort (11) - : : : +- * Filter (10) - : : : +- * HashAggregate (9) - : : : +- * ColumnarToRow (8) - : : : +- CometColumnarExchange (7) - : : : +- RowToColumnar (6) - : : : +- * HashAggregate (5) - : : : +- * ColumnarToRow (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometSort (40) - : : +- CometColumnarExchange (39) - : : +- RowToColumnar (38) - : : +- * Project (37) - : : +- * Filter (36) - : : +- Window (35) - : : +- WindowGroupLimit (34) - : : +- * ColumnarToRow (33) - : : +- CometSort (32) - : : +- CometColumnarExchange (31) - : : +- RowToColumnar (30) - : : +- WindowGroupLimit (29) - : : +- * Sort (28) - : : +- * Filter (27) - : : +- * HashAggregate (26) - : : +- * ColumnarToRow (25) - : : +- ReusedExchange (24) - : +- CometBroadcastExchange (45) - : +- CometFilter (44) - : +- CometScan parquet spark_catalog.default.item (43) - +- ReusedExchange (48) +TakeOrderedAndProject (43) ++- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (39) + : +- * BroadcastHashJoin Inner BuildRight (38) + : :- * Project (33) + : : +- * SortMergeJoin Inner (32) + : : :- * Sort (18) + : : : +- Exchange (17) + : : : +- * Project (16) + : : : +- * Filter (15) + : : : +- Window (14) + : : : +- WindowGroupLimit (13) + : : : +- * Sort (12) + : : : +- Exchange (11) + : : : +- WindowGroupLimit (10) + : : : +- * Sort (9) + : : : +- * Filter (8) + : : : +- * HashAggregate (7) + : : : +- Exchange (6) + : : : +- * HashAggregate (5) + : : : +- * ColumnarToRow (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- * Sort (31) + : : +- Exchange (30) + : : +- * Project (29) + : : +- * Filter (28) + : : +- Window (27) + : : +- WindowGroupLimit (26) + : : +- * Sort (25) + : : +- Exchange (24) + : : +- WindowGroupLimit (23) + : : +- * Sort (22) + : : +- * Filter (21) + : : +- * HashAggregate (20) + : : +- ReusedExchange (19) + : +- BroadcastExchange (37) + : +- * ColumnarToRow (36) + : +- CometFilter (35) + : +- CometScan parquet spark_catalog.default.item (34) + +- ReusedExchange (40) (1) Scan parquet spark_catalog.default.store_sales @@ -78,250 +69,218 @@ Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] Aggregate Attributes [2]: [sum#5, count#6] Results [3]: [ss_item_sk#1, sum#7, count#8] -(6) RowToColumnar +(6) Exchange Input [3]: [ss_item_sk#1, sum#7, count#8] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(7) CometColumnarExchange -Input [3]: [ss_item_sk#1, sum#7, count#8] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(8) ColumnarToRow [codegen id : 2] -Input [3]: [ss_item_sk#1, sum#7, count#8] - -(9) HashAggregate [codegen id : 2] +(7) HashAggregate [codegen id : 2] Input [3]: [ss_item_sk#1, sum#7, count#8] Keys [1]: [ss_item_sk#1] Functions [1]: [avg(UnscaledValue(ss_net_profit#3))] Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#3))#9] Results [2]: [ss_item_sk#1 AS item_sk#10, cast((avg(UnscaledValue(ss_net_profit#3))#9 / 100.0) as decimal(11,6)) AS rank_col#11] -(10) Filter [codegen id : 2] +(8) Filter [codegen id : 2] Input [2]: [item_sk#10, rank_col#11] Condition : (isnotnull(rank_col#11) AND (cast(rank_col#11 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#12, [id=#13]))) -(11) Sort [codegen id : 2] +(9) Sort [codegen id : 2] Input [2]: [item_sk#10, rank_col#11] Arguments: [rank_col#11 ASC NULLS FIRST], false, 0 -(12) WindowGroupLimit +(10) WindowGroupLimit Input [2]: [item_sk#10, rank_col#11] Arguments: [rank_col#11 ASC NULLS FIRST], rank(rank_col#11), 10, Partial -(13) RowToColumnar -Input [2]: [item_sk#10, rank_col#11] - -(14) CometColumnarExchange -Input [2]: [item_sk#10, rank_col#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(15) CometSort +(11) Exchange Input [2]: [item_sk#10, rank_col#11] -Arguments: [item_sk#10, rank_col#11], [rank_col#11 ASC NULLS FIRST] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(16) ColumnarToRow [codegen id : 3] +(12) Sort [codegen id : 3] Input [2]: [item_sk#10, rank_col#11] +Arguments: [rank_col#11 ASC NULLS FIRST], false, 0 -(17) WindowGroupLimit +(13) WindowGroupLimit Input [2]: [item_sk#10, rank_col#11] Arguments: [rank_col#11 ASC NULLS FIRST], rank(rank_col#11), 10, Final -(18) Window +(14) Window Input [2]: [item_sk#10, rank_col#11] Arguments: [rank(rank_col#11) windowspecdefinition(rank_col#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#14], [rank_col#11 ASC NULLS FIRST] -(19) Filter [codegen id : 4] +(15) Filter [codegen id : 4] Input [3]: [item_sk#10, rank_col#11, rnk#14] Condition : ((rnk#14 < 11) AND isnotnull(item_sk#10)) -(20) Project [codegen id : 4] +(16) Project [codegen id : 4] Output [2]: [item_sk#10, rnk#14] Input [3]: [item_sk#10, rank_col#11, rnk#14] -(21) RowToColumnar -Input [2]: [item_sk#10, rnk#14] - -(22) CometColumnarExchange +(17) Exchange Input [2]: [item_sk#10, rnk#14] -Arguments: hashpartitioning(rnk#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(rnk#14, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(23) CometSort +(18) Sort [codegen id : 5] Input [2]: [item_sk#10, rnk#14] -Arguments: [item_sk#10, rnk#14], [rnk#14 ASC NULLS FIRST] +Arguments: [rnk#14 ASC NULLS FIRST], false, 0 -(24) ReusedExchange [Reuses operator id: 7] +(19) ReusedExchange [Reuses operator id: 6] Output [3]: [ss_item_sk#15, sum#16, count#17] -(25) ColumnarToRow [codegen id : 6] -Input [3]: [ss_item_sk#15, sum#16, count#17] - -(26) HashAggregate [codegen id : 6] +(20) HashAggregate [codegen id : 7] Input [3]: [ss_item_sk#15, sum#16, count#17] Keys [1]: [ss_item_sk#15] Functions [1]: [avg(UnscaledValue(ss_net_profit#18))] Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#18))#19] Results [2]: [ss_item_sk#15 AS item_sk#20, cast((avg(UnscaledValue(ss_net_profit#18))#19 / 100.0) as decimal(11,6)) AS rank_col#21] -(27) Filter [codegen id : 6] +(21) Filter [codegen id : 7] Input [2]: [item_sk#20, rank_col#21] Condition : (isnotnull(rank_col#21) AND (cast(rank_col#21 as decimal(13,7)) > (0.9 * ReusedSubquery Subquery scalar-subquery#12, [id=#13]))) -(28) Sort [codegen id : 6] +(22) Sort [codegen id : 7] Input [2]: [item_sk#20, rank_col#21] Arguments: [rank_col#21 DESC NULLS LAST], false, 0 -(29) WindowGroupLimit +(23) WindowGroupLimit Input [2]: [item_sk#20, rank_col#21] Arguments: [rank_col#21 DESC NULLS LAST], rank(rank_col#21), 10, Partial -(30) RowToColumnar -Input [2]: [item_sk#20, rank_col#21] - -(31) CometColumnarExchange -Input [2]: [item_sk#20, rank_col#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(32) CometSort +(24) Exchange Input [2]: [item_sk#20, rank_col#21] -Arguments: [item_sk#20, rank_col#21], [rank_col#21 DESC NULLS LAST] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] -(33) ColumnarToRow [codegen id : 7] +(25) Sort [codegen id : 8] Input [2]: [item_sk#20, rank_col#21] +Arguments: [rank_col#21 DESC NULLS LAST], false, 0 -(34) WindowGroupLimit +(26) WindowGroupLimit Input [2]: [item_sk#20, rank_col#21] Arguments: [rank_col#21 DESC NULLS LAST], rank(rank_col#21), 10, Final -(35) Window +(27) Window Input [2]: [item_sk#20, rank_col#21] Arguments: [rank(rank_col#21) windowspecdefinition(rank_col#21 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rnk#22], [rank_col#21 DESC NULLS LAST] -(36) Filter [codegen id : 8] +(28) Filter [codegen id : 9] Input [3]: [item_sk#20, rank_col#21, rnk#22] Condition : ((rnk#22 < 11) AND isnotnull(item_sk#20)) -(37) Project [codegen id : 8] +(29) Project [codegen id : 9] Output [2]: [item_sk#20, rnk#22] Input [3]: [item_sk#20, rank_col#21, rnk#22] -(38) RowToColumnar +(30) Exchange Input [2]: [item_sk#20, rnk#22] +Arguments: hashpartitioning(rnk#22, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(39) CometColumnarExchange +(31) Sort [codegen id : 10] Input [2]: [item_sk#20, rnk#22] -Arguments: hashpartitioning(rnk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: [rnk#22 ASC NULLS FIRST], false, 0 -(40) CometSort -Input [2]: [item_sk#20, rnk#22] -Arguments: [item_sk#20, rnk#22], [rnk#22 ASC NULLS FIRST] - -(41) CometSortMergeJoin -Left output [2]: [item_sk#10, rnk#14] -Right output [2]: [item_sk#20, rnk#22] -Arguments: [rnk#14], [rnk#22], Inner +(32) SortMergeJoin [codegen id : 13] +Left keys [1]: [rnk#14] +Right keys [1]: [rnk#22] +Join type: Inner +Join condition: None -(42) CometProject +(33) Project [codegen id : 13] +Output [3]: [item_sk#10, rnk#14, item_sk#20] Input [4]: [item_sk#10, rnk#14, item_sk#20, rnk#22] -Arguments: [item_sk#10, rnk#14, item_sk#20], [item_sk#10, rnk#14, item_sk#20] -(43) Scan parquet spark_catalog.default.item +(34) Scan parquet spark_catalog.default.item Output [2]: [i_item_sk#23, i_product_name#24] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(44) CometFilter +(35) CometFilter Input [2]: [i_item_sk#23, i_product_name#24] Condition : isnotnull(i_item_sk#23) -(45) CometBroadcastExchange +(36) ColumnarToRow [codegen id : 11] +Input [2]: [i_item_sk#23, i_product_name#24] + +(37) BroadcastExchange Input [2]: [i_item_sk#23, i_product_name#24] -Arguments: [i_item_sk#23, i_product_name#24] +Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] -(46) CometBroadcastHashJoin -Left output [3]: [item_sk#10, rnk#14, item_sk#20] -Right output [2]: [i_item_sk#23, i_product_name#24] -Arguments: [item_sk#10], [i_item_sk#23], Inner, BuildRight +(38) BroadcastHashJoin [codegen id : 13] +Left keys [1]: [item_sk#10] +Right keys [1]: [i_item_sk#23] +Join type: Inner +Join condition: None -(47) CometProject +(39) Project [codegen id : 13] +Output [3]: [rnk#14, item_sk#20, i_product_name#24] Input [5]: [item_sk#10, rnk#14, item_sk#20, i_item_sk#23, i_product_name#24] -Arguments: [rnk#14, item_sk#20, i_product_name#24], [rnk#14, item_sk#20, i_product_name#24] -(48) ReusedExchange [Reuses operator id: 45] +(40) ReusedExchange [Reuses operator id: 37] Output [2]: [i_item_sk#25, i_product_name#26] -(49) CometBroadcastHashJoin -Left output [3]: [rnk#14, item_sk#20, i_product_name#24] -Right output [2]: [i_item_sk#25, i_product_name#26] -Arguments: [item_sk#20], [i_item_sk#25], Inner, BuildRight +(41) BroadcastHashJoin [codegen id : 13] +Left keys [1]: [item_sk#20] +Right keys [1]: [i_item_sk#25] +Join type: Inner +Join condition: None -(50) CometProject +(42) Project [codegen id : 13] +Output [3]: [rnk#14, i_product_name#24 AS best_performing#27, i_product_name#26 AS worst_performing#28] Input [5]: [rnk#14, item_sk#20, i_product_name#24, i_item_sk#25, i_product_name#26] -Arguments: [rnk#14, best_performing#27, worst_performing#28], [rnk#14, i_product_name#24 AS best_performing#27, i_product_name#26 AS worst_performing#28] - -(51) CometTakeOrderedAndProject -Input [3]: [rnk#14, best_performing#27, worst_performing#28] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[rnk#14 ASC NULLS FIRST], output=[rnk#14,best_performing#27,worst_performing#28]), [rnk#14, best_performing#27, worst_performing#28], 100, [rnk#14 ASC NULLS FIRST], [rnk#14, best_performing#27, worst_performing#28] -(52) ColumnarToRow [codegen id : 9] +(43) TakeOrderedAndProject Input [3]: [rnk#14, best_performing#27, worst_performing#28] +Arguments: 100, [rnk#14 ASC NULLS FIRST], [rnk#14, best_performing#27, worst_performing#28] ===== Subqueries ===== -Subquery:1 Hosting operator id = 10 Hosting Expression = Subquery scalar-subquery#12, [id=#13] -* HashAggregate (61) -+- * ColumnarToRow (60) - +- CometColumnarExchange (59) - +- RowToColumnar (58) - +- * HashAggregate (57) - +- * ColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan parquet spark_catalog.default.store_sales (53) +Subquery:1 Hosting operator id = 8 Hosting Expression = Subquery scalar-subquery#12, [id=#13] +* HashAggregate (50) ++- Exchange (49) + +- * HashAggregate (48) + +- * ColumnarToRow (47) + +- CometProject (46) + +- CometFilter (45) + +- CometScan parquet spark_catalog.default.store_sales (44) -(53) Scan parquet spark_catalog.default.store_sales +(44) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_store_sk), EqualTo(ss_store_sk,4), IsNull(ss_addr_sk)] ReadSchema: struct -(54) CometFilter +(45) CometFilter Input [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] Condition : ((isnotnull(ss_store_sk#30) AND (ss_store_sk#30 = 4)) AND isnull(ss_addr_sk#29)) -(55) CometProject +(46) CometProject Input [4]: [ss_addr_sk#29, ss_store_sk#30, ss_net_profit#31, ss_sold_date_sk#32] Arguments: [ss_store_sk#30, ss_net_profit#31], [ss_store_sk#30, ss_net_profit#31] -(56) ColumnarToRow [codegen id : 1] +(47) ColumnarToRow [codegen id : 1] Input [2]: [ss_store_sk#30, ss_net_profit#31] -(57) HashAggregate [codegen id : 1] +(48) HashAggregate [codegen id : 1] Input [2]: [ss_store_sk#30, ss_net_profit#31] Keys [1]: [ss_store_sk#30] Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#31))] Aggregate Attributes [2]: [sum#33, count#34] Results [3]: [ss_store_sk#30, sum#35, count#36] -(58) RowToColumnar -Input [3]: [ss_store_sk#30, sum#35, count#36] - -(59) CometColumnarExchange -Input [3]: [ss_store_sk#30, sum#35, count#36] -Arguments: hashpartitioning(ss_store_sk#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(60) ColumnarToRow [codegen id : 2] +(49) Exchange Input [3]: [ss_store_sk#30, sum#35, count#36] +Arguments: hashpartitioning(ss_store_sk#30, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(61) HashAggregate [codegen id : 2] +(50) HashAggregate [codegen id : 2] Input [3]: [ss_store_sk#30, sum#35, count#36] Keys [1]: [ss_store_sk#30] Functions [1]: [avg(UnscaledValue(ss_net_profit#31))] Aggregate Attributes [1]: [avg(UnscaledValue(ss_net_profit#31))#37] Results [1]: [cast((avg(UnscaledValue(ss_net_profit#31))#37 / 100.0) as decimal(11,6)) AS rank_col#38] -Subquery:2 Hosting operator id = 27 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] +Subquery:2 Hosting operator id = 21 Hosting Expression = ReusedSubquery Subquery scalar-subquery#12, [id=#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt index 9794c9374..12c8e0af9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q44/simplified.txt @@ -1,16 +1,16 @@ -WholeStageCodegen (9) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [rnk,best_performing,worst_performing] - CometProject [i_product_name,i_product_name] [rnk,best_performing,worst_performing] - CometBroadcastHashJoin [rnk,item_sk,i_product_name,i_item_sk,i_product_name] - CometProject [rnk,item_sk,i_product_name] - CometBroadcastHashJoin [item_sk,rnk,item_sk,i_item_sk,i_product_name] - CometProject [item_sk,rnk,item_sk] - CometSortMergeJoin [item_sk,rnk,item_sk,rnk] - CometSort [item_sk,rnk] - CometColumnarExchange [rnk] #1 - RowToColumnar +TakeOrderedAndProject [rnk,best_performing,worst_performing] + WholeStageCodegen (13) + Project [rnk,i_product_name,i_product_name] + BroadcastHashJoin [item_sk,i_item_sk] + Project [rnk,item_sk,i_product_name] + BroadcastHashJoin [item_sk,i_item_sk] + Project [item_sk,rnk,item_sk] + SortMergeJoin [rnk,rnk] + InputAdapter + WholeStageCodegen (5) + Sort [rnk] + InputAdapter + Exchange [rnk] #1 WholeStageCodegen (4) Project [item_sk,rnk] Filter [rnk,item_sk] @@ -18,66 +18,64 @@ WholeStageCodegen (9) Window [rank_col] WindowGroupLimit [rank_col] WholeStageCodegen (3) - ColumnarToRow + Sort [rank_col] InputAdapter - CometSort [item_sk,rank_col] - CometColumnarExchange #2 - RowToColumnar - WindowGroupLimit [rank_col] - WholeStageCodegen (2) - Sort [rank_col] - Filter [rank_col] - Subquery #1 - WholeStageCodegen (2) - HashAggregate [ss_store_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),rank_col,sum,count] + Exchange #2 + WindowGroupLimit [rank_col] + WholeStageCodegen (2) + Sort [rank_col] + Filter [rank_col] + Subquery #1 + WholeStageCodegen (2) + HashAggregate [ss_store_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),rank_col,sum,count] + InputAdapter + Exchange [ss_store_sk] #4 + WholeStageCodegen (1) + HashAggregate [ss_store_sk,ss_net_profit] [sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] + InputAdapter + Exchange [ss_item_sk] #3 + WholeStageCodegen (1) + HashAggregate [ss_item_sk,ss_net_profit] [sum,count,sum,count] ColumnarToRow InputAdapter - CometColumnarExchange [ss_store_sk] #4 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_store_sk,ss_net_profit] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_item_sk] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_item_sk,ss_net_profit] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_net_profit] - CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometSort [item_sk,rnk] - CometColumnarExchange [rnk] #5 - RowToColumnar - WholeStageCodegen (8) + CometProject [ss_item_sk,ss_net_profit] + CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] + InputAdapter + WholeStageCodegen (10) + Sort [rnk] + InputAdapter + Exchange [rnk] #5 + WholeStageCodegen (9) Project [item_sk,rnk] Filter [rnk,item_sk] InputAdapter Window [rank_col] WindowGroupLimit [rank_col] - WholeStageCodegen (7) - ColumnarToRow + WholeStageCodegen (8) + Sort [rank_col] InputAdapter - CometSort [item_sk,rank_col] - CometColumnarExchange #6 - RowToColumnar - WindowGroupLimit [rank_col] - WholeStageCodegen (6) - Sort [rank_col] - Filter [rank_col] - ReusedSubquery [rank_col] #1 - HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] - ColumnarToRow - InputAdapter - ReusedExchange [ss_item_sk,sum,count] #3 - CometBroadcastExchange [i_item_sk,i_product_name] #7 - CometFilter [i_item_sk,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_product_name] - ReusedExchange [i_item_sk,i_product_name] #7 + Exchange #6 + WindowGroupLimit [rank_col] + WholeStageCodegen (7) + Sort [rank_col] + Filter [rank_col] + ReusedSubquery [rank_col] #1 + HashAggregate [ss_item_sk,sum,count] [avg(UnscaledValue(ss_net_profit)),item_sk,rank_col,sum,count] + InputAdapter + ReusedExchange [ss_item_sk,sum,count] #3 + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (11) + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_product_name] + InputAdapter + ReusedExchange [i_item_sk,i_product_name] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt index 044247540..f128499e3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/explain.txt @@ -1,42 +1,40 @@ == Physical Plan == -TakeOrderedAndProject (38) -+- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometColumnarExchange (35) - +- RowToColumnar (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) - :- * ColumnarToRow (24) - : +- CometProject (23) - : +- CometBroadcastHashJoin (22) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.customer (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.customer_address (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.date_dim (13) - : +- CometBroadcastExchange (21) - : +- CometFilter (20) - : +- CometScan parquet spark_catalog.default.item (19) - +- BroadcastExchange (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.item (25) +TakeOrderedAndProject (36) ++- * HashAggregate (35) + +- Exchange (34) + +- * HashAggregate (33) + +- * Project (32) + +- * Filter (31) + +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) + :- * ColumnarToRow (24) + : +- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.customer (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (16) + : : +- CometProject (15) + : : +- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.date_dim (13) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometScan parquet spark_catalog.default.item (19) + +- BroadcastExchange (29) + +- * ColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.item (25) (1) Scan parquet spark_catalog.default.web_sales @@ -197,56 +195,50 @@ Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] Aggregate Attributes [1]: [sum#19] Results [3]: [ca_zip#11, ca_city#10, sum#20] -(34) RowToColumnar +(34) Exchange Input [3]: [ca_zip#11, ca_city#10, sum#20] +Arguments: hashpartitioning(ca_zip#11, ca_city#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(35) CometColumnarExchange -Input [3]: [ca_zip#11, ca_city#10, sum#20] -Arguments: hashpartitioning(ca_zip#11, ca_city#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(36) ColumnarToRow [codegen id : 3] -Input [3]: [ca_zip#11, ca_city#10, sum#20] - -(37) HashAggregate [codegen id : 3] +(35) HashAggregate [codegen id : 3] Input [3]: [ca_zip#11, ca_city#10, sum#20] Keys [2]: [ca_zip#11, ca_city#10] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#21] Results [3]: [ca_zip#11, ca_city#10, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#21,17,2) AS sum(ws_sales_price)#22] -(38) TakeOrderedAndProject +(36) TakeOrderedAndProject Input [3]: [ca_zip#11, ca_city#10, sum(ws_sales_price)#22] Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#10 ASC NULLS FIRST], [ca_zip#11, ca_city#10, sum(ws_sales_price)#22] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (43) -+- * ColumnarToRow (42) - +- CometProject (41) - +- CometFilter (40) - +- CometScan parquet spark_catalog.default.date_dim (39) +BroadcastExchange (41) ++- * ColumnarToRow (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan parquet spark_catalog.default.date_dim (37) -(39) Scan parquet spark_catalog.default.date_dim +(37) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(40) CometFilter +(38) CometFilter Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) -(41) CometProject +(39) CometProject Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] Arguments: [d_date_sk#12], [d_date_sk#12] -(42) ColumnarToRow [codegen id : 1] +(40) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#12] -(43) BroadcastExchange +(41) BroadcastExchange Input [1]: [d_date_sk#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt index 8550b91c6..f60fdb18a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q45/simplified.txt @@ -1,53 +1,51 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] WholeStageCodegen (3) HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_zip,ca_city] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - ColumnarToRow - InputAdapter - CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] - CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 - CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #4 - CometFilter [ca_address_sk,ca_city,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #6 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_id] - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + Exchange [ca_zip,ca_city] #1 + WholeStageCodegen (2) + HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] + Project [ws_sales_price,ca_city,ca_zip] + Filter [ca_zip,exists] + BroadcastHashJoin [i_item_id,i_item_id] + ColumnarToRow + InputAdapter + CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #4 + CometFilter [ca_address_sk,ca_city,ca_zip] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt index 549b421d7..71f97587a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/explain.txt @@ -1,48 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * HashAggregate (31) - : : +- * ColumnarToRow (30) - : : +- CometColumnarExchange (29) - : : +- RowToColumnar (28) - : : +- * HashAggregate (27) - : : +- * ColumnarToRow (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (20) - : : : +- CometBroadcastHashJoin (19) - : : : :- CometProject (14) - : : : : +- CometBroadcastHashJoin (13) - : : : : :- CometProject (8) - : : : : : +- CometBroadcastHashJoin (7) - : : : : : :- CometFilter (2) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : +- CometBroadcastExchange (6) - : : : : : +- CometProject (5) - : : : : : +- CometFilter (4) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : : +- CometBroadcastExchange (12) - : : : : +- CometProject (11) - : : : : +- CometFilter (10) - : : : : +- CometScan parquet spark_catalog.default.store (9) - : : : +- CometBroadcastExchange (18) - : : : +- CometProject (17) - : : : +- CometFilter (16) - : : : +- CometScan parquet spark_catalog.default.household_demographics (15) - : : +- CometBroadcastExchange (23) - : : +- CometFilter (22) - : : +- CometScan parquet spark_catalog.default.customer_address (21) - : +- BroadcastExchange (35) - : +- * ColumnarToRow (34) - : +- CometFilter (33) - : +- CometScan parquet spark_catalog.default.customer (32) - +- BroadcastExchange (41) - +- * ColumnarToRow (40) - +- CometFilter (39) - +- CometScan parquet spark_catalog.default.customer_address (38) +TakeOrderedAndProject (42) ++- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * HashAggregate (29) + : : +- Exchange (28) + : : +- * HashAggregate (27) + : : +- * ColumnarToRow (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (20) + : : : +- CometBroadcastHashJoin (19) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan parquet spark_catalog.default.store (9) + : : : +- CometBroadcastExchange (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) + : : : +- CometScan parquet spark_catalog.default.household_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometFilter (22) + : : +- CometScan parquet spark_catalog.default.customer_address (21) + : +- BroadcastExchange (33) + : +- * ColumnarToRow (32) + : +- CometFilter (31) + : +- CometScan parquet spark_catalog.default.customer (30) + +- BroadcastExchange (39) + +- * ColumnarToRow (38) + +- CometFilter (37) + +- CometScan parquet spark_catalog.default.customer_address (36) (1) Scan parquet spark_catalog.default.store_sales @@ -175,112 +173,106 @@ Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(Unscale Aggregate Attributes [2]: [sum#20, sum#21] Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#22, sum#23] -(28) RowToColumnar +(28) Exchange Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#22, sum#23] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(29) CometColumnarExchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#22, sum#23] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(30) ColumnarToRow [codegen id : 4] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#22, sum#23] - -(31) HashAggregate [codegen id : 4] +(29) HashAggregate [codegen id : 4] Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#22, sum#23] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#24, sum(UnscaledValue(ss_net_profit#7))#25] Results [5]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#19 AS bought_city#26, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#24,17,2) AS amt#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#25,17,2) AS profit#28] -(32) Scan parquet spark_catalog.default.customer +(30) Scan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(33) CometFilter +(31) CometFilter Input [4]: [c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Condition : (isnotnull(c_customer_sk#29) AND isnotnull(c_current_addr_sk#30)) -(34) ColumnarToRow [codegen id : 2] +(32) ColumnarToRow [codegen id : 2] Input [4]: [c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] -(35) BroadcastExchange +(33) BroadcastExchange Input [4]: [c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] -(36) BroadcastHashJoin [codegen id : 4] +(34) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_customer_sk#1] Right keys [1]: [c_customer_sk#29] Join type: Inner Join condition: None -(37) Project [codegen id : 4] +(35) Project [codegen id : 4] Output [7]: [ss_ticket_number#5, bought_city#26, amt#27, profit#28, c_current_addr_sk#30, c_first_name#31, c_last_name#32] Input [9]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#26, amt#27, profit#28, c_customer_sk#29, c_current_addr_sk#30, c_first_name#31, c_last_name#32] -(38) Scan parquet spark_catalog.default.customer_address +(36) Scan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#33, ca_city#34] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(39) CometFilter +(37) CometFilter Input [2]: [ca_address_sk#33, ca_city#34] Condition : (isnotnull(ca_address_sk#33) AND isnotnull(ca_city#34)) -(40) ColumnarToRow [codegen id : 3] +(38) ColumnarToRow [codegen id : 3] Input [2]: [ca_address_sk#33, ca_city#34] -(41) BroadcastExchange +(39) BroadcastExchange Input [2]: [ca_address_sk#33, ca_city#34] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(42) BroadcastHashJoin [codegen id : 4] +(40) BroadcastHashJoin [codegen id : 4] Left keys [1]: [c_current_addr_sk#30] Right keys [1]: [ca_address_sk#33] Join type: Inner Join condition: NOT (ca_city#34 = bought_city#26) -(43) Project [codegen id : 4] +(41) Project [codegen id : 4] Output [7]: [c_last_name#32, c_first_name#31, ca_city#34, bought_city#26, ss_ticket_number#5, amt#27, profit#28] Input [9]: [ss_ticket_number#5, bought_city#26, amt#27, profit#28, c_current_addr_sk#30, c_first_name#31, c_last_name#32, ca_address_sk#33, ca_city#34] -(44) TakeOrderedAndProject +(42) TakeOrderedAndProject Input [7]: [c_last_name#32, c_first_name#31, ca_city#34, bought_city#26, ss_ticket_number#5, amt#27, profit#28] Arguments: 100, [c_last_name#32 ASC NULLS FIRST, c_first_name#31 ASC NULLS FIRST, ca_city#34 ASC NULLS FIRST, bought_city#26 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#32, c_first_name#31, ca_city#34, bought_city#26, ss_ticket_number#5, amt#27, profit#28] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (49) -+- * ColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan parquet spark_catalog.default.date_dim (45) +BroadcastExchange (47) ++- * ColumnarToRow (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan parquet spark_catalog.default.date_dim (43) -(45) Scan parquet spark_catalog.default.date_dim +(43) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_year#11, d_dow#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_dow, [0,6]), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(46) CometFilter +(44) CometFilter Input [3]: [d_date_sk#10, d_year#11, d_dow#12] Condition : ((d_dow#12 IN (6,0) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) -(47) CometProject +(45) CometProject Input [3]: [d_date_sk#10, d_year#11, d_dow#12] Arguments: [d_date_sk#10], [d_date_sk#10] -(48) ColumnarToRow [codegen id : 1] +(46) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#10] -(49) BroadcastExchange +(47) BroadcastExchange Input [1]: [d_date_sk#10] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt index 794e4979c..9ce3a98eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q46/simplified.txt @@ -5,47 +5,45 @@ TakeOrderedAndProject [c_last_name,c_first_name,ca_city,bought_city,ss_ticket_nu Project [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] BroadcastHashJoin [ss_customer_sk,c_customer_sk] HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),bought_city,amt,profit,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] - CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_city] - CometScan parquet spark_catalog.default.store [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #5 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #6 - CometFilter [ca_address_sk,ca_city] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + InputAdapter + Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + WholeStageCodegen (1) + HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] + CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_city] + CometScan parquet spark_catalog.default.store [s_store_sk,s_city] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [ca_address_sk,ca_city] #6 + CometFilter [ca_address_sk,ca_city] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] InputAdapter BroadcastExchange #7 WholeStageCodegen (2) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt index 355f13a9a..12f171f31 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/explain.txt @@ -1,57 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (53) -+- * Project (52) - +- * BroadcastHashJoin Inner BuildRight (51) - :- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (32) - : : +- * Filter (31) - : : +- Window (30) - : : +- * Filter (29) - : : +- Window (28) - : : +- * ColumnarToRow (27) - : : +- CometSort (26) - : : +- CometColumnarExchange (25) - : : +- RowToColumnar (24) - : : +- * HashAggregate (23) - : : +- * ColumnarToRow (22) - : : +- CometColumnarExchange (21) - : : +- RowToColumnar (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.item (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.store (13) - : +- BroadcastExchange (42) - : +- * Project (41) - : +- Window (40) - : +- * ColumnarToRow (39) - : +- CometSort (38) - : +- CometColumnarExchange (37) - : +- RowToColumnar (36) - : +- * HashAggregate (35) - : +- * ColumnarToRow (34) - : +- ReusedExchange (33) - +- BroadcastExchange (50) - +- * Project (49) - +- Window (48) - +- * ColumnarToRow (47) - +- CometSort (46) - +- ReusedExchange (45) +TakeOrderedAndProject (45) ++- * Project (44) + +- * BroadcastHashJoin Inner BuildRight (43) + :- * Project (37) + : +- * BroadcastHashJoin Inner BuildRight (36) + : :- * Project (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * Filter (25) + : : +- Window (24) + : : +- * Sort (23) + : : +- Exchange (22) + : : +- * HashAggregate (21) + : : +- Exchange (20) + : : +- * HashAggregate (19) + : : +- * ColumnarToRow (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.store (13) + : +- BroadcastExchange (35) + : +- * Project (34) + : +- Window (33) + : +- * Sort (32) + : +- Exchange (31) + : +- * HashAggregate (30) + : +- ReusedExchange (29) + +- BroadcastExchange (42) + +- * Project (41) + +- Window (40) + +- * Sort (39) + +- ReusedExchange (38) (1) Scan parquet spark_catalog.default.item @@ -148,166 +140,142 @@ Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] Aggregate Attributes [1]: [sum#15] Results [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#16] -(20) RowToColumnar +(20) Exchange Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#16] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(21) CometColumnarExchange -Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(22) ColumnarToRow [codegen id : 2] -Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#16] - -(23) HashAggregate [codegen id : 2] +(21) HashAggregate [codegen id : 2] Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#16] Keys [6]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#17] Results [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#17,17,2) AS _w0#19] -(24) RowToColumnar -Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19] - -(25) CometColumnarExchange +(22) Exchange Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(26) CometSort +(23) Sort [codegen id : 3] Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19] -Arguments: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST] +Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST], false, 0 -(27) ColumnarToRow [codegen id : 3] -Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19] - -(28) Window +(24) Window Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19] Arguments: [rank(d_year#10, d_moy#11) windowspecdefinition(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#3, i_brand#2, s_store_name#13, s_company_name#14], [d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST] -(29) Filter [codegen id : 4] +(25) Filter [codegen id : 4] Input [9]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19, rn#20] Condition : (isnotnull(d_year#10) AND (d_year#10 = 1999)) -(30) Window +(26) Window Input [9]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19, rn#20] Arguments: [avg(_w0#19) windowspecdefinition(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10] -(31) Filter [codegen id : 13] +(27) Filter [codegen id : 13] Input [10]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) -(32) Project [codegen id : 13] +(28) Project [codegen id : 13] Output [9]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, avg_monthly_sales#21, rn#20] Input [10]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] -(33) ReusedExchange [Reuses operator id: 21] +(29) ReusedExchange [Reuses operator id: 20] Output [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] -(34) ColumnarToRow [codegen id : 6] -Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] - -(35) HashAggregate [codegen id : 6] +(30) HashAggregate [codegen id : 6] Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] Keys [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27] Functions [1]: [sum(UnscaledValue(ss_sales_price#29))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#29))#17] Results [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#29))#17,17,2) AS sum_sales#30] -(36) RowToColumnar -Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] - -(37) CometColumnarExchange +(31) Exchange Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] -Arguments: hashpartitioning(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(38) CometSort +(32) Sort [codegen id : 7] Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] -Arguments: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30], [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, s_store_name#24 ASC NULLS FIRST, s_company_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] +Arguments: [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, s_store_name#24 ASC NULLS FIRST, s_company_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST], false, 0 -(39) ColumnarToRow [codegen id : 7] -Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] - -(40) Window +(33) Window Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] Arguments: [rank(d_year#26, d_moy#27) windowspecdefinition(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#31], [i_category#22, i_brand#23, s_store_name#24, s_company_name#25], [d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] -(41) Project [codegen id : 8] +(34) Project [codegen id : 8] Output [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] Input [8]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30, rn#31] -(42) BroadcastExchange +(35) BroadcastExchange Input [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(43) BroadcastHashJoin [codegen id : 13] +(36) BroadcastHashJoin [codegen id : 13] Left keys [5]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, rn#20] Right keys [5]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, (rn#31 + 1)] Join type: Inner Join condition: None -(44) Project [codegen id : 13] +(37) Project [codegen id : 13] Output [10]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30] Input [15]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] -(45) ReusedExchange [Reuses operator id: 37] +(38) ReusedExchange [Reuses operator id: 31] Output [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] -(46) CometSort +(39) Sort [codegen id : 11] Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] -Arguments: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38], [i_category#32 ASC NULLS FIRST, i_brand#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST, s_company_name#35 ASC NULLS FIRST, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] +Arguments: [i_category#32 ASC NULLS FIRST, i_brand#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST, s_company_name#35 ASC NULLS FIRST, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST], false, 0 -(47) ColumnarToRow [codegen id : 11] -Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] - -(48) Window +(40) Window Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] Arguments: [rank(d_year#36, d_moy#37) windowspecdefinition(i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#32, i_brand#33, s_store_name#34, s_company_name#35], [d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] -(49) Project [codegen id : 12] +(41) Project [codegen id : 12] Output [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] Input [8]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38, rn#39] -(50) BroadcastExchange +(42) BroadcastExchange Input [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(51) BroadcastHashJoin [codegen id : 13] +(43) BroadcastHashJoin [codegen id : 13] Left keys [5]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, rn#20] Right keys [5]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, (rn#39 - 1)] Join type: Inner Join condition: None -(52) Project [codegen id : 13] +(44) Project [codegen id : 13] Output [10]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, avg_monthly_sales#21, sum_sales#18, sum_sales#30 AS psum#40, sum_sales#38 AS nsum#41] Input [16]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30, i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] -(53) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [10]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST], [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (57) -+- * ColumnarToRow (56) - +- CometFilter (55) - +- CometScan parquet spark_catalog.default.date_dim (54) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(54) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(55) CometFilter +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) -(56) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -(57) BroadcastExchange +(49) BroadcastExchange Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt index 4c9474631..1db8e7528 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q47/simplified.txt @@ -13,45 +13,41 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (3) - ColumnarToRow + Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 - CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + Exchange [i_category,i_brand,s_store_name,s_company_name] #1 + WholeStageCodegen (2) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + InputAdapter + Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometFilter [i_item_sk,i_brand,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) @@ -59,16 +55,13 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (7) - ColumnarToRow + Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #8 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 + Exchange [i_category,i_brand,s_store_name,s_company_name] #8 + WholeStageCodegen (6) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] + InputAdapter + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #9 WholeStageCodegen (12) @@ -76,7 +69,6 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (11) - ColumnarToRow + Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt index 9b30bfe48..5197d99f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/explain.txt @@ -1,34 +1,32 @@ == Physical Plan == -* HashAggregate (30) -+- * ColumnarToRow (29) - +- CometColumnarExchange (28) - +- RowToColumnar (27) - +- * HashAggregate (26) - +- * ColumnarToRow (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.store (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.customer_demographics (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.customer_address (13) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometScan parquet spark_catalog.default.date_dim (19) +* HashAggregate (28) ++- Exchange (27) + +- * HashAggregate (26) + +- * ColumnarToRow (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.store (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.customer_demographics (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometScan parquet spark_catalog.default.customer_address (13) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan parquet spark_catalog.default.date_dim (19) (1) Scan parquet spark_catalog.default.store_sales @@ -157,17 +155,11 @@ Functions [1]: [partial_sum(ss_quantity#4)] Aggregate Attributes [1]: [sum#18] Results [1]: [sum#19] -(27) RowToColumnar +(27) Exchange Input [1]: [sum#19] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] -(28) CometColumnarExchange -Input [1]: [sum#19] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(29) ColumnarToRow [codegen id : 2] -Input [1]: [sum#19] - -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [1]: [sum#19] Keys: [] Functions [1]: [sum(ss_quantity#4)] @@ -177,32 +169,32 @@ Results [1]: [sum(ss_quantity#4)#20 AS sum(ss_quantity)#21] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (35) -+- * ColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan parquet spark_catalog.default.date_dim (31) +BroadcastExchange (33) ++- * ColumnarToRow (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan parquet spark_catalog.default.date_dim (29) -(31) Scan parquet spark_catalog.default.date_dim +(29) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#16, d_year#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(32) CometFilter +(30) CometFilter Input [2]: [d_date_sk#16, d_year#17] Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(33) CometProject +(31) CometProject Input [2]: [d_date_sk#16, d_year#17] Arguments: [d_date_sk#16], [d_date_sk#16] -(34) ColumnarToRow [codegen id : 1] +(32) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#16] -(35) BroadcastExchange +(33) BroadcastExchange Input [1]: [d_date_sk#16] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt index e1595f3e7..23e083535 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q48/simplified.txt @@ -1,42 +1,40 @@ WholeStageCodegen (2) HashAggregate [sum] [sum(ss_quantity),sum(ss_quantity),sum] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_quantity] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_quantity] - CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] - CometProject [ss_quantity,ss_sold_date_sk] - CometBroadcastHashJoin [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] - CometProject [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] - CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk] #3 - CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #5 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + Exchange #1 + WholeStageCodegen (1) + HashAggregate [ss_quantity] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_quantity] + CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] + CometProject [ss_quantity,ss_sold_date_sk] + CometBroadcastHashJoin [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk,ca_address_sk,ca_state] + CometProject [ss_addr_sk,ss_quantity,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ss_cdemo_sk,ss_addr_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometBroadcastHashJoin [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk,s_store_sk] + CometFilter [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_cdemo_sk,ss_addr_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk] #3 + CometFilter [s_store_sk] + CometScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #4 + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [ca_address_sk,ca_state] #5 + CometProject [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt index f8e0c93d7..124a22ef9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/explain.txt @@ -1,95 +1,81 @@ == Physical Plan == -* ColumnarToRow (91) -+- CometTakeOrderedAndProject (90) - +- CometHashAggregate (89) - +- CometColumnarExchange (88) - +- RowToColumnar (87) - +- * HashAggregate (86) - +- Union (85) - :- * Project (30) - : +- * Filter (29) - : +- Window (28) - : +- * Sort (27) - : +- Window (26) - : +- * ColumnarToRow (25) - : +- CometSort (24) - : +- CometColumnarExchange (23) - : +- RowToColumnar (22) - : +- * HashAggregate (21) - : +- * ColumnarToRow (20) - : +- CometColumnarExchange (19) - : +- RowToColumnar (18) - : +- * HashAggregate (17) - : +- * ColumnarToRow (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometScan parquet spark_catalog.default.web_returns (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.date_dim (10) - :- * Project (57) - : +- * Filter (56) - : +- Window (55) - : +- * Sort (54) - : +- Window (53) - : +- * ColumnarToRow (52) - : +- CometSort (51) - : +- CometColumnarExchange (50) - : +- RowToColumnar (49) - : +- * HashAggregate (48) - : +- * ColumnarToRow (47) - : +- CometColumnarExchange (46) - : +- RowToColumnar (45) - : +- * HashAggregate (44) - : +- * ColumnarToRow (43) - : +- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (39) - : : +- CometBroadcastHashJoin (38) - : : :- CometBroadcastExchange (34) - : : : +- CometProject (33) - : : : +- CometFilter (32) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (31) - : : +- CometProject (37) - : : +- CometFilter (36) - : : +- CometScan parquet spark_catalog.default.catalog_returns (35) - : +- ReusedExchange (40) - +- * Project (84) - +- * Filter (83) - +- Window (82) - +- * Sort (81) - +- Window (80) - +- * ColumnarToRow (79) - +- CometSort (78) - +- CometColumnarExchange (77) - +- RowToColumnar (76) - +- * HashAggregate (75) - +- * ColumnarToRow (74) - +- CometColumnarExchange (73) - +- RowToColumnar (72) - +- * HashAggregate (71) - +- * ColumnarToRow (70) - +- CometProject (69) - +- CometBroadcastHashJoin (68) - :- CometProject (66) - : +- CometBroadcastHashJoin (65) - : :- CometBroadcastExchange (61) - : : +- CometProject (60) - : : +- CometFilter (59) - : : +- CometScan parquet spark_catalog.default.store_sales (58) - : +- CometProject (64) - : +- CometFilter (63) - : +- CometScan parquet spark_catalog.default.store_returns (62) - +- ReusedExchange (67) +TakeOrderedAndProject (77) ++- * HashAggregate (76) + +- Exchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * Sort (21) + : +- Exchange (20) + : +- * HashAggregate (19) + : +- Exchange (18) + : +- * HashAggregate (17) + : +- * ColumnarToRow (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * Sort (44) + : +- Exchange (43) + : +- * HashAggregate (42) + : +- Exchange (41) + : +- * HashAggregate (40) + : +- * ColumnarToRow (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * Sort (67) + +- Exchange (66) + +- * HashAggregate (65) + +- Exchange (64) + +- * HashAggregate (63) + +- * ColumnarToRow (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) (1) Scan parquet spark_catalog.default.web_sales @@ -174,58 +160,46 @@ Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coa Aggregate Attributes [6]: [sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] Results [7]: [ws_item_sk#1, sum#22, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -(18) RowToColumnar +(18) Exchange Input [7]: [ws_item_sk#1, sum#22, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(19) CometColumnarExchange -Input [7]: [ws_item_sk#1, sum#22, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(20) ColumnarToRow [codegen id : 2] -Input [7]: [ws_item_sk#1, sum#22, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] - -(21) HashAggregate [codegen id : 2] +(19) HashAggregate [codegen id : 2] Input [7]: [ws_item_sk#1, sum#22, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#10, 0))#28, sum(coalesce(ws_quantity#3, 0))#29, sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00))#30, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#31] Results [3]: [ws_item_sk#1 AS item#32, (cast(sum(coalesce(wr_return_quantity#10, 0))#28 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#29 as decimal(15,4))) AS return_ratio#33, (cast(sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00))#30 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#31 as decimal(15,4))) AS currency_ratio#34] -(22) RowToColumnar +(20) Exchange Input [3]: [item#32, return_ratio#33, currency_ratio#34] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(23) CometColumnarExchange +(21) Sort [codegen id : 3] Input [3]: [item#32, return_ratio#33, currency_ratio#34] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: [return_ratio#33 ASC NULLS FIRST], false, 0 -(24) CometSort -Input [3]: [item#32, return_ratio#33, currency_ratio#34] -Arguments: [item#32, return_ratio#33, currency_ratio#34], [return_ratio#33 ASC NULLS FIRST] - -(25) ColumnarToRow [codegen id : 3] -Input [3]: [item#32, return_ratio#33, currency_ratio#34] - -(26) Window +(22) Window Input [3]: [item#32, return_ratio#33, currency_ratio#34] Arguments: [rank(return_ratio#33) windowspecdefinition(return_ratio#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#35], [return_ratio#33 ASC NULLS FIRST] -(27) Sort [codegen id : 4] +(23) Sort [codegen id : 4] Input [4]: [item#32, return_ratio#33, currency_ratio#34, return_rank#35] Arguments: [currency_ratio#34 ASC NULLS FIRST], false, 0 -(28) Window +(24) Window Input [4]: [item#32, return_ratio#33, currency_ratio#34, return_rank#35] Arguments: [rank(currency_ratio#34) windowspecdefinition(currency_ratio#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#36], [currency_ratio#34 ASC NULLS FIRST] -(29) Filter [codegen id : 5] +(25) Filter [codegen id : 5] Input [5]: [item#32, return_ratio#33, currency_ratio#34, return_rank#35, currency_rank#36] Condition : ((return_rank#35 <= 10) OR (currency_rank#36 <= 10)) -(30) Project [codegen id : 5] +(26) Project [codegen id : 5] Output [5]: [web AS channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Input [5]: [item#32, return_ratio#33, currency_ratio#34, return_rank#35, currency_rank#36] -(31) Scan parquet spark_catalog.default.catalog_sales +(27) Scan parquet spark_catalog.default.catalog_sales Output [6]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_net_profit#42, cs_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] @@ -233,116 +207,104 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct -(32) CometFilter +(28) CometFilter Input [6]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_net_profit#42, cs_sold_date_sk#43] Condition : (((((((isnotnull(cs_net_profit#42) AND isnotnull(cs_net_paid#41)) AND isnotnull(cs_quantity#40)) AND (cs_net_profit#42 > 1.00)) AND (cs_net_paid#41 > 0.00)) AND (cs_quantity#40 > 0)) AND isnotnull(cs_order_number#39)) AND isnotnull(cs_item_sk#38)) -(33) CometProject +(29) CometProject Input [6]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_net_profit#42, cs_sold_date_sk#43] Arguments: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43], [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43] -(34) CometBroadcastExchange +(30) CometBroadcastExchange Input [5]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43] Arguments: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43] -(35) Scan parquet spark_catalog.default.catalog_returns +(31) Scan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(36) CometFilter +(32) CometFilter Input [5]: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48, cr_returned_date_sk#49] Condition : (((isnotnull(cr_return_amount#48) AND (cr_return_amount#48 > 10000.00)) AND isnotnull(cr_order_number#46)) AND isnotnull(cr_item_sk#45)) -(37) CometProject +(33) CometProject Input [5]: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48, cr_returned_date_sk#49] Arguments: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48], [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48] -(38) CometBroadcastHashJoin +(34) CometBroadcastHashJoin Left output [5]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43] Right output [4]: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48] Arguments: [cs_order_number#39, cs_item_sk#38], [cr_order_number#46, cr_item_sk#45], Inner, BuildLeft -(39) CometProject +(35) CometProject Input [9]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43, cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48] Arguments: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43, cr_return_quantity#47, cr_return_amount#48], [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43, cr_return_quantity#47, cr_return_amount#48] -(40) ReusedExchange [Reuses operator id: 13] +(36) ReusedExchange [Reuses operator id: 13] Output [1]: [d_date_sk#50] -(41) CometBroadcastHashJoin +(37) CometBroadcastHashJoin Left output [6]: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43, cr_return_quantity#47, cr_return_amount#48] Right output [1]: [d_date_sk#50] Arguments: [cs_sold_date_sk#43], [d_date_sk#50], Inner, BuildRight -(42) CometProject +(38) CometProject Input [7]: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43, cr_return_quantity#47, cr_return_amount#48, d_date_sk#50] Arguments: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cr_return_quantity#47, cr_return_amount#48], [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cr_return_quantity#47, cr_return_amount#48] -(43) ColumnarToRow [codegen id : 6] +(39) ColumnarToRow [codegen id : 6] Input [5]: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cr_return_quantity#47, cr_return_amount#48] -(44) HashAggregate [codegen id : 6] +(40) HashAggregate [codegen id : 6] Input [5]: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cr_return_quantity#47, cr_return_amount#48] Keys [1]: [cs_item_sk#38] Functions [4]: [partial_sum(coalesce(cr_return_quantity#47, 0)), partial_sum(coalesce(cs_quantity#40, 0)), partial_sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#41 as decimal(12,2)), 0.00))] Aggregate Attributes [6]: [sum#51, sum#52, sum#53, isEmpty#54, sum#55, isEmpty#56] Results [7]: [cs_item_sk#38, sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] -(45) RowToColumnar -Input [7]: [cs_item_sk#38, sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] - -(46) CometColumnarExchange -Input [7]: [cs_item_sk#38, sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] -Arguments: hashpartitioning(cs_item_sk#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(47) ColumnarToRow [codegen id : 7] +(41) Exchange Input [7]: [cs_item_sk#38, sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] +Arguments: hashpartitioning(cs_item_sk#38, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(48) HashAggregate [codegen id : 7] +(42) HashAggregate [codegen id : 7] Input [7]: [cs_item_sk#38, sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] Keys [1]: [cs_item_sk#38] Functions [4]: [sum(coalesce(cr_return_quantity#47, 0)), sum(coalesce(cs_quantity#40, 0)), sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#41 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#47, 0))#63, sum(coalesce(cs_quantity#40, 0))#64, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#65, sum(coalesce(cast(cs_net_paid#41 as decimal(12,2)), 0.00))#66] Results [3]: [cs_item_sk#38 AS item#67, (cast(sum(coalesce(cr_return_quantity#47, 0))#63 as decimal(15,4)) / cast(sum(coalesce(cs_quantity#40, 0))#64 as decimal(15,4))) AS return_ratio#68, (cast(sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#65 as decimal(15,4)) / cast(sum(coalesce(cast(cs_net_paid#41 as decimal(12,2)), 0.00))#66 as decimal(15,4))) AS currency_ratio#69] -(49) RowToColumnar -Input [3]: [item#67, return_ratio#68, currency_ratio#69] - -(50) CometColumnarExchange -Input [3]: [item#67, return_ratio#68, currency_ratio#69] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(51) CometSort +(43) Exchange Input [3]: [item#67, return_ratio#68, currency_ratio#69] -Arguments: [item#67, return_ratio#68, currency_ratio#69], [return_ratio#68 ASC NULLS FIRST] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] -(52) ColumnarToRow [codegen id : 8] +(44) Sort [codegen id : 8] Input [3]: [item#67, return_ratio#68, currency_ratio#69] +Arguments: [return_ratio#68 ASC NULLS FIRST], false, 0 -(53) Window +(45) Window Input [3]: [item#67, return_ratio#68, currency_ratio#69] Arguments: [rank(return_ratio#68) windowspecdefinition(return_ratio#68 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#70], [return_ratio#68 ASC NULLS FIRST] -(54) Sort [codegen id : 9] +(46) Sort [codegen id : 9] Input [4]: [item#67, return_ratio#68, currency_ratio#69, return_rank#70] Arguments: [currency_ratio#69 ASC NULLS FIRST], false, 0 -(55) Window +(47) Window Input [4]: [item#67, return_ratio#68, currency_ratio#69, return_rank#70] Arguments: [rank(currency_ratio#69) windowspecdefinition(currency_ratio#69 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#71], [currency_ratio#69 ASC NULLS FIRST] -(56) Filter [codegen id : 10] +(48) Filter [codegen id : 10] Input [5]: [item#67, return_ratio#68, currency_ratio#69, return_rank#70, currency_rank#71] Condition : ((return_rank#70 <= 10) OR (currency_rank#71 <= 10)) -(57) Project [codegen id : 10] +(49) Project [codegen id : 10] Output [5]: [catalog AS channel#72, item#67, return_ratio#68, return_rank#70, currency_rank#71] Input [5]: [item#67, return_ratio#68, currency_ratio#69, return_rank#70, currency_rank#71] -(58) Scan parquet spark_catalog.default.store_sales +(50) Scan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_net_profit#77, ss_sold_date_sk#78] Batched: true Location: InMemoryFileIndex [] @@ -350,177 +312,161 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#78), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct -(59) CometFilter +(51) CometFilter Input [6]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_net_profit#77, ss_sold_date_sk#78] Condition : (((((((isnotnull(ss_net_profit#77) AND isnotnull(ss_net_paid#76)) AND isnotnull(ss_quantity#75)) AND (ss_net_profit#77 > 1.00)) AND (ss_net_paid#76 > 0.00)) AND (ss_quantity#75 > 0)) AND isnotnull(ss_ticket_number#74)) AND isnotnull(ss_item_sk#73)) -(60) CometProject +(52) CometProject Input [6]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_net_profit#77, ss_sold_date_sk#78] Arguments: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78], [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78] -(61) CometBroadcastExchange +(53) CometBroadcastExchange Input [5]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78] Arguments: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78] -(62) Scan parquet spark_catalog.default.store_returns +(54) Scan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83, sr_returned_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(63) CometFilter +(55) CometFilter Input [5]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83, sr_returned_date_sk#84] Condition : (((isnotnull(sr_return_amt#83) AND (sr_return_amt#83 > 10000.00)) AND isnotnull(sr_ticket_number#81)) AND isnotnull(sr_item_sk#80)) -(64) CometProject +(56) CometProject Input [5]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83, sr_returned_date_sk#84] Arguments: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83], [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -(65) CometBroadcastHashJoin +(57) CometBroadcastHashJoin Left output [5]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78] Right output [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] Arguments: [ss_ticket_number#74, ss_item_sk#73], [sr_ticket_number#81, sr_item_sk#80], Inner, BuildLeft -(66) CometProject +(58) CometProject Input [9]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78, sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] Arguments: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78, sr_return_quantity#82, sr_return_amt#83], [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78, sr_return_quantity#82, sr_return_amt#83] -(67) ReusedExchange [Reuses operator id: 13] +(59) ReusedExchange [Reuses operator id: 13] Output [1]: [d_date_sk#85] -(68) CometBroadcastHashJoin +(60) CometBroadcastHashJoin Left output [6]: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78, sr_return_quantity#82, sr_return_amt#83] Right output [1]: [d_date_sk#85] Arguments: [ss_sold_date_sk#78], [d_date_sk#85], Inner, BuildRight -(69) CometProject +(61) CometProject Input [7]: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78, sr_return_quantity#82, sr_return_amt#83, d_date_sk#85] Arguments: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, sr_return_quantity#82, sr_return_amt#83], [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, sr_return_quantity#82, sr_return_amt#83] -(70) ColumnarToRow [codegen id : 11] +(62) ColumnarToRow [codegen id : 11] Input [5]: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, sr_return_quantity#82, sr_return_amt#83] -(71) HashAggregate [codegen id : 11] +(63) HashAggregate [codegen id : 11] Input [5]: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, sr_return_quantity#82, sr_return_amt#83] Keys [1]: [ss_item_sk#73] Functions [4]: [partial_sum(coalesce(sr_return_quantity#82, 0)), partial_sum(coalesce(ss_quantity#75, 0)), partial_sum(coalesce(cast(sr_return_amt#83 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#76 as decimal(12,2)), 0.00))] Aggregate Attributes [6]: [sum#86, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] Results [7]: [ss_item_sk#73, sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97] -(72) RowToColumnar -Input [7]: [ss_item_sk#73, sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97] - -(73) CometColumnarExchange +(64) Exchange Input [7]: [ss_item_sk#73, sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97] -Arguments: hashpartitioning(ss_item_sk#73, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ss_item_sk#73, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(74) ColumnarToRow [codegen id : 12] -Input [7]: [ss_item_sk#73, sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97] - -(75) HashAggregate [codegen id : 12] +(65) HashAggregate [codegen id : 12] Input [7]: [ss_item_sk#73, sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97] Keys [1]: [ss_item_sk#73] Functions [4]: [sum(coalesce(sr_return_quantity#82, 0)), sum(coalesce(ss_quantity#75, 0)), sum(coalesce(cast(sr_return_amt#83 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#76 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#82, 0))#98, sum(coalesce(ss_quantity#75, 0))#99, sum(coalesce(cast(sr_return_amt#83 as decimal(12,2)), 0.00))#100, sum(coalesce(cast(ss_net_paid#76 as decimal(12,2)), 0.00))#101] Results [3]: [ss_item_sk#73 AS item#102, (cast(sum(coalesce(sr_return_quantity#82, 0))#98 as decimal(15,4)) / cast(sum(coalesce(ss_quantity#75, 0))#99 as decimal(15,4))) AS return_ratio#103, (cast(sum(coalesce(cast(sr_return_amt#83 as decimal(12,2)), 0.00))#100 as decimal(15,4)) / cast(sum(coalesce(cast(ss_net_paid#76 as decimal(12,2)), 0.00))#101 as decimal(15,4))) AS currency_ratio#104] -(76) RowToColumnar -Input [3]: [item#102, return_ratio#103, currency_ratio#104] - -(77) CometColumnarExchange +(66) Exchange Input [3]: [item#102, return_ratio#103, currency_ratio#104] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(78) CometSort +(67) Sort [codegen id : 13] Input [3]: [item#102, return_ratio#103, currency_ratio#104] -Arguments: [item#102, return_ratio#103, currency_ratio#104], [return_ratio#103 ASC NULLS FIRST] +Arguments: [return_ratio#103 ASC NULLS FIRST], false, 0 -(79) ColumnarToRow [codegen id : 13] -Input [3]: [item#102, return_ratio#103, currency_ratio#104] - -(80) Window +(68) Window Input [3]: [item#102, return_ratio#103, currency_ratio#104] Arguments: [rank(return_ratio#103) windowspecdefinition(return_ratio#103 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#105], [return_ratio#103 ASC NULLS FIRST] -(81) Sort [codegen id : 14] +(69) Sort [codegen id : 14] Input [4]: [item#102, return_ratio#103, currency_ratio#104, return_rank#105] Arguments: [currency_ratio#104 ASC NULLS FIRST], false, 0 -(82) Window +(70) Window Input [4]: [item#102, return_ratio#103, currency_ratio#104, return_rank#105] Arguments: [rank(currency_ratio#104) windowspecdefinition(currency_ratio#104 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#106], [currency_ratio#104 ASC NULLS FIRST] -(83) Filter [codegen id : 15] +(71) Filter [codegen id : 15] Input [5]: [item#102, return_ratio#103, currency_ratio#104, return_rank#105, currency_rank#106] Condition : ((return_rank#105 <= 10) OR (currency_rank#106 <= 10)) -(84) Project [codegen id : 15] +(72) Project [codegen id : 15] Output [5]: [store AS channel#107, item#102, return_ratio#103, return_rank#105, currency_rank#106] Input [5]: [item#102, return_ratio#103, currency_ratio#104, return_rank#105, currency_rank#106] -(85) Union +(73) Union -(86) HashAggregate [codegen id : 16] +(74) HashAggregate [codegen id : 16] Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Keys [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Functions: [] Aggregate Attributes: [] Results [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] -(87) RowToColumnar -Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] - -(88) CometColumnarExchange +(75) Exchange Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] -Arguments: hashpartitioning(channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(89) CometHashAggregate +(76) HashAggregate [codegen id : 17] Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Keys [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] -(90) CometTakeOrderedAndProject -Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#37 ASC NULLS FIRST,return_rank#35 ASC NULLS FIRST,currency_rank#36 ASC NULLS FIRST], output=[channel#37,item#32,return_ratio#33,return_rank#35,currency_rank#36]), [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36], 100, [channel#37 ASC NULLS FIRST, return_rank#35 ASC NULLS FIRST, currency_rank#36 ASC NULLS FIRST], [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] - -(91) ColumnarToRow [codegen id : 17] +(77) TakeOrderedAndProject Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] +Arguments: 100, [channel#37 ASC NULLS FIRST, return_rank#35 ASC NULLS FIRST, currency_rank#36 ASC NULLS FIRST], [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (96) -+- * ColumnarToRow (95) - +- CometProject (94) - +- CometFilter (93) - +- CometScan parquet spark_catalog.default.date_dim (92) +BroadcastExchange (82) ++- * ColumnarToRow (81) + +- CometProject (80) + +- CometFilter (79) + +- CometScan parquet spark_catalog.default.date_dim (78) -(92) Scan parquet spark_catalog.default.date_dim +(78) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(93) CometFilter +(79) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) -(94) CometProject +(80) CometProject Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Arguments: [d_date_sk#13], [d_date_sk#13] -(95) ColumnarToRow [codegen id : 1] +(81) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#13] -(96) BroadcastExchange +(82) BroadcastExchange Input [1]: [d_date_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 31 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#7 +Subquery:2 Hosting operator id = 27 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#7 -Subquery:3 Hosting operator id = 58 Hosting Expression = ss_sold_date_sk#78 IN dynamicpruning#7 +Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#78 IN dynamicpruning#7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt index 4d4e531a0..bd3b21cdb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q49/simplified.txt @@ -1,135 +1,121 @@ -WholeStageCodegen (17) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] - CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] - CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (5) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (4) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #2 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] +TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] + WholeStageCodegen (17) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (16) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (5) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (4) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (3) + Sort [return_ratio] + InputAdapter + Exchange #2 + WholeStageCodegen (2) + HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [ws_item_sk] #3 + WholeStageCodegen (1) + HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] ColumnarToRow InputAdapter - CometColumnarExchange [ws_item_sk] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (10) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (9) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #7 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (10) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (9) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (8) + Sort [return_ratio] + InputAdapter + Exchange #7 + WholeStageCodegen (7) + HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [cs_item_sk] #8 + WholeStageCodegen (6) + HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] ColumnarToRow InputAdapter - CometColumnarExchange [cs_item_sk] #8 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - WholeStageCodegen (15) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (14) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (13) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #10 - RowToColumnar - WholeStageCodegen (12) - HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + WholeStageCodegen (15) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (14) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (13) + Sort [return_ratio] + InputAdapter + Exchange #10 + WholeStageCodegen (12) + HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [ss_item_sk] #11 + WholeStageCodegen (11) + HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] ColumnarToRow InputAdapter - CometColumnarExchange [ss_item_sk] #11 - RowToColumnar - WholeStageCodegen (11) - HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt index 905c7bd88..e6d836892 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/explain.txt @@ -1,82 +1,74 @@ == Physical Plan == -TakeOrderedAndProject (78) -+- * HashAggregate (77) - +- * ColumnarToRow (76) - +- CometColumnarExchange (75) - +- RowToColumnar (74) - +- * HashAggregate (73) - +- * Expand (72) - +- Union (71) - :- * HashAggregate (24) - : +- * ColumnarToRow (23) - : +- CometColumnarExchange (22) - : +- RowToColumnar (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometUnion (7) - : : : :- CometProject (3) - : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometScan parquet spark_catalog.default.store_returns (4) - : : +- CometBroadcastExchange (11) - : : +- CometProject (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.date_dim (8) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.store (14) - :- * HashAggregate (45) - : +- * ColumnarToRow (44) - : +- CometColumnarExchange (43) - : +- RowToColumnar (42) - : +- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (34) - : : +- CometBroadcastHashJoin (33) - : : :- CometUnion (31) - : : : :- CometProject (27) - : : : : +- CometFilter (26) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (25) - : : : +- CometProject (30) - : : : +- CometFilter (29) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (28) - : : +- ReusedExchange (32) - : +- CometBroadcastExchange (37) - : +- CometFilter (36) - : +- CometScan parquet spark_catalog.default.catalog_page (35) - +- * HashAggregate (70) - +- * ColumnarToRow (69) - +- CometColumnarExchange (68) - +- RowToColumnar (67) - +- * HashAggregate (66) - +- * ColumnarToRow (65) - +- CometProject (64) - +- CometBroadcastHashJoin (63) - :- CometProject (59) - : +- CometBroadcastHashJoin (58) - : :- CometUnion (56) - : : :- CometProject (48) - : : : +- CometFilter (47) - : : : +- CometScan parquet spark_catalog.default.web_sales (46) - : : +- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometBroadcastExchange (50) - : : : +- CometScan parquet spark_catalog.default.web_returns (49) - : : +- CometProject (53) - : : +- CometFilter (52) - : : +- CometScan parquet spark_catalog.default.web_sales (51) - : +- ReusedExchange (57) - +- CometBroadcastExchange (62) - +- CometFilter (61) - +- CometScan parquet spark_catalog.default.web_site (60) +TakeOrderedAndProject (70) ++- * HashAggregate (69) + +- Exchange (68) + +- * HashAggregate (67) + +- * Expand (66) + +- Union (65) + :- * HashAggregate (22) + : +- Exchange (21) + : +- * HashAggregate (20) + : +- * ColumnarToRow (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometUnion (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.date_dim (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan parquet spark_catalog.default.store (14) + :- * HashAggregate (41) + : +- Exchange (40) + : +- * HashAggregate (39) + : +- * ColumnarToRow (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometUnion (29) + : : : :- CometProject (25) + : : : : +- CometFilter (24) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) + : : : +- CometProject (28) + : : : +- CometFilter (27) + : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) + : : +- ReusedExchange (30) + : +- CometBroadcastExchange (35) + : +- CometFilter (34) + : +- CometScan parquet spark_catalog.default.catalog_page (33) + +- * HashAggregate (64) + +- Exchange (63) + +- * HashAggregate (62) + +- * ColumnarToRow (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometUnion (52) + : : :- CometProject (44) + : : : +- CometFilter (43) + : : : +- CometScan parquet spark_catalog.default.web_sales (42) + : : +- CometProject (51) + : : +- CometBroadcastHashJoin (50) + : : :- CometBroadcastExchange (46) + : : : +- CometScan parquet spark_catalog.default.web_returns (45) + : : +- CometProject (49) + : : +- CometFilter (48) + : : +- CometScan parquet spark_catalog.default.web_sales (47) + : +- ReusedExchange (53) + +- CometBroadcastExchange (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.web_site (56) (1) Scan parquet spark_catalog.default.store_sales @@ -177,24 +169,18 @@ Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledV Aggregate Attributes [4]: [sum#26, sum#27, sum#28, sum#29] Results [5]: [s_store_id#25, sum#30, sum#31, sum#32, sum#33] -(21) RowToColumnar +(21) Exchange Input [5]: [s_store_id#25, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(22) CometColumnarExchange -Input [5]: [s_store_id#25, sum#30, sum#31, sum#32, sum#33] -Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(23) ColumnarToRow [codegen id : 2] -Input [5]: [s_store_id#25, sum#30, sum#31, sum#32, sum#33] - -(24) HashAggregate [codegen id : 2] +(22) HashAggregate [codegen id : 2] Input [5]: [s_store_id#25, sum#30, sum#31, sum#32, sum#33] Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#34, sum(UnscaledValue(return_amt#10))#35, sum(UnscaledValue(profit#9))#36, sum(UnscaledValue(net_loss#11))#37] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#8))#34,17,2) AS sales#38, MakeDecimal(sum(UnscaledValue(return_amt#10))#35,17,2) AS returns#39, (MakeDecimal(sum(UnscaledValue(profit#9))#36,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#37,17,2)) AS profit#40, store channel AS channel#41, concat(store, s_store_id#25) AS id#42] -(25) Scan parquet spark_catalog.default.catalog_sales +(23) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_catalog_page_sk#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] @@ -202,15 +188,15 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#46), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct -(26) CometFilter +(24) CometFilter Input [4]: [cs_catalog_page_sk#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Condition : isnotnull(cs_catalog_page_sk#43) -(27) CometProject +(25) CometProject Input [4]: [cs_catalog_page_sk#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Arguments: [page_sk#48, date_sk#49, sales_price#50, profit#51, return_amt#52, net_loss#53], [cs_catalog_page_sk#43 AS page_sk#48, cs_sold_date_sk#46 AS date_sk#49, cs_ext_sales_price#44 AS sales_price#50, cs_net_profit#45 AS profit#51, 0.00 AS return_amt#52, 0.00 AS net_loss#53] -(28) Scan parquet spark_catalog.default.catalog_returns +(26) Scan parquet spark_catalog.default.catalog_returns Output [4]: [cr_catalog_page_sk#54, cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57] Batched: true Location: InMemoryFileIndex [] @@ -218,82 +204,76 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#57), dynamicpruningexpression(c PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct -(29) CometFilter +(27) CometFilter Input [4]: [cr_catalog_page_sk#54, cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57] Condition : isnotnull(cr_catalog_page_sk#54) -(30) CometProject +(28) CometProject Input [4]: [cr_catalog_page_sk#54, cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57] Arguments: [page_sk#58, date_sk#59, sales_price#60, profit#61, return_amt#62, net_loss#63], [cr_catalog_page_sk#54 AS page_sk#58, cr_returned_date_sk#57 AS date_sk#59, 0.00 AS sales_price#60, 0.00 AS profit#61, cr_return_amount#55 AS return_amt#62, cr_net_loss#56 AS net_loss#63] -(31) CometUnion +(29) CometUnion Child 0 Input [6]: [page_sk#48, date_sk#49, sales_price#50, profit#51, return_amt#52, net_loss#53] Child 1 Input [6]: [page_sk#58, date_sk#59, sales_price#60, profit#61, return_amt#62, net_loss#63] -(32) ReusedExchange [Reuses operator id: 11] +(30) ReusedExchange [Reuses operator id: 11] Output [1]: [d_date_sk#64] -(33) CometBroadcastHashJoin +(31) CometBroadcastHashJoin Left output [6]: [page_sk#48, date_sk#49, sales_price#50, profit#51, return_amt#52, net_loss#53] Right output [1]: [d_date_sk#64] Arguments: [date_sk#49], [d_date_sk#64], Inner, BuildRight -(34) CometProject +(32) CometProject Input [7]: [page_sk#48, date_sk#49, sales_price#50, profit#51, return_amt#52, net_loss#53, d_date_sk#64] Arguments: [page_sk#48, sales_price#50, profit#51, return_amt#52, net_loss#53], [page_sk#48, sales_price#50, profit#51, return_amt#52, net_loss#53] -(35) Scan parquet spark_catalog.default.catalog_page +(33) Scan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#65, cp_catalog_page_id#66] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(36) CometFilter +(34) CometFilter Input [2]: [cp_catalog_page_sk#65, cp_catalog_page_id#66] Condition : isnotnull(cp_catalog_page_sk#65) -(37) CometBroadcastExchange +(35) CometBroadcastExchange Input [2]: [cp_catalog_page_sk#65, cp_catalog_page_id#66] Arguments: [cp_catalog_page_sk#65, cp_catalog_page_id#66] -(38) CometBroadcastHashJoin +(36) CometBroadcastHashJoin Left output [5]: [page_sk#48, sales_price#50, profit#51, return_amt#52, net_loss#53] Right output [2]: [cp_catalog_page_sk#65, cp_catalog_page_id#66] Arguments: [page_sk#48], [cp_catalog_page_sk#65], Inner, BuildRight -(39) CometProject +(37) CometProject Input [7]: [page_sk#48, sales_price#50, profit#51, return_amt#52, net_loss#53, cp_catalog_page_sk#65, cp_catalog_page_id#66] Arguments: [sales_price#50, profit#51, return_amt#52, net_loss#53, cp_catalog_page_id#66], [sales_price#50, profit#51, return_amt#52, net_loss#53, cp_catalog_page_id#66] -(40) ColumnarToRow [codegen id : 3] +(38) ColumnarToRow [codegen id : 3] Input [5]: [sales_price#50, profit#51, return_amt#52, net_loss#53, cp_catalog_page_id#66] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [5]: [sales_price#50, profit#51, return_amt#52, net_loss#53, cp_catalog_page_id#66] Keys [1]: [cp_catalog_page_id#66] Functions [4]: [partial_sum(UnscaledValue(sales_price#50)), partial_sum(UnscaledValue(return_amt#52)), partial_sum(UnscaledValue(profit#51)), partial_sum(UnscaledValue(net_loss#53))] Aggregate Attributes [4]: [sum#67, sum#68, sum#69, sum#70] Results [5]: [cp_catalog_page_id#66, sum#71, sum#72, sum#73, sum#74] -(42) RowToColumnar -Input [5]: [cp_catalog_page_id#66, sum#71, sum#72, sum#73, sum#74] - -(43) CometColumnarExchange -Input [5]: [cp_catalog_page_id#66, sum#71, sum#72, sum#73, sum#74] -Arguments: hashpartitioning(cp_catalog_page_id#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [cp_catalog_page_id#66, sum#71, sum#72, sum#73, sum#74] +Arguments: hashpartitioning(cp_catalog_page_id#66, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [cp_catalog_page_id#66, sum#71, sum#72, sum#73, sum#74] Keys [1]: [cp_catalog_page_id#66] Functions [4]: [sum(UnscaledValue(sales_price#50)), sum(UnscaledValue(return_amt#52)), sum(UnscaledValue(profit#51)), sum(UnscaledValue(net_loss#53))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#50))#75, sum(UnscaledValue(return_amt#52))#76, sum(UnscaledValue(profit#51))#77, sum(UnscaledValue(net_loss#53))#78] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#50))#75,17,2) AS sales#79, MakeDecimal(sum(UnscaledValue(return_amt#52))#76,17,2) AS returns#80, (MakeDecimal(sum(UnscaledValue(profit#51))#77,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#53))#78,17,2)) AS profit#81, catalog channel AS channel#82, concat(catalog_page, cp_catalog_page_id#66) AS id#83] -(46) Scan parquet spark_catalog.default.web_sales +(42) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_web_site_sk#84, ws_ext_sales_price#85, ws_net_profit#86, ws_sold_date_sk#87] Batched: true Location: InMemoryFileIndex [] @@ -301,190 +281,178 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#87), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct -(47) CometFilter +(43) CometFilter Input [4]: [ws_web_site_sk#84, ws_ext_sales_price#85, ws_net_profit#86, ws_sold_date_sk#87] Condition : isnotnull(ws_web_site_sk#84) -(48) CometProject +(44) CometProject Input [4]: [ws_web_site_sk#84, ws_ext_sales_price#85, ws_net_profit#86, ws_sold_date_sk#87] Arguments: [wsr_web_site_sk#89, date_sk#90, sales_price#91, profit#92, return_amt#93, net_loss#94], [ws_web_site_sk#84 AS wsr_web_site_sk#89, ws_sold_date_sk#87 AS date_sk#90, ws_ext_sales_price#85 AS sales_price#91, ws_net_profit#86 AS profit#92, 0.00 AS return_amt#93, 0.00 AS net_loss#94] -(49) Scan parquet spark_catalog.default.web_returns +(45) Scan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#95, wr_order_number#96, wr_return_amt#97, wr_net_loss#98, wr_returned_date_sk#99] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(wr_returned_date_sk#99), dynamicpruningexpression(wr_returned_date_sk#99 IN dynamicpruning#88)] ReadSchema: struct -(50) CometBroadcastExchange +(46) CometBroadcastExchange Input [5]: [wr_item_sk#95, wr_order_number#96, wr_return_amt#97, wr_net_loss#98, wr_returned_date_sk#99] Arguments: [wr_item_sk#95, wr_order_number#96, wr_return_amt#97, wr_net_loss#98, wr_returned_date_sk#99] -(51) Scan parquet spark_catalog.default.web_sales +(47) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102, ws_sold_date_sk#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(52) CometFilter +(48) CometFilter Input [4]: [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102, ws_sold_date_sk#103] Condition : ((isnotnull(ws_item_sk#100) AND isnotnull(ws_order_number#102)) AND isnotnull(ws_web_site_sk#101)) -(53) CometProject +(49) CometProject Input [4]: [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102, ws_sold_date_sk#103] Arguments: [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102], [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [5]: [wr_item_sk#95, wr_order_number#96, wr_return_amt#97, wr_net_loss#98, wr_returned_date_sk#99] Right output [3]: [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102] Arguments: [wr_item_sk#95, wr_order_number#96], [ws_item_sk#100, ws_order_number#102], Inner, BuildLeft -(55) CometProject +(51) CometProject Input [8]: [wr_item_sk#95, wr_order_number#96, wr_return_amt#97, wr_net_loss#98, wr_returned_date_sk#99, ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102] Arguments: [wsr_web_site_sk#104, date_sk#105, sales_price#106, profit#107, return_amt#108, net_loss#109], [ws_web_site_sk#101 AS wsr_web_site_sk#104, wr_returned_date_sk#99 AS date_sk#105, 0.00 AS sales_price#106, 0.00 AS profit#107, wr_return_amt#97 AS return_amt#108, wr_net_loss#98 AS net_loss#109] -(56) CometUnion +(52) CometUnion Child 0 Input [6]: [wsr_web_site_sk#89, date_sk#90, sales_price#91, profit#92, return_amt#93, net_loss#94] Child 1 Input [6]: [wsr_web_site_sk#104, date_sk#105, sales_price#106, profit#107, return_amt#108, net_loss#109] -(57) ReusedExchange [Reuses operator id: 11] +(53) ReusedExchange [Reuses operator id: 11] Output [1]: [d_date_sk#110] -(58) CometBroadcastHashJoin +(54) CometBroadcastHashJoin Left output [6]: [wsr_web_site_sk#89, date_sk#90, sales_price#91, profit#92, return_amt#93, net_loss#94] Right output [1]: [d_date_sk#110] Arguments: [date_sk#90], [d_date_sk#110], Inner, BuildRight -(59) CometProject +(55) CometProject Input [7]: [wsr_web_site_sk#89, date_sk#90, sales_price#91, profit#92, return_amt#93, net_loss#94, d_date_sk#110] Arguments: [wsr_web_site_sk#89, sales_price#91, profit#92, return_amt#93, net_loss#94], [wsr_web_site_sk#89, sales_price#91, profit#92, return_amt#93, net_loss#94] -(60) Scan parquet spark_catalog.default.web_site +(56) Scan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#111, web_site_id#112] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(61) CometFilter +(57) CometFilter Input [2]: [web_site_sk#111, web_site_id#112] Condition : isnotnull(web_site_sk#111) -(62) CometBroadcastExchange +(58) CometBroadcastExchange Input [2]: [web_site_sk#111, web_site_id#112] Arguments: [web_site_sk#111, web_site_id#112] -(63) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [5]: [wsr_web_site_sk#89, sales_price#91, profit#92, return_amt#93, net_loss#94] Right output [2]: [web_site_sk#111, web_site_id#112] Arguments: [wsr_web_site_sk#89], [web_site_sk#111], Inner, BuildRight -(64) CometProject +(60) CometProject Input [7]: [wsr_web_site_sk#89, sales_price#91, profit#92, return_amt#93, net_loss#94, web_site_sk#111, web_site_id#112] Arguments: [sales_price#91, profit#92, return_amt#93, net_loss#94, web_site_id#112], [sales_price#91, profit#92, return_amt#93, net_loss#94, web_site_id#112] -(65) ColumnarToRow [codegen id : 5] +(61) ColumnarToRow [codegen id : 5] Input [5]: [sales_price#91, profit#92, return_amt#93, net_loss#94, web_site_id#112] -(66) HashAggregate [codegen id : 5] +(62) HashAggregate [codegen id : 5] Input [5]: [sales_price#91, profit#92, return_amt#93, net_loss#94, web_site_id#112] Keys [1]: [web_site_id#112] Functions [4]: [partial_sum(UnscaledValue(sales_price#91)), partial_sum(UnscaledValue(return_amt#93)), partial_sum(UnscaledValue(profit#92)), partial_sum(UnscaledValue(net_loss#94))] Aggregate Attributes [4]: [sum#113, sum#114, sum#115, sum#116] Results [5]: [web_site_id#112, sum#117, sum#118, sum#119, sum#120] -(67) RowToColumnar -Input [5]: [web_site_id#112, sum#117, sum#118, sum#119, sum#120] - -(68) CometColumnarExchange -Input [5]: [web_site_id#112, sum#117, sum#118, sum#119, sum#120] -Arguments: hashpartitioning(web_site_id#112, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(69) ColumnarToRow [codegen id : 6] +(63) Exchange Input [5]: [web_site_id#112, sum#117, sum#118, sum#119, sum#120] +Arguments: hashpartitioning(web_site_id#112, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(70) HashAggregate [codegen id : 6] +(64) HashAggregate [codegen id : 6] Input [5]: [web_site_id#112, sum#117, sum#118, sum#119, sum#120] Keys [1]: [web_site_id#112] Functions [4]: [sum(UnscaledValue(sales_price#91)), sum(UnscaledValue(return_amt#93)), sum(UnscaledValue(profit#92)), sum(UnscaledValue(net_loss#94))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#91))#121, sum(UnscaledValue(return_amt#93))#122, sum(UnscaledValue(profit#92))#123, sum(UnscaledValue(net_loss#94))#124] Results [5]: [MakeDecimal(sum(UnscaledValue(sales_price#91))#121,17,2) AS sales#125, MakeDecimal(sum(UnscaledValue(return_amt#93))#122,17,2) AS returns#126, (MakeDecimal(sum(UnscaledValue(profit#92))#123,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#94))#124,17,2)) AS profit#127, web channel AS channel#128, concat(web_site, web_site_id#112) AS id#129] -(71) Union +(65) Union -(72) Expand [codegen id : 7] +(66) Expand [codegen id : 7] Input [5]: [sales#38, returns#39, profit#40, channel#41, id#42] Arguments: [[sales#38, returns#39, profit#40, channel#41, id#42, 0], [sales#38, returns#39, profit#40, channel#41, null, 1], [sales#38, returns#39, profit#40, null, null, 3]], [sales#38, returns#39, profit#40, channel#130, id#131, spark_grouping_id#132] -(73) HashAggregate [codegen id : 7] +(67) HashAggregate [codegen id : 7] Input [6]: [sales#38, returns#39, profit#40, channel#130, id#131, spark_grouping_id#132] Keys [3]: [channel#130, id#131, spark_grouping_id#132] Functions [3]: [partial_sum(sales#38), partial_sum(returns#39), partial_sum(profit#40)] Aggregate Attributes [6]: [sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] Results [9]: [channel#130, id#131, spark_grouping_id#132, sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] -(74) RowToColumnar -Input [9]: [channel#130, id#131, spark_grouping_id#132, sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] - -(75) CometColumnarExchange -Input [9]: [channel#130, id#131, spark_grouping_id#132, sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] -Arguments: hashpartitioning(channel#130, id#131, spark_grouping_id#132, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(76) ColumnarToRow [codegen id : 8] +(68) Exchange Input [9]: [channel#130, id#131, spark_grouping_id#132, sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] +Arguments: hashpartitioning(channel#130, id#131, spark_grouping_id#132, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(77) HashAggregate [codegen id : 8] +(69) HashAggregate [codegen id : 8] Input [9]: [channel#130, id#131, spark_grouping_id#132, sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] Keys [3]: [channel#130, id#131, spark_grouping_id#132] Functions [3]: [sum(sales#38), sum(returns#39), sum(profit#40)] Aggregate Attributes [3]: [sum(sales#38)#145, sum(returns#39)#146, sum(profit#40)#147] Results [5]: [channel#130, id#131, sum(sales#38)#145 AS sales#148, sum(returns#39)#146 AS returns#149, sum(profit#40)#147 AS profit#150] -(78) TakeOrderedAndProject +(70) TakeOrderedAndProject Input [5]: [channel#130, id#131, sales#148, returns#149, profit#150] Arguments: 100, [channel#130 ASC NULLS FIRST, id#131 ASC NULLS FIRST], [channel#130, id#131, sales#148, returns#149, profit#150] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (83) -+- * ColumnarToRow (82) - +- CometProject (81) - +- CometFilter (80) - +- CometScan parquet spark_catalog.default.date_dim (79) +BroadcastExchange (75) ++- * ColumnarToRow (74) + +- CometProject (73) + +- CometFilter (72) + +- CometScan parquet spark_catalog.default.date_dim (71) -(79) Scan parquet spark_catalog.default.date_dim +(71) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] ReadSchema: struct -(80) CometFilter +(72) CometFilter Input [2]: [d_date_sk#22, d_date#23] Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-06)) AND isnotnull(d_date_sk#22)) -(81) CometProject +(73) CometProject Input [2]: [d_date_sk#22, d_date#23] Arguments: [d_date_sk#22], [d_date_sk#22] -(82) ColumnarToRow [codegen id : 1] +(74) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#22] -(83) BroadcastExchange +(75) BroadcastExchange Input [1]: [d_date_sk#22] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 -Subquery:3 Hosting operator id = 25 Hosting Expression = cs_sold_date_sk#46 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 23 Hosting Expression = cs_sold_date_sk#46 IN dynamicpruning#5 -Subquery:4 Hosting operator id = 28 Hosting Expression = cr_returned_date_sk#57 IN dynamicpruning#5 +Subquery:4 Hosting operator id = 26 Hosting Expression = cr_returned_date_sk#57 IN dynamicpruning#5 -Subquery:5 Hosting operator id = 46 Hosting Expression = ws_sold_date_sk#87 IN dynamicpruning#5 +Subquery:5 Hosting operator id = 42 Hosting Expression = ws_sold_date_sk#87 IN dynamicpruning#5 -Subquery:6 Hosting operator id = 49 Hosting Expression = wr_returned_date_sk#99 IN dynamicpruning#5 +Subquery:6 Hosting operator id = 45 Hosting Expression = wr_returned_date_sk#99 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt index 9e3aaf3d7..ceb3e9213 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q5/simplified.txt @@ -1,107 +1,99 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (8) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #5 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (4) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [cp_catalog_page_id] #6 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [page_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (6) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [web_site_id] #8 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [web_site_sk,web_site_id] #10 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + InputAdapter + Exchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (7) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (2) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + InputAdapter + Exchange [s_store_id] #2 + WholeStageCodegen (1) + HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (4) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + InputAdapter + Exchange [cp_catalog_page_id] #6 + WholeStageCodegen (3) + HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (6) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + InputAdapter + Exchange [web_site_id] #8 + WholeStageCodegen (5) + HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt index 856d6238b..ff35888ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/explain.txt @@ -1,34 +1,32 @@ == Physical Plan == -TakeOrderedAndProject (30) -+- * HashAggregate (29) - +- * ColumnarToRow (28) - +- CometColumnarExchange (27) - +- RowToColumnar (26) - +- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.store_returns (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.store (8) - : +- CometBroadcastExchange (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.date_dim (13) - +- CometBroadcastExchange (21) - +- CometProject (20) - +- CometFilter (19) - +- CometScan parquet spark_catalog.default.date_dim (18) +TakeOrderedAndProject (28) ++- * HashAggregate (27) + +- Exchange (26) + +- * HashAggregate (25) + +- * ColumnarToRow (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometProject (17) + : +- CometBroadcastHashJoin (16) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.store_returns (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.store (8) + : +- CometBroadcastExchange (15) + : +- CometFilter (14) + : +- CometScan parquet spark_catalog.default.date_dim (13) + +- CometBroadcastExchange (21) + +- CometProject (20) + +- CometFilter (19) + +- CometScan parquet spark_catalog.default.date_dim (18) (1) Scan parquet spark_catalog.default.store_sales @@ -154,56 +152,50 @@ Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk# Aggregate Attributes [5]: [sum#26, sum#27, sum#28, sum#29, sum#30] Results [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, sum#31, sum#32, sum#33, sum#34, sum#35] -(26) RowToColumnar +(26) Exchange Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, sum#31, sum#32, sum#33, sum#34, sum#35] +Arguments: hashpartitioning(s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(27) CometColumnarExchange -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, sum#31, sum#32, sum#33, sum#34, sum#35] -Arguments: hashpartitioning(s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(28) ColumnarToRow [codegen id : 2] -Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, sum#31, sum#32, sum#33, sum#34, sum#35] - -(29) HashAggregate [codegen id : 2] +(27) HashAggregate [codegen id : 2] Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, sum#31, sum#32, sum#33, sum#34, sum#35] Keys [10]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] Functions [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] Aggregate Attributes [5]: [sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#36, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#37, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#38, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#39, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#40] Results [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#36 AS 30 days #41, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#37 AS 31 - 60 days #42, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#38 AS 61 - 90 days #43, sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#39 AS 91 - 120 days #44, sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#40 AS >120 days #45] -(30) TakeOrderedAndProject +(28) TakeOrderedAndProject Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, 30 days #41, 31 - 60 days #42, 61 - 90 days #43, 91 - 120 days #44, >120 days #45] Arguments: 100, [s_store_name#12 ASC NULLS FIRST, s_company_id#13 ASC NULLS FIRST, s_street_number#14 ASC NULLS FIRST, s_street_name#15 ASC NULLS FIRST, s_street_type#16 ASC NULLS FIRST, s_suite_number#17 ASC NULLS FIRST, s_city#18 ASC NULLS FIRST, s_county#19 ASC NULLS FIRST, s_state#20 ASC NULLS FIRST, s_zip#21 ASC NULLS FIRST], [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, 30 days #41, 31 - 60 days #42, 61 - 90 days #43, 91 - 120 days #44, >120 days #45] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = sr_returned_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (35) -+- * ColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan parquet spark_catalog.default.date_dim (31) +BroadcastExchange (33) ++- * ColumnarToRow (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan parquet spark_catalog.default.date_dim (29) -(31) Scan parquet spark_catalog.default.date_dim +(29) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#23, d_year#24, d_moy#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,8), IsNotNull(d_date_sk)] ReadSchema: struct -(32) CometFilter +(30) CometFilter Input [3]: [d_date_sk#23, d_year#24, d_moy#25] Condition : ((((isnotnull(d_year#24) AND isnotnull(d_moy#25)) AND (d_year#24 = 2001)) AND (d_moy#25 = 8)) AND isnotnull(d_date_sk#23)) -(33) CometProject +(31) CometProject Input [3]: [d_date_sk#23, d_year#24, d_moy#25] Arguments: [d_date_sk#23], [d_date_sk#23] -(34) ColumnarToRow [codegen id : 1] +(32) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#23] -(35) BroadcastExchange +(33) BroadcastExchange Input [1]: [d_date_sk#23] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt index ae77bbd8a..aa3f7f2cf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q50/simplified.txt @@ -1,42 +1,40 @@ TakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] WholeStageCodegen (2) HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum] [sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sr_returned_date_sk,ss_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter + InputAdapter + Exchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 + WholeStageCodegen (1) + HashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sr_returned_date_sk,ss_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] - CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastHashJoin [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk,s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] - CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 - CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #4 - CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] - CometBroadcastExchange [d_date_sk] #5 - CometFilter [d_date_sk] - CometScan parquet spark_catalog.default.date_dim [d_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastHashJoin [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk,s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometProject [ss_store_sk,ss_sold_date_sk,sr_returned_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk,sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_sold_date_sk] + CometBroadcastExchange [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] #2 + CometFilter [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_customer_sk,sr_ticket_number,sr_returned_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #4 + CometFilter [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] + CometBroadcastExchange [d_date_sk] #5 + CometFilter [d_date_sk] + CometScan parquet spark_catalog.default.date_dim [d_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt index bb3968041..20fdfd4a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/explain.txt @@ -1,55 +1,44 @@ == Physical Plan == -* ColumnarToRow (51) -+- CometTakeOrderedAndProject (50) - +- CometFilter (49) - +- CometWindowExec (48) - +- CometSort (47) - +- CometColumnarExchange (46) - +- CometProject (45) - +- CometSortMergeJoin (44) - :- CometSort (23) - : +- CometColumnarExchange (22) - : +- RowToColumnar (21) - : +- * Project (20) - : +- Window (19) - : +- * ColumnarToRow (18) - : +- CometSort (17) - : +- CometColumnarExchange (16) - : +- RowToColumnar (15) - : +- * HashAggregate (14) - : +- * ColumnarToRow (13) - : +- CometColumnarExchange (12) - : +- RowToColumnar (11) - : +- * HashAggregate (10) - : +- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.web_sales (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.date_dim (3) - +- CometSort (43) - +- CometColumnarExchange (42) - +- RowToColumnar (41) - +- * Project (40) - +- Window (39) - +- * ColumnarToRow (38) - +- CometSort (37) - +- CometColumnarExchange (36) - +- RowToColumnar (35) - +- * HashAggregate (34) - +- * ColumnarToRow (33) - +- CometColumnarExchange (32) - +- RowToColumnar (31) - +- * HashAggregate (30) - +- * ColumnarToRow (29) - +- CometProject (28) - +- CometBroadcastHashJoin (27) - :- CometFilter (25) - : +- CometScan parquet spark_catalog.default.store_sales (24) - +- ReusedExchange (26) +TakeOrderedAndProject (40) ++- * Filter (39) + +- Window (38) + +- * Sort (37) + +- Exchange (36) + +- * Project (35) + +- * SortMergeJoin FullOuter (34) + :- * Sort (18) + : +- Exchange (17) + : +- * Project (16) + : +- Window (15) + : +- * Sort (14) + : +- Exchange (13) + : +- * HashAggregate (12) + : +- Exchange (11) + : +- * HashAggregate (10) + : +- * ColumnarToRow (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.date_dim (3) + +- * Sort (33) + +- Exchange (32) + +- * Project (31) + +- Window (30) + +- * Sort (29) + +- Exchange (28) + +- * HashAggregate (27) + +- Exchange (26) + +- * HashAggregate (25) + +- * ColumnarToRow (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometFilter (20) + : +- CometScan parquet spark_catalog.default.store_sales (19) + +- ReusedExchange (21) (1) Scan parquet spark_catalog.default.web_sales @@ -102,57 +91,42 @@ Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] Aggregate Attributes [1]: [sum#8] Results [3]: [ws_item_sk#1, d_date#6, sum#9] -(11) RowToColumnar +(11) Exchange Input [3]: [ws_item_sk#1, d_date#6, sum#9] +Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(12) CometColumnarExchange -Input [3]: [ws_item_sk#1, d_date#6, sum#9] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(13) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#1, d_date#6, sum#9] - -(14) HashAggregate [codegen id : 2] +(12) HashAggregate [codegen id : 2] Input [3]: [ws_item_sk#1, d_date#6, sum#9] Keys [2]: [ws_item_sk#1, d_date#6] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#10] Results [4]: [ws_item_sk#1 AS item_sk#11, d_date#6, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#10,17,2) AS _w0#12, ws_item_sk#1] -(15) RowToColumnar -Input [4]: [item_sk#11, d_date#6, _w0#12, ws_item_sk#1] - -(16) CometColumnarExchange +(13) Exchange Input [4]: [item_sk#11, d_date#6, _w0#12, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(17) CometSort +(14) Sort [codegen id : 3] Input [4]: [item_sk#11, d_date#6, _w0#12, ws_item_sk#1] -Arguments: [item_sk#11, d_date#6, _w0#12, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0 -(18) ColumnarToRow [codegen id : 3] -Input [4]: [item_sk#11, d_date#6, _w0#12, ws_item_sk#1] - -(19) Window +(15) Window Input [4]: [item_sk#11, d_date#6, _w0#12, ws_item_sk#1] Arguments: [sum(_w0#12) windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#13], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST] -(20) Project [codegen id : 4] +(16) Project [codegen id : 4] Output [3]: [item_sk#11, d_date#6, cume_sales#13] Input [5]: [item_sk#11, d_date#6, _w0#12, ws_item_sk#1, cume_sales#13] -(21) RowToColumnar +(17) Exchange Input [3]: [item_sk#11, d_date#6, cume_sales#13] +Arguments: hashpartitioning(item_sk#11, d_date#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(22) CometColumnarExchange +(18) Sort [codegen id : 5] Input [3]: [item_sk#11, d_date#6, cume_sales#13] -Arguments: hashpartitioning(item_sk#11, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: [item_sk#11 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0 -(23) CometSort -Input [3]: [item_sk#11, d_date#6, cume_sales#13] -Arguments: [item_sk#11, d_date#6, cume_sales#13], [item_sk#11 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] - -(24) Scan parquet spark_catalog.default.store_sales +(19) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#14, ss_sales_price#15, ss_sold_date_sk#16] Batched: true Location: InMemoryFileIndex [] @@ -160,146 +134,129 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#16), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(25) CometFilter +(20) CometFilter Input [3]: [ss_item_sk#14, ss_sales_price#15, ss_sold_date_sk#16] Condition : isnotnull(ss_item_sk#14) -(26) ReusedExchange [Reuses operator id: 6] +(21) ReusedExchange [Reuses operator id: 6] Output [2]: [d_date_sk#18, d_date#19] -(27) CometBroadcastHashJoin +(22) CometBroadcastHashJoin Left output [3]: [ss_item_sk#14, ss_sales_price#15, ss_sold_date_sk#16] Right output [2]: [d_date_sk#18, d_date#19] Arguments: [ss_sold_date_sk#16], [d_date_sk#18], Inner, BuildRight -(28) CometProject +(23) CometProject Input [5]: [ss_item_sk#14, ss_sales_price#15, ss_sold_date_sk#16, d_date_sk#18, d_date#19] Arguments: [ss_item_sk#14, ss_sales_price#15, d_date#19], [ss_item_sk#14, ss_sales_price#15, d_date#19] -(29) ColumnarToRow [codegen id : 5] +(24) ColumnarToRow [codegen id : 6] Input [3]: [ss_item_sk#14, ss_sales_price#15, d_date#19] -(30) HashAggregate [codegen id : 5] +(25) HashAggregate [codegen id : 6] Input [3]: [ss_item_sk#14, ss_sales_price#15, d_date#19] Keys [2]: [ss_item_sk#14, d_date#19] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#15))] Aggregate Attributes [1]: [sum#20] Results [3]: [ss_item_sk#14, d_date#19, sum#21] -(31) RowToColumnar -Input [3]: [ss_item_sk#14, d_date#19, sum#21] - -(32) CometColumnarExchange -Input [3]: [ss_item_sk#14, d_date#19, sum#21] -Arguments: hashpartitioning(ss_item_sk#14, d_date#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(33) ColumnarToRow [codegen id : 6] +(26) Exchange Input [3]: [ss_item_sk#14, d_date#19, sum#21] +Arguments: hashpartitioning(ss_item_sk#14, d_date#19, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(34) HashAggregate [codegen id : 6] +(27) HashAggregate [codegen id : 7] Input [3]: [ss_item_sk#14, d_date#19, sum#21] Keys [2]: [ss_item_sk#14, d_date#19] Functions [1]: [sum(UnscaledValue(ss_sales_price#15))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#15))#22] Results [4]: [ss_item_sk#14 AS item_sk#23, d_date#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#15))#22,17,2) AS _w0#24, ss_item_sk#14] -(35) RowToColumnar +(28) Exchange Input [4]: [item_sk#23, d_date#19, _w0#24, ss_item_sk#14] +Arguments: hashpartitioning(ss_item_sk#14, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(36) CometColumnarExchange +(29) Sort [codegen id : 8] Input [4]: [item_sk#23, d_date#19, _w0#24, ss_item_sk#14] -Arguments: hashpartitioning(ss_item_sk#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: [ss_item_sk#14 ASC NULLS FIRST, d_date#19 ASC NULLS FIRST], false, 0 -(37) CometSort -Input [4]: [item_sk#23, d_date#19, _w0#24, ss_item_sk#14] -Arguments: [item_sk#23, d_date#19, _w0#24, ss_item_sk#14], [ss_item_sk#14 ASC NULLS FIRST, d_date#19 ASC NULLS FIRST] - -(38) ColumnarToRow [codegen id : 7] -Input [4]: [item_sk#23, d_date#19, _w0#24, ss_item_sk#14] - -(39) Window +(30) Window Input [4]: [item_sk#23, d_date#19, _w0#24, ss_item_sk#14] Arguments: [sum(_w0#24) windowspecdefinition(ss_item_sk#14, d_date#19 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#25], [ss_item_sk#14], [d_date#19 ASC NULLS FIRST] -(40) Project [codegen id : 8] +(31) Project [codegen id : 9] Output [3]: [item_sk#23, d_date#19, cume_sales#25] Input [5]: [item_sk#23, d_date#19, _w0#24, ss_item_sk#14, cume_sales#25] -(41) RowToColumnar -Input [3]: [item_sk#23, d_date#19, cume_sales#25] - -(42) CometColumnarExchange +(32) Exchange Input [3]: [item_sk#23, d_date#19, cume_sales#25] -Arguments: hashpartitioning(item_sk#23, d_date#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(item_sk#23, d_date#19, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(43) CometSort +(33) Sort [codegen id : 10] Input [3]: [item_sk#23, d_date#19, cume_sales#25] -Arguments: [item_sk#23, d_date#19, cume_sales#25], [item_sk#23 ASC NULLS FIRST, d_date#19 ASC NULLS FIRST] +Arguments: [item_sk#23 ASC NULLS FIRST, d_date#19 ASC NULLS FIRST], false, 0 -(44) CometSortMergeJoin -Left output [3]: [item_sk#11, d_date#6, cume_sales#13] -Right output [3]: [item_sk#23, d_date#19, cume_sales#25] -Arguments: [item_sk#11, d_date#6], [item_sk#23, d_date#19], FullOuter +(34) SortMergeJoin [codegen id : 11] +Left keys [2]: [item_sk#11, d_date#6] +Right keys [2]: [item_sk#23, d_date#19] +Join type: FullOuter +Join condition: None -(45) CometProject +(35) Project [codegen id : 11] +Output [4]: [CASE WHEN isnotnull(item_sk#11) THEN item_sk#11 ELSE item_sk#23 END AS item_sk#26, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#19 END AS d_date#27, cume_sales#13 AS web_sales#28, cume_sales#25 AS store_sales#29] Input [6]: [item_sk#11, d_date#6, cume_sales#13, item_sk#23, d_date#19, cume_sales#25] -Arguments: [item_sk#26, d_date#27, web_sales#28, store_sales#29], [CASE WHEN isnotnull(item_sk#11) THEN item_sk#11 ELSE item_sk#23 END AS item_sk#26, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#19 END AS d_date#27, cume_sales#13 AS web_sales#28, cume_sales#25 AS store_sales#29] -(46) CometColumnarExchange +(36) Exchange Input [4]: [item_sk#26, d_date#27, web_sales#28, store_sales#29] -Arguments: hashpartitioning(item_sk#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(item_sk#26, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(47) CometSort +(37) Sort [codegen id : 12] Input [4]: [item_sk#26, d_date#27, web_sales#28, store_sales#29] -Arguments: [item_sk#26, d_date#27, web_sales#28, store_sales#29], [item_sk#26 ASC NULLS FIRST, d_date#27 ASC NULLS FIRST] +Arguments: [item_sk#26 ASC NULLS FIRST, d_date#27 ASC NULLS FIRST], false, 0 -(48) CometWindowExec +(38) Window Input [4]: [item_sk#26, d_date#27, web_sales#28, store_sales#29] -Arguments: [item_sk#26, d_date#27, web_sales#28, store_sales#29, web_cumulative#30, store_cumulative#31], [max(web_sales#28) windowspecdefinition(item_sk#26, d_date#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#30, max(store_sales#29) windowspecdefinition(item_sk#26, d_date#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#31], [item_sk#26], [d_date#27 ASC NULLS FIRST] +Arguments: [max(web_sales#28) windowspecdefinition(item_sk#26, d_date#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#30, max(store_sales#29) windowspecdefinition(item_sk#26, d_date#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#31], [item_sk#26], [d_date#27 ASC NULLS FIRST] -(49) CometFilter +(39) Filter [codegen id : 13] Input [6]: [item_sk#26, d_date#27, web_sales#28, store_sales#29, web_cumulative#30, store_cumulative#31] Condition : ((isnotnull(web_cumulative#30) AND isnotnull(store_cumulative#31)) AND (web_cumulative#30 > store_cumulative#31)) -(50) CometTakeOrderedAndProject -Input [6]: [item_sk#26, d_date#27, web_sales#28, store_sales#29, web_cumulative#30, store_cumulative#31] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_sk#26 ASC NULLS FIRST,d_date#27 ASC NULLS FIRST], output=[item_sk#26,d_date#27,web_sales#28,store_sales#29,web_cumulative#30,store_cumulative#31]), [item_sk#26, d_date#27, web_sales#28, store_sales#29, web_cumulative#30, store_cumulative#31], 100, [item_sk#26 ASC NULLS FIRST, d_date#27 ASC NULLS FIRST], [item_sk#26, d_date#27, web_sales#28, store_sales#29, web_cumulative#30, store_cumulative#31] - -(51) ColumnarToRow [codegen id : 9] +(40) TakeOrderedAndProject Input [6]: [item_sk#26, d_date#27, web_sales#28, store_sales#29, web_cumulative#30, store_cumulative#31] +Arguments: 100, [item_sk#26 ASC NULLS FIRST, d_date#27 ASC NULLS FIRST], [item_sk#26, d_date#27, web_sales#28, store_sales#29, web_cumulative#30, store_cumulative#31] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (56) -+- * ColumnarToRow (55) - +- CometProject (54) - +- CometFilter (53) - +- CometScan parquet spark_catalog.default.date_dim (52) +BroadcastExchange (45) ++- * ColumnarToRow (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan parquet spark_catalog.default.date_dim (41) -(52) Scan parquet spark_catalog.default.date_dim +(41) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(53) CometFilter +(42) CometFilter Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5)) -(54) CometProject +(43) CometProject Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] -(55) ColumnarToRow [codegen id : 1] +(44) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#5, d_date#6] -(56) BroadcastExchange +(45) BroadcastExchange Input [2]: [d_date_sk#5, d_date#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 24 Hosting Expression = ss_sold_date_sk#16 IN dynamicpruning#4 +Subquery:2 Hosting operator id = 19 Hosting Expression = ss_sold_date_sk#16 IN dynamicpruning#4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt index 6738c7cef..f19312ca8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q51/simplified.txt @@ -1,32 +1,32 @@ -WholeStageCodegen (9) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - CometFilter [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - CometWindowExec [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - CometSort [item_sk,d_date,web_sales,store_sales] - CometColumnarExchange [item_sk] #1 - CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] - CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #2 - RowToColumnar - WholeStageCodegen (4) - Project [item_sk,d_date,cume_sales] +TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] + WholeStageCodegen (13) + Filter [web_cumulative,store_cumulative] + InputAdapter + Window [web_sales,item_sk,d_date,store_sales] + WholeStageCodegen (12) + Sort [item_sk,d_date] + InputAdapter + Exchange [item_sk] #1 + WholeStageCodegen (11) + Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] + SortMergeJoin [item_sk,d_date,item_sk,d_date] + InputAdapter + WholeStageCodegen (5) + Sort [item_sk,d_date] InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (3) - ColumnarToRow + Exchange [item_sk,d_date] #2 + WholeStageCodegen (4) + Project [item_sk,d_date,cume_sales] InputAdapter - CometSort [item_sk,d_date,_w0,ws_item_sk] - CometColumnarExchange [ws_item_sk] #3 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ws_item_sk,d_date] #4 - RowToColumnar + Window [_w0,ws_item_sk,d_date] + WholeStageCodegen (3) + Sort [ws_item_sk,d_date] + InputAdapter + Exchange [ws_item_sk] #3 + WholeStageCodegen (2) + HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,_w0,sum] + InputAdapter + Exchange [ws_item_sk,d_date] #4 WholeStageCodegen (1) HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] ColumnarToRow @@ -47,26 +47,24 @@ WholeStageCodegen (9) CometProject [d_date_sk,d_date] CometFilter [d_date_sk,d_date,d_month_seq] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #7 - RowToColumnar - WholeStageCodegen (8) - Project [item_sk,d_date,cume_sales] + InputAdapter + WholeStageCodegen (10) + Sort [item_sk,d_date] InputAdapter - Window [_w0,ss_item_sk,d_date] - WholeStageCodegen (7) - ColumnarToRow + Exchange [item_sk,d_date] #7 + WholeStageCodegen (9) + Project [item_sk,d_date,cume_sales] InputAdapter - CometSort [item_sk,d_date,_w0,ss_item_sk] - CometColumnarExchange [ss_item_sk] #8 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_item_sk,d_date] #9 - RowToColumnar - WholeStageCodegen (5) + Window [_w0,ss_item_sk,d_date] + WholeStageCodegen (8) + Sort [ss_item_sk,d_date] + InputAdapter + Exchange [ss_item_sk] #8 + WholeStageCodegen (7) + HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,_w0,sum] + InputAdapter + Exchange [ss_item_sk,d_date] #9 + WholeStageCodegen (6) HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] ColumnarToRow InputAdapter diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt index 3feea9a63..6d4feea57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/explain.txt @@ -1,25 +1,23 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometColumnarExchange (18) - +- RowToColumnar (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.date_dim (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometScan parquet spark_catalog.default.store_sales (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometScan parquet spark_catalog.default.item (9) +TakeOrderedAndProject (19) ++- * HashAggregate (18) + +- Exchange (17) + +- * HashAggregate (16) + +- * ColumnarToRow (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan parquet spark_catalog.default.item (9) (1) Scan parquet spark_catalog.default.date_dim @@ -100,24 +98,18 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum#11] Results [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -(17) RowToColumnar +(17) Exchange Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] +Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(18) CometColumnarExchange -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(19) ColumnarToRow [codegen id : 2] -Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] - -(20) HashAggregate [codegen id : 2] +(18) HashAggregate [codegen id : 2] Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#12] Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] Results [4]: [d_year#2, i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] -(21) TakeOrderedAndProject +(19) TakeOrderedAndProject Input [4]: [d_year#2, brand_id#14, brand#15, ext_price#16] Arguments: 100, [d_year#2 ASC NULLS FIRST, ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [d_year#2, brand_id#14, brand#15, ext_price#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt index 9acfebf22..d7661b7be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q52/simplified.txt @@ -1,25 +1,23 @@ TakeOrderedAndProject [d_year,ext_price,brand_id,brand] WholeStageCodegen (2) HashAggregate [d_year,i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,i_brand,i_brand_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [d_year,ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk,d_year] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + InputAdapter + Exchange [d_year,i_brand,i_brand_id] #1 + WholeStageCodegen (1) + HashAggregate [d_year,i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [d_year,ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,d_year,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk,d_year] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt index 5ed76db88..e85d73a07 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/explain.txt @@ -1,37 +1,33 @@ == Physical Plan == -TakeOrderedAndProject (33) -+- * Project (32) - +- * Filter (31) - +- Window (30) - +- * ColumnarToRow (29) - +- CometSort (28) - +- CometColumnarExchange (27) - +- RowToColumnar (26) - +- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometColumnarExchange (23) - +- RowToColumnar (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (14) - : +- CometBroadcastHashJoin (13) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.item (1) - : : +- CometBroadcastExchange (6) - : : +- CometFilter (5) - : : +- CometScan parquet spark_catalog.default.store_sales (4) - : +- CometBroadcastExchange (12) - : +- CometProject (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.date_dim (9) - +- CometBroadcastExchange (17) - +- CometFilter (16) - +- CometScan parquet spark_catalog.default.store (15) +TakeOrderedAndProject (29) ++- * Project (28) + +- * Filter (27) + +- Window (26) + +- * Sort (25) + +- Exchange (24) + +- * HashAggregate (23) + +- Exchange (22) + +- * HashAggregate (21) + +- * ColumnarToRow (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometScan parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan parquet spark_catalog.default.store (15) (1) Scan parquet spark_catalog.default.item @@ -136,82 +132,70 @@ Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] Aggregate Attributes [1]: [sum#19] Results [3]: [i_manufact_id#5, d_qoy#17, sum#20] -(22) RowToColumnar +(22) Exchange Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(23) CometColumnarExchange -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(24) ColumnarToRow [codegen id : 2] -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] - -(25) HashAggregate [codegen id : 2] +(23) HashAggregate [codegen id : 2] Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] Keys [2]: [i_manufact_id#5, d_qoy#17] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#21] Results [3]: [i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#21,17,2) AS sum_sales#22, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#21,17,2) AS _w0#23] -(26) RowToColumnar -Input [3]: [i_manufact_id#5, sum_sales#22, _w0#23] - -(27) CometColumnarExchange -Input [3]: [i_manufact_id#5, sum_sales#22, _w0#23] -Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(28) CometSort +(24) Exchange Input [3]: [i_manufact_id#5, sum_sales#22, _w0#23] -Arguments: [i_manufact_id#5, sum_sales#22, _w0#23], [i_manufact_id#5 ASC NULLS FIRST] +Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(29) ColumnarToRow [codegen id : 3] +(25) Sort [codegen id : 3] Input [3]: [i_manufact_id#5, sum_sales#22, _w0#23] +Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 -(30) Window +(26) Window Input [3]: [i_manufact_id#5, sum_sales#22, _w0#23] Arguments: [avg(_w0#23) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#24], [i_manufact_id#5] -(31) Filter [codegen id : 4] +(27) Filter [codegen id : 4] Input [4]: [i_manufact_id#5, sum_sales#22, _w0#23, avg_quarterly_sales#24] Condition : CASE WHEN (avg_quarterly_sales#24 > 0.000000) THEN ((abs((sum_sales#22 - avg_quarterly_sales#24)) / avg_quarterly_sales#24) > 0.1000000000000000) ELSE false END -(32) Project [codegen id : 4] +(28) Project [codegen id : 4] Output [3]: [i_manufact_id#5, sum_sales#22, avg_quarterly_sales#24] Input [4]: [i_manufact_id#5, sum_sales#22, _w0#23, avg_quarterly_sales#24] -(33) TakeOrderedAndProject +(29) TakeOrderedAndProject Input [3]: [i_manufact_id#5, sum_sales#22, avg_quarterly_sales#24] Arguments: 100, [avg_quarterly_sales#24 ASC NULLS FIRST, sum_sales#22 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#22, avg_quarterly_sales#24] ===== Subqueries ===== Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 -BroadcastExchange (38) -+- * ColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan parquet spark_catalog.default.date_dim (34) +BroadcastExchange (34) ++- * ColumnarToRow (33) + +- CometProject (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.date_dim (30) -(34) Scan parquet spark_catalog.default.date_dim +(30) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct -(35) CometFilter +(31) CometFilter Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) -(36) CometProject +(32) CometProject Input [3]: [d_date_sk#15, d_month_seq#16, d_qoy#17] Arguments: [d_date_sk#15, d_qoy#17], [d_date_sk#15, d_qoy#17] -(37) ColumnarToRow [codegen id : 1] +(33) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#15, d_qoy#17] -(38) BroadcastExchange +(34) BroadcastExchange Input [2]: [d_date_sk#15, d_qoy#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt index f08e3a806..b90cb42d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q53/simplified.txt @@ -5,45 +5,41 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] InputAdapter Window [_w0,i_manufact_id] WholeStageCodegen (3) - ColumnarToRow + Sort [i_manufact_id] InputAdapter - CometSort [i_manufact_id,sum_sales,_w0] - CometColumnarExchange [i_manufact_id] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_manufact_id,d_qoy] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_manufact_id,ss_sales_price,d_qoy] - CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] - CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] - CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_qoy] - CometProject [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [i_item_sk,i_manufact_id] - CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [d_date_sk,d_qoy] #5 - CometProject [d_date_sk,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] - CometBroadcastExchange [s_store_sk] #6 - CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] + Exchange [i_manufact_id] #1 + WholeStageCodegen (2) + HashAggregate [i_manufact_id,d_qoy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + InputAdapter + Exchange [i_manufact_id,d_qoy] #2 + WholeStageCodegen (1) + HashAggregate [i_manufact_id,d_qoy,ss_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [i_manufact_id,ss_sales_price,d_qoy] + CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] + CometProject [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy] + CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_qoy] + CometProject [i_manufact_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_manufact_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_item_sk,i_manufact_id] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manufact_id] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometBroadcastExchange [d_date_sk,d_qoy] #5 + CometProject [d_date_sk,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_qoy] + CometBroadcastExchange [s_store_sk] #6 + CometFilter [s_store_sk] + CometScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt index 050301d07..a0ce63a83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/explain.txt @@ -1,64 +1,60 @@ == Physical Plan == -TakeOrderedAndProject (60) -+- * HashAggregate (59) - +- * ColumnarToRow (58) - +- CometColumnarExchange (57) - +- RowToColumnar (56) - +- * HashAggregate (55) - +- * HashAggregate (54) - +- * ColumnarToRow (53) - +- CometColumnarExchange (52) - +- RowToColumnar (51) - +- * HashAggregate (50) - +- * ColumnarToRow (49) - +- CometProject (48) - +- CometBroadcastHashJoin (47) - :- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (37) - : : +- CometBroadcastHashJoin (36) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometHashAggregate (27) - : : : : +- CometColumnarExchange (26) - : : : : +- CometHashAggregate (25) - : : : : +- CometProject (24) - : : : : +- CometBroadcastHashJoin (23) - : : : : :- CometProject (19) - : : : : : +- CometBroadcastHashJoin (18) - : : : : : :- CometProject (13) - : : : : : : +- CometBroadcastHashJoin (12) - : : : : : : :- CometUnion (7) - : : : : : : : :- CometProject (3) - : : : : : : : : +- CometFilter (2) - : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : +- CometProject (6) - : : : : : : : +- CometFilter (5) - : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (4) - : : : : : : +- CometBroadcastExchange (11) - : : : : : : +- CometProject (10) - : : : : : : +- CometFilter (9) - : : : : : : +- CometScan parquet spark_catalog.default.item (8) - : : : : : +- CometBroadcastExchange (17) - : : : : : +- CometProject (16) - : : : : : +- CometFilter (15) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (14) - : : : : +- CometBroadcastExchange (22) - : : : : +- CometFilter (21) - : : : : +- CometScan parquet spark_catalog.default.customer (20) - : : : +- CometBroadcastExchange (30) - : : : +- CometFilter (29) - : : : +- CometScan parquet spark_catalog.default.store_sales (28) - : : +- CometBroadcastExchange (35) - : : +- CometFilter (34) - : : +- CometScan parquet spark_catalog.default.customer_address (33) - : +- CometBroadcastExchange (40) - : +- CometFilter (39) - : +- CometScan parquet spark_catalog.default.store (38) - +- CometBroadcastExchange (46) - +- CometProject (45) - +- CometFilter (44) - +- CometScan parquet spark_catalog.default.date_dim (43) +TakeOrderedAndProject (56) ++- * HashAggregate (55) + +- Exchange (54) + +- * HashAggregate (53) + +- * HashAggregate (52) + +- Exchange (51) + +- * HashAggregate (50) + +- * ColumnarToRow (49) + +- CometProject (48) + +- CometBroadcastHashJoin (47) + :- CometProject (42) + : +- CometBroadcastHashJoin (41) + : :- CometProject (37) + : : +- CometBroadcastHashJoin (36) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometHashAggregate (27) + : : : : +- CometExchange (26) + : : : : +- CometHashAggregate (25) + : : : : +- CometProject (24) + : : : : +- CometBroadcastHashJoin (23) + : : : : :- CometProject (19) + : : : : : +- CometBroadcastHashJoin (18) + : : : : : :- CometProject (13) + : : : : : : +- CometBroadcastHashJoin (12) + : : : : : : :- CometUnion (7) + : : : : : : : :- CometProject (3) + : : : : : : : : +- CometFilter (2) + : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : +- CometProject (6) + : : : : : : : +- CometFilter (5) + : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (4) + : : : : : : +- CometBroadcastExchange (11) + : : : : : : +- CometProject (10) + : : : : : : +- CometFilter (9) + : : : : : : +- CometScan parquet spark_catalog.default.item (8) + : : : : : +- CometBroadcastExchange (17) + : : : : : +- CometProject (16) + : : : : : +- CometFilter (15) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (14) + : : : : +- CometBroadcastExchange (22) + : : : : +- CometFilter (21) + : : : : +- CometScan parquet spark_catalog.default.customer (20) + : : : +- CometBroadcastExchange (30) + : : : +- CometFilter (29) + : : : +- CometScan parquet spark_catalog.default.store_sales (28) + : : +- CometBroadcastExchange (35) + : : +- CometFilter (34) + : : +- CometScan parquet spark_catalog.default.customer_address (33) + : +- CometBroadcastExchange (40) + : +- CometFilter (39) + : +- CometScan parquet spark_catalog.default.store (38) + +- CometBroadcastExchange (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan parquet spark_catalog.default.date_dim (43) (1) Scan parquet spark_catalog.default.catalog_sales @@ -182,9 +178,9 @@ Input [2]: [c_customer_sk#20, c_current_addr_sk#21] Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] Functions: [] -(26) CometColumnarExchange +(26) CometExchange Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: hashpartitioning(c_customer_sk#20, c_current_addr_sk#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_sk#20, c_current_addr_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [2]: [c_customer_sk#20, c_current_addr_sk#21] @@ -302,201 +298,189 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#23))] Aggregate Attributes [1]: [sum#37] Results [2]: [c_customer_sk#20, sum#38] -(51) RowToColumnar +(51) Exchange Input [2]: [c_customer_sk#20, sum#38] +Arguments: hashpartitioning(c_customer_sk#20, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(52) CometColumnarExchange -Input [2]: [c_customer_sk#20, sum#38] -Arguments: hashpartitioning(c_customer_sk#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(53) ColumnarToRow [codegen id : 2] -Input [2]: [c_customer_sk#20, sum#38] - -(54) HashAggregate [codegen id : 2] +(52) HashAggregate [codegen id : 2] Input [2]: [c_customer_sk#20, sum#38] Keys [1]: [c_customer_sk#20] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#23))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#23))#39] Results [1]: [cast((MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#23))#39,17,2) / 50) as int) AS segment#40] -(55) HashAggregate [codegen id : 2] +(53) HashAggregate [codegen id : 2] Input [1]: [segment#40] Keys [1]: [segment#40] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#41] Results [2]: [segment#40, count#42] -(56) RowToColumnar -Input [2]: [segment#40, count#42] - -(57) CometColumnarExchange -Input [2]: [segment#40, count#42] -Arguments: hashpartitioning(segment#40, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(58) ColumnarToRow [codegen id : 3] +(54) Exchange Input [2]: [segment#40, count#42] +Arguments: hashpartitioning(segment#40, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(59) HashAggregate [codegen id : 3] +(55) HashAggregate [codegen id : 3] Input [2]: [segment#40, count#42] Keys [1]: [segment#40] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#43] Results [3]: [segment#40, count(1)#43 AS num_customers#44, (segment#40 * 50) AS segment_base#45] -(60) TakeOrderedAndProject +(56) TakeOrderedAndProject Input [3]: [segment#40, num_customers#44, segment_base#45] Arguments: 100, [segment#40 ASC NULLS FIRST, num_customers#44 ASC NULLS FIRST], [segment#40, num_customers#44, segment_base#45] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (65) -+- * ColumnarToRow (64) - +- CometProject (63) - +- CometFilter (62) - +- CometScan parquet spark_catalog.default.date_dim (61) +BroadcastExchange (61) ++- * ColumnarToRow (60) + +- CometProject (59) + +- CometFilter (58) + +- CometScan parquet spark_catalog.default.date_dim (57) -(61) Scan parquet spark_catalog.default.date_dim +(57) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#17, d_year#18, d_moy#19] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,12), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(62) CometFilter +(58) CometFilter Input [3]: [d_date_sk#17, d_year#18, d_moy#19] Condition : ((((isnotnull(d_moy#19) AND isnotnull(d_year#18)) AND (d_moy#19 = 12)) AND (d_year#18 = 1998)) AND isnotnull(d_date_sk#17)) -(63) CometProject +(59) CometProject Input [3]: [d_date_sk#17, d_year#18, d_moy#19] Arguments: [d_date_sk#17], [d_date_sk#17] -(64) ColumnarToRow [codegen id : 1] +(60) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#17] -(65) BroadcastExchange +(61) BroadcastExchange Input [1]: [d_date_sk#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] Subquery:2 Hosting operator id = 4 Hosting Expression = ws_sold_date_sk#10 IN dynamicpruning#4 Subquery:3 Hosting operator id = 28 Hosting Expression = ss_sold_date_sk#24 IN dynamicpruning#25 -BroadcastExchange (70) -+- * ColumnarToRow (69) - +- CometProject (68) - +- CometFilter (67) - +- CometScan parquet spark_catalog.default.date_dim (66) +BroadcastExchange (66) ++- * ColumnarToRow (65) + +- CometProject (64) + +- CometFilter (63) + +- CometScan parquet spark_catalog.default.date_dim (62) -(66) Scan parquet spark_catalog.default.date_dim +(62) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#31, d_month_seq#32] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,ScalarSubquery#33), LessThanOrEqual(d_month_seq,ScalarSubquery#34), IsNotNull(d_date_sk)] ReadSchema: struct -(67) CometFilter +(63) CometFilter Input [2]: [d_date_sk#31, d_month_seq#32] Condition : (((isnotnull(d_month_seq#32) AND (d_month_seq#32 >= ReusedSubquery Subquery scalar-subquery#33, [id=#35])) AND (d_month_seq#32 <= ReusedSubquery Subquery scalar-subquery#34, [id=#36])) AND isnotnull(d_date_sk#31)) -(68) CometProject +(64) CometProject Input [2]: [d_date_sk#31, d_month_seq#32] Arguments: [d_date_sk#31], [d_date_sk#31] -(69) ColumnarToRow [codegen id : 1] +(65) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#31] -(70) BroadcastExchange +(66) BroadcastExchange Input [1]: [d_date_sk#31] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -Subquery:4 Hosting operator id = 67 Hosting Expression = ReusedSubquery Subquery scalar-subquery#33, [id=#35] +Subquery:4 Hosting operator id = 63 Hosting Expression = ReusedSubquery Subquery scalar-subquery#33, [id=#35] -Subquery:5 Hosting operator id = 67 Hosting Expression = ReusedSubquery Subquery scalar-subquery#34, [id=#36] +Subquery:5 Hosting operator id = 63 Hosting Expression = ReusedSubquery Subquery scalar-subquery#34, [id=#36] -Subquery:6 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquery#33, [id=#35] -* ColumnarToRow (77) -+- CometHashAggregate (76) - +- CometColumnarExchange (75) - +- CometHashAggregate (74) - +- CometProject (73) - +- CometFilter (72) - +- CometScan parquet spark_catalog.default.date_dim (71) +Subquery:6 Hosting operator id = 62 Hosting Expression = Subquery scalar-subquery#33, [id=#35] +* ColumnarToRow (73) ++- CometHashAggregate (72) + +- CometExchange (71) + +- CometHashAggregate (70) + +- CometProject (69) + +- CometFilter (68) + +- CometScan parquet spark_catalog.default.date_dim (67) -(71) Scan parquet spark_catalog.default.date_dim +(67) Scan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#46, d_year#47, d_moy#48] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(72) CometFilter +(68) CometFilter Input [3]: [d_month_seq#46, d_year#47, d_moy#48] Condition : (((isnotnull(d_year#47) AND isnotnull(d_moy#48)) AND (d_year#47 = 1998)) AND (d_moy#48 = 12)) -(73) CometProject +(69) CometProject Input [3]: [d_month_seq#46, d_year#47, d_moy#48] Arguments: [(d_month_seq + 1)#49], [(d_month_seq#46 + 1) AS (d_month_seq + 1)#49] -(74) CometHashAggregate +(70) CometHashAggregate Input [1]: [(d_month_seq + 1)#49] Keys [1]: [(d_month_seq + 1)#49] Functions: [] -(75) CometColumnarExchange +(71) CometExchange Input [1]: [(d_month_seq + 1)#49] -Arguments: hashpartitioning((d_month_seq + 1)#49, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning((d_month_seq + 1)#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(76) CometHashAggregate +(72) CometHashAggregate Input [1]: [(d_month_seq + 1)#49] Keys [1]: [(d_month_seq + 1)#49] Functions: [] -(77) ColumnarToRow [codegen id : 1] +(73) ColumnarToRow [codegen id : 1] Input [1]: [(d_month_seq + 1)#49] -Subquery:7 Hosting operator id = 66 Hosting Expression = Subquery scalar-subquery#34, [id=#36] -* ColumnarToRow (84) -+- CometHashAggregate (83) - +- CometColumnarExchange (82) - +- CometHashAggregate (81) - +- CometProject (80) - +- CometFilter (79) - +- CometScan parquet spark_catalog.default.date_dim (78) +Subquery:7 Hosting operator id = 62 Hosting Expression = Subquery scalar-subquery#34, [id=#36] +* ColumnarToRow (80) ++- CometHashAggregate (79) + +- CometExchange (78) + +- CometHashAggregate (77) + +- CometProject (76) + +- CometFilter (75) + +- CometScan parquet spark_catalog.default.date_dim (74) -(78) Scan parquet spark_catalog.default.date_dim +(74) Scan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#50, d_year#51, d_moy#52] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,12)] ReadSchema: struct -(79) CometFilter +(75) CometFilter Input [3]: [d_month_seq#50, d_year#51, d_moy#52] Condition : (((isnotnull(d_year#51) AND isnotnull(d_moy#52)) AND (d_year#51 = 1998)) AND (d_moy#52 = 12)) -(80) CometProject +(76) CometProject Input [3]: [d_month_seq#50, d_year#51, d_moy#52] Arguments: [(d_month_seq + 3)#53], [(d_month_seq#50 + 3) AS (d_month_seq + 3)#53] -(81) CometHashAggregate +(77) CometHashAggregate Input [1]: [(d_month_seq + 3)#53] Keys [1]: [(d_month_seq + 3)#53] Functions: [] -(82) CometColumnarExchange +(78) CometExchange Input [1]: [(d_month_seq + 3)#53] -Arguments: hashpartitioning((d_month_seq + 3)#53, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning((d_month_seq + 3)#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(83) CometHashAggregate +(79) CometHashAggregate Input [1]: [(d_month_seq + 3)#53] Keys [1]: [(d_month_seq + 3)#53] Functions: [] -(84) ColumnarToRow [codegen id : 1] +(80) ColumnarToRow [codegen id : 1] Input [1]: [(d_month_seq + 3)#53] Subquery:8 Hosting operator id = 44 Hosting Expression = ReusedSubquery Subquery scalar-subquery#33, [id=#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt index e1696b97d..1dddb45ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q54/simplified.txt @@ -1,109 +1,105 @@ TakeOrderedAndProject [segment,num_customers,segment_base] WholeStageCodegen (3) HashAggregate [segment,count] [count(1),num_customers,segment_base,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [segment] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [segment] [count,count] - HashAggregate [c_customer_sk,sum] [sum(UnscaledValue(ss_ext_sales_price)),segment,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_sk] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [c_customer_sk,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_sk,ss_ext_sales_price] - CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state,s_county,s_state] - CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state] - CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk,ca_address_sk,ca_county,ca_state] - CometProject [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometHashAggregate [c_customer_sk,c_current_addr_sk] - CometColumnarExchange [c_customer_sk,c_current_addr_sk] #3 - CometHashAggregate [c_customer_sk,c_current_addr_sk] - CometProject [c_customer_sk,c_current_addr_sk] - CometBroadcastHashJoin [customer_sk,c_customer_sk,c_current_addr_sk] - CometProject [customer_sk] - CometBroadcastHashJoin [sold_date_sk,customer_sk,d_date_sk] - CometProject [sold_date_sk,customer_sk] - CometBroadcastHashJoin [sold_date_sk,customer_sk,item_sk,i_item_sk] - CometUnion [sold_date_sk,customer_sk,item_sk] - CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [i_item_sk] #5 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #7 - CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #8 - CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #9 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [(d_month_seq + 1)] #3 - ReusedSubquery [(d_month_seq + 3)] #4 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - Subquery #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [(d_month_seq + 1)] - CometColumnarExchange [(d_month_seq + 1)] #10 - CometHashAggregate [(d_month_seq + 1)] - CometProject [d_month_seq] [(d_month_seq + 1)] - CometFilter [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - Subquery #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [(d_month_seq + 3)] - CometColumnarExchange [(d_month_seq + 3)] #11 - CometHashAggregate [(d_month_seq + 3)] - CometProject [d_month_seq] [(d_month_seq + 3)] - CometFilter [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #12 - CometFilter [ca_address_sk,ca_county,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] - CometBroadcastExchange [s_county,s_state] #13 - CometFilter [s_county,s_state] - CometScan parquet spark_catalog.default.store [s_county,s_state] - CometBroadcastExchange [d_date_sk] #14 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [(d_month_seq + 1)] #3 - ReusedSubquery [(d_month_seq + 3)] #4 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - ReusedSubquery [(d_month_seq + 1)] #3 - ReusedSubquery [(d_month_seq + 3)] #4 + InputAdapter + Exchange [segment] #1 + WholeStageCodegen (2) + HashAggregate [segment] [count,count] + HashAggregate [c_customer_sk,sum] [sum(UnscaledValue(ss_ext_sales_price)),segment,sum] + InputAdapter + Exchange [c_customer_sk] #2 + WholeStageCodegen (1) + HashAggregate [c_customer_sk,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state,s_county,s_state] + CometProject [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,ca_county,ca_state] + CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk,ca_address_sk,ca_county,ca_state] + CometProject [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometHashAggregate [c_customer_sk,c_current_addr_sk] + CometExchange [c_customer_sk,c_current_addr_sk] #3 + CometHashAggregate [c_customer_sk,c_current_addr_sk] + CometProject [c_customer_sk,c_current_addr_sk] + CometBroadcastHashJoin [customer_sk,c_customer_sk,c_current_addr_sk] + CometProject [customer_sk] + CometBroadcastHashJoin [sold_date_sk,customer_sk,d_date_sk] + CometProject [sold_date_sk,customer_sk] + CometBroadcastHashJoin [sold_date_sk,customer_sk,item_sk,i_item_sk] + CometUnion [sold_date_sk,customer_sk,item_sk] + CometProject [cs_sold_date_sk,cs_bill_customer_sk,cs_item_sk] [sold_date_sk,customer_sk,item_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [ws_sold_date_sk,ws_bill_customer_sk,ws_item_sk] [sold_date_sk,customer_sk,item_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [i_item_sk] #5 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #7 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] #8 + CometFilter [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #9 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + ReusedSubquery [(d_month_seq + 1)] #3 + ReusedSubquery [(d_month_seq + 3)] #4 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + Subquery #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 1)] + CometExchange [(d_month_seq + 1)] #10 + CometHashAggregate [(d_month_seq + 1)] + CometProject [d_month_seq] [(d_month_seq + 1)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + Subquery #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometHashAggregate [(d_month_seq + 3)] + CometExchange [(d_month_seq + 3)] #11 + CometHashAggregate [(d_month_seq + 3)] + CometProject [d_month_seq] [(d_month_seq + 3)] + CometFilter [d_month_seq,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state] #12 + CometFilter [ca_address_sk,ca_county,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state] + CometBroadcastExchange [s_county,s_state] #13 + CometFilter [s_county,s_state] + CometScan parquet spark_catalog.default.store [s_county,s_state] + CometBroadcastExchange [d_date_sk] #14 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + ReusedSubquery [(d_month_seq + 1)] #3 + ReusedSubquery [(d_month_seq + 3)] #4 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + ReusedSubquery [(d_month_seq + 1)] #3 + ReusedSubquery [(d_month_seq + 3)] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt index 36e8ac073..dcd32a96a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/explain.txt @@ -1,25 +1,23 @@ == Physical Plan == -TakeOrderedAndProject (21) -+- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometColumnarExchange (18) - +- RowToColumnar (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometProject (14) - +- CometBroadcastHashJoin (13) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometProject (3) - : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.date_dim (1) - : +- CometBroadcastExchange (6) - : +- CometFilter (5) - : +- CometScan parquet spark_catalog.default.store_sales (4) - +- CometBroadcastExchange (12) - +- CometProject (11) - +- CometFilter (10) - +- CometScan parquet spark_catalog.default.item (9) +TakeOrderedAndProject (19) ++- * HashAggregate (18) + +- Exchange (17) + +- * HashAggregate (16) + +- * ColumnarToRow (15) + +- CometProject (14) + +- CometBroadcastHashJoin (13) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.date_dim (1) + : +- CometBroadcastExchange (6) + : +- CometFilter (5) + : +- CometScan parquet spark_catalog.default.store_sales (4) + +- CometBroadcastExchange (12) + +- CometProject (11) + +- CometFilter (10) + +- CometScan parquet spark_catalog.default.item (9) (1) Scan parquet spark_catalog.default.date_dim @@ -100,24 +98,18 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum#11] Results [3]: [i_brand#9, i_brand_id#8, sum#12] -(17) RowToColumnar +(17) Exchange Input [3]: [i_brand#9, i_brand_id#8, sum#12] +Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(18) CometColumnarExchange -Input [3]: [i_brand#9, i_brand_id#8, sum#12] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(19) ColumnarToRow [codegen id : 2] -Input [3]: [i_brand#9, i_brand_id#8, sum#12] - -(20) HashAggregate [codegen id : 2] +(18) HashAggregate [codegen id : 2] Input [3]: [i_brand#9, i_brand_id#8, sum#12] Keys [2]: [i_brand#9, i_brand_id#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#13] Results [3]: [i_brand_id#8 AS brand_id#14, i_brand#9 AS brand#15, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#13,17,2) AS ext_price#16] -(21) TakeOrderedAndProject +(19) TakeOrderedAndProject Input [3]: [brand_id#14, brand#15, ext_price#16] Arguments: 100, [ext_price#16 DESC NULLS LAST, brand_id#14 ASC NULLS FIRST], [brand_id#14, brand#15, ext_price#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt index 290d5cb88..2461ab830 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q55/simplified.txt @@ -1,25 +1,23 @@ TakeOrderedAndProject [ext_price,brand_id,brand] WholeStageCodegen (2) HashAggregate [i_brand,i_brand_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),brand_id,brand,ext_price,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_brand,i_brand_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_brand_id,i_brand] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + InputAdapter + Exchange [i_brand,i_brand_id] #1 + WholeStageCodegen (1) + HashAggregate [i_brand,i_brand_id,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,i_brand_id,i_brand] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [d_date_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] #2 + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt index b3e0794df..bbfd9dcaf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/explain.txt @@ -1,75 +1,67 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * HashAggregate (70) - +- * ColumnarToRow (69) - +- CometColumnarExchange (68) - +- RowToColumnar (67) - +- * HashAggregate (66) - +- Union (65) - :- * HashAggregate (30) - : +- * ColumnarToRow (29) - : +- CometColumnarExchange (28) - : +- RowToColumnar (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer_address (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.item (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometScan parquet spark_catalog.default.item (17) - :- * HashAggregate (47) - : +- * ColumnarToRow (46) - : +- CometColumnarExchange (45) - : +- RowToColumnar (44) - : +- * HashAggregate (43) - : +- * ColumnarToRow (42) - : +- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (38) - : : +- CometBroadcastHashJoin (37) - : : :- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometFilter (32) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (31) - : : : +- ReusedExchange (33) - : : +- ReusedExchange (36) - : +- ReusedExchange (39) - +- * HashAggregate (64) - +- * ColumnarToRow (63) - +- CometColumnarExchange (62) - +- RowToColumnar (61) - +- * HashAggregate (60) - +- * ColumnarToRow (59) - +- CometProject (58) - +- CometBroadcastHashJoin (57) - :- CometProject (55) - : +- CometBroadcastHashJoin (54) - : :- CometProject (52) - : : +- CometBroadcastHashJoin (51) - : : :- CometFilter (49) - : : : +- CometScan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (50) - : +- ReusedExchange (53) - +- ReusedExchange (56) +TakeOrderedAndProject (63) ++- * HashAggregate (62) + +- Exchange (61) + +- * HashAggregate (60) + +- Union (59) + :- * HashAggregate (28) + : +- Exchange (27) + : +- * HashAggregate (26) + : +- * ColumnarToRow (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan parquet spark_catalog.default.item (17) + :- * HashAggregate (43) + : +- Exchange (42) + : +- * HashAggregate (41) + : +- * ColumnarToRow (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- * HashAggregate (58) + +- Exchange (57) + +- * HashAggregate (56) + +- * ColumnarToRow (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometProject (48) + : : +- CometBroadcastHashJoin (47) + : : :- CometFilter (45) + : : : +- CometScan parquet spark_catalog.default.web_sales (44) + : : +- ReusedExchange (46) + : +- ReusedExchange (49) + +- ReusedExchange (52) (1) Scan parquet spark_catalog.default.store_sales @@ -198,24 +190,18 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum#15] Results [2]: [i_item_id#12, sum#16] -(27) RowToColumnar +(27) Exchange Input [2]: [i_item_id#12, sum#16] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(28) CometColumnarExchange -Input [2]: [i_item_id#12, sum#16] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(29) ColumnarToRow [codegen id : 2] -Input [2]: [i_item_id#12, sum#16] - -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [2]: [i_item_id#12, sum#16] Keys [1]: [i_item_id#12] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#17] Results [2]: [i_item_id#12, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#17,17,2) AS total_sales#18] -(31) Scan parquet spark_catalog.default.catalog_sales +(29) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22] Batched: true Location: InMemoryFileIndex [] @@ -223,74 +209,68 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#22), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(32) CometFilter +(30) CometFilter Input [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22] Condition : (isnotnull(cs_bill_addr_sk#19) AND isnotnull(cs_item_sk#20)) -(33) ReusedExchange [Reuses operator id: 6] +(31) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#24] -(34) CometBroadcastHashJoin +(32) CometBroadcastHashJoin Left output [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22] Right output [1]: [d_date_sk#24] Arguments: [cs_sold_date_sk#22], [d_date_sk#24], Inner, BuildRight -(35) CometProject +(33) CometProject Input [5]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22, d_date_sk#24] Arguments: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21], [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21] -(36) ReusedExchange [Reuses operator id: 12] +(34) ReusedExchange [Reuses operator id: 12] Output [1]: [ca_address_sk#25] -(37) CometBroadcastHashJoin +(35) CometBroadcastHashJoin Left output [3]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21] Right output [1]: [ca_address_sk#25] Arguments: [cs_bill_addr_sk#19], [ca_address_sk#25], Inner, BuildRight -(38) CometProject +(36) CometProject Input [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, ca_address_sk#25] Arguments: [cs_item_sk#20, cs_ext_sales_price#21], [cs_item_sk#20, cs_ext_sales_price#21] -(39) ReusedExchange [Reuses operator id: 22] +(37) ReusedExchange [Reuses operator id: 22] Output [2]: [i_item_sk#26, i_item_id#27] -(40) CometBroadcastHashJoin +(38) CometBroadcastHashJoin Left output [2]: [cs_item_sk#20, cs_ext_sales_price#21] Right output [2]: [i_item_sk#26, i_item_id#27] Arguments: [cs_item_sk#20], [i_item_sk#26], Inner, BuildRight -(41) CometProject +(39) CometProject Input [4]: [cs_item_sk#20, cs_ext_sales_price#21, i_item_sk#26, i_item_id#27] Arguments: [cs_ext_sales_price#21, i_item_id#27], [cs_ext_sales_price#21, i_item_id#27] -(42) ColumnarToRow [codegen id : 3] +(40) ColumnarToRow [codegen id : 3] Input [2]: [cs_ext_sales_price#21, i_item_id#27] -(43) HashAggregate [codegen id : 3] +(41) HashAggregate [codegen id : 3] Input [2]: [cs_ext_sales_price#21, i_item_id#27] Keys [1]: [i_item_id#27] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#21))] Aggregate Attributes [1]: [sum#28] Results [2]: [i_item_id#27, sum#29] -(44) RowToColumnar -Input [2]: [i_item_id#27, sum#29] - -(45) CometColumnarExchange -Input [2]: [i_item_id#27, sum#29] -Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(46) ColumnarToRow [codegen id : 4] +(42) Exchange Input [2]: [i_item_id#27, sum#29] +Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(47) HashAggregate [codegen id : 4] +(43) HashAggregate [codegen id : 4] Input [2]: [i_item_id#27, sum#29] Keys [1]: [i_item_id#27] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#21))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#21))#30] Results [2]: [i_item_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#21))#30,17,2) AS total_sales#31] -(48) Scan parquet spark_catalog.default.web_sales +(44) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35] Batched: true Location: InMemoryFileIndex [] @@ -298,137 +278,125 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#35), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(49) CometFilter +(45) CometFilter Input [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35] Condition : (isnotnull(ws_bill_addr_sk#33) AND isnotnull(ws_item_sk#32)) -(50) ReusedExchange [Reuses operator id: 6] +(46) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#37] -(51) CometBroadcastHashJoin +(47) CometBroadcastHashJoin Left output [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35] Right output [1]: [d_date_sk#37] Arguments: [ws_sold_date_sk#35], [d_date_sk#37], Inner, BuildRight -(52) CometProject +(48) CometProject Input [5]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35, d_date_sk#37] Arguments: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34], [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34] -(53) ReusedExchange [Reuses operator id: 12] +(49) ReusedExchange [Reuses operator id: 12] Output [1]: [ca_address_sk#38] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [3]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34] Right output [1]: [ca_address_sk#38] Arguments: [ws_bill_addr_sk#33], [ca_address_sk#38], Inner, BuildRight -(55) CometProject +(51) CometProject Input [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ca_address_sk#38] Arguments: [ws_item_sk#32, ws_ext_sales_price#34], [ws_item_sk#32, ws_ext_sales_price#34] -(56) ReusedExchange [Reuses operator id: 22] +(52) ReusedExchange [Reuses operator id: 22] Output [2]: [i_item_sk#39, i_item_id#40] -(57) CometBroadcastHashJoin +(53) CometBroadcastHashJoin Left output [2]: [ws_item_sk#32, ws_ext_sales_price#34] Right output [2]: [i_item_sk#39, i_item_id#40] Arguments: [ws_item_sk#32], [i_item_sk#39], Inner, BuildRight -(58) CometProject +(54) CometProject Input [4]: [ws_item_sk#32, ws_ext_sales_price#34, i_item_sk#39, i_item_id#40] Arguments: [ws_ext_sales_price#34, i_item_id#40], [ws_ext_sales_price#34, i_item_id#40] -(59) ColumnarToRow [codegen id : 5] +(55) ColumnarToRow [codegen id : 5] Input [2]: [ws_ext_sales_price#34, i_item_id#40] -(60) HashAggregate [codegen id : 5] +(56) HashAggregate [codegen id : 5] Input [2]: [ws_ext_sales_price#34, i_item_id#40] Keys [1]: [i_item_id#40] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#34))] Aggregate Attributes [1]: [sum#41] Results [2]: [i_item_id#40, sum#42] -(61) RowToColumnar +(57) Exchange Input [2]: [i_item_id#40, sum#42] +Arguments: hashpartitioning(i_item_id#40, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(62) CometColumnarExchange -Input [2]: [i_item_id#40, sum#42] -Arguments: hashpartitioning(i_item_id#40, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(63) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_id#40, sum#42] - -(64) HashAggregate [codegen id : 6] +(58) HashAggregate [codegen id : 6] Input [2]: [i_item_id#40, sum#42] Keys [1]: [i_item_id#40] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#34))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#34))#43] Results [2]: [i_item_id#40, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#34))#43,17,2) AS total_sales#44] -(65) Union +(59) Union -(66) HashAggregate [codegen id : 7] +(60) HashAggregate [codegen id : 7] Input [2]: [i_item_id#12, total_sales#18] Keys [1]: [i_item_id#12] Functions [1]: [partial_sum(total_sales#18)] Aggregate Attributes [2]: [sum#45, isEmpty#46] Results [3]: [i_item_id#12, sum#47, isEmpty#48] -(67) RowToColumnar -Input [3]: [i_item_id#12, sum#47, isEmpty#48] - -(68) CometColumnarExchange -Input [3]: [i_item_id#12, sum#47, isEmpty#48] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(69) ColumnarToRow [codegen id : 8] +(61) Exchange Input [3]: [i_item_id#12, sum#47, isEmpty#48] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(70) HashAggregate [codegen id : 8] +(62) HashAggregate [codegen id : 8] Input [3]: [i_item_id#12, sum#47, isEmpty#48] Keys [1]: [i_item_id#12] Functions [1]: [sum(total_sales#18)] Aggregate Attributes [1]: [sum(total_sales#18)#49] Results [2]: [i_item_id#12, sum(total_sales#18)#49 AS total_sales#50] -(71) TakeOrderedAndProject +(63) TakeOrderedAndProject Input [2]: [i_item_id#12, total_sales#50] Arguments: 100, [total_sales#50 ASC NULLS FIRST], [i_item_id#12, total_sales#50] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (76) -+- * ColumnarToRow (75) - +- CometProject (74) - +- CometFilter (73) - +- CometScan parquet spark_catalog.default.date_dim (72) +BroadcastExchange (68) ++- * ColumnarToRow (67) + +- CometProject (66) + +- CometFilter (65) + +- CometScan parquet spark_catalog.default.date_dim (64) -(72) Scan parquet spark_catalog.default.date_dim +(64) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,2), IsNotNull(d_date_sk)] ReadSchema: struct -(73) CometFilter +(65) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_moy#8] Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 2001)) AND (d_moy#8 = 2)) AND isnotnull(d_date_sk#6)) -(74) CometProject +(66) CometProject Input [3]: [d_date_sk#6, d_year#7, d_moy#8] Arguments: [d_date_sk#6], [d_date_sk#6] -(75) ColumnarToRow [codegen id : 1] +(67) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(76) BroadcastExchange +(68) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -Subquery:2 Hosting operator id = 31 Hosting Expression = cs_sold_date_sk#22 IN dynamicpruning#5 +Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#22 IN dynamicpruning#5 -Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#35 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#35 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt index e1f157a88..c4ce35e15 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q56/simplified.txt @@ -1,97 +1,89 @@ TakeOrderedAndProject [total_sales,i_item_id] WholeStageCodegen (8) HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #1 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #6 - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #7 - CometProject [i_item_id] - CometFilter [i_item_id,i_color] - CometScan parquet spark_catalog.default.item [i_item_id,i_color] - WholeStageCodegen (4) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #8 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #9 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] - CometProject [ws_item_sk,ws_ext_sales_price] - CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] - CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + InputAdapter + Exchange [i_item_id] #1 + WholeStageCodegen (7) + HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (2) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] + InputAdapter + Exchange [i_item_id] #2 + WholeStageCodegen (1) + HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #5 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #7 + CometProject [i_item_id] + CometFilter [i_item_id,i_color] + CometScan parquet spark_catalog.default.item [i_item_id,i_color] + WholeStageCodegen (4) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] + InputAdapter + Exchange [i_item_id] #8 + WholeStageCodegen (3) + HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [ca_address_sk] #5 + ReusedExchange [i_item_sk,i_item_id] #6 + WholeStageCodegen (6) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] + InputAdapter + Exchange [i_item_id] #9 + WholeStageCodegen (5) + HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [ca_address_sk] #5 + ReusedExchange [i_item_sk,i_item_id] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt index 324701330..76b1adf1a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/explain.txt @@ -1,57 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (53) -+- * Project (52) - +- * BroadcastHashJoin Inner BuildRight (51) - :- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (32) - : : +- * Filter (31) - : : +- Window (30) - : : +- * Filter (29) - : : +- Window (28) - : : +- * ColumnarToRow (27) - : : +- CometSort (26) - : : +- CometColumnarExchange (25) - : : +- RowToColumnar (24) - : : +- * HashAggregate (23) - : : +- * ColumnarToRow (22) - : : +- CometColumnarExchange (21) - : : +- RowToColumnar (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.item (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.call_center (13) - : +- BroadcastExchange (42) - : +- * Project (41) - : +- Window (40) - : +- * ColumnarToRow (39) - : +- CometSort (38) - : +- CometColumnarExchange (37) - : +- RowToColumnar (36) - : +- * HashAggregate (35) - : +- * ColumnarToRow (34) - : +- ReusedExchange (33) - +- BroadcastExchange (50) - +- * Project (49) - +- Window (48) - +- * ColumnarToRow (47) - +- CometSort (46) - +- ReusedExchange (45) +TakeOrderedAndProject (45) ++- * Project (44) + +- * BroadcastHashJoin Inner BuildRight (43) + :- * Project (37) + : +- * BroadcastHashJoin Inner BuildRight (36) + : :- * Project (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * Filter (25) + : : +- Window (24) + : : +- * Sort (23) + : : +- Exchange (22) + : : +- * HashAggregate (21) + : : +- Exchange (20) + : : +- * HashAggregate (19) + : : +- * ColumnarToRow (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.call_center (13) + : +- BroadcastExchange (35) + : +- * Project (34) + : +- Window (33) + : +- * Sort (32) + : +- Exchange (31) + : +- * HashAggregate (30) + : +- ReusedExchange (29) + +- BroadcastExchange (42) + +- * Project (41) + +- Window (40) + +- * Sort (39) + +- ReusedExchange (38) (1) Scan parquet spark_catalog.default.item @@ -148,166 +140,142 @@ Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] Aggregate Attributes [1]: [sum#14] Results [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#15] -(20) RowToColumnar +(20) Exchange Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#15] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(21) CometColumnarExchange -Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(22) ColumnarToRow [codegen id : 2] -Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#15] - -(23) HashAggregate [codegen id : 2] +(21) HashAggregate [codegen id : 2] Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#15] Keys [5]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#16] Results [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#16,17,2) AS _w0#18] -(24) RowToColumnar -Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18] - -(25) CometColumnarExchange +(22) Exchange Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(26) CometSort +(23) Sort [codegen id : 3] Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18] -Arguments: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#13 ASC NULLS FIRST, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST] +Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#13 ASC NULLS FIRST, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST], false, 0 -(27) ColumnarToRow [codegen id : 3] -Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18] - -(28) Window +(24) Window Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18] Arguments: [rank(d_year#10, d_moy#11) windowspecdefinition(i_category#3, i_brand#2, cc_name#13, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#3, i_brand#2, cc_name#13], [d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST] -(29) Filter [codegen id : 4] +(25) Filter [codegen id : 4] Input [8]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18, rn#19] Condition : (isnotnull(d_year#10) AND (d_year#10 = 1999)) -(30) Window +(26) Window Input [8]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18, rn#19] Arguments: [avg(_w0#18) windowspecdefinition(i_category#3, i_brand#2, cc_name#13, d_year#10, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#3, i_brand#2, cc_name#13, d_year#10] -(31) Filter [codegen id : 13] +(27) Filter [codegen id : 13] Input [9]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) -(32) Project [codegen id : 13] +(28) Project [codegen id : 13] Output [8]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, avg_monthly_sales#20, rn#19] Input [9]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -(33) ReusedExchange [Reuses operator id: 21] +(29) ReusedExchange [Reuses operator id: 20] Output [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] -(34) ColumnarToRow [codegen id : 6] -Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] - -(35) HashAggregate [codegen id : 6] +(30) HashAggregate [codegen id : 6] Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] Keys [5]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25] Functions [1]: [sum(UnscaledValue(cs_sales_price#27))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#27))#16] Results [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, MakeDecimal(sum(UnscaledValue(cs_sales_price#27))#16,17,2) AS sum_sales#28] -(36) RowToColumnar -Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] - -(37) CometColumnarExchange +(31) Exchange Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] -Arguments: hashpartitioning(i_category#21, i_brand#22, cc_name#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_category#21, i_brand#22, cc_name#23, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(38) CometSort +(32) Sort [codegen id : 7] Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] -Arguments: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28], [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, cc_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] +Arguments: [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, cc_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST], false, 0 -(39) ColumnarToRow [codegen id : 7] -Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] - -(40) Window +(33) Window Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#21, i_brand#22, cc_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, cc_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] -(41) Project [codegen id : 8] +(34) Project [codegen id : 8] Output [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] Input [7]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28, rn#29] -(42) BroadcastExchange +(35) BroadcastExchange Input [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(43) BroadcastHashJoin [codegen id : 13] +(36) BroadcastHashJoin [codegen id : 13] Left keys [4]: [i_category#3, i_brand#2, cc_name#13, rn#19] Right keys [4]: [i_category#21, i_brand#22, cc_name#23, (rn#29 + 1)] Join type: Inner Join condition: None -(44) Project [codegen id : 13] +(37) Project [codegen id : 13] Output [9]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28] Input [13]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] -(45) ReusedExchange [Reuses operator id: 37] +(38) ReusedExchange [Reuses operator id: 31] Output [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] -(46) CometSort +(39) Sort [codegen id : 11] Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] -Arguments: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35], [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, cc_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] +Arguments: [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, cc_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST], false, 0 -(47) ColumnarToRow [codegen id : 11] -Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] - -(48) Window +(40) Window Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#30, i_brand#31, cc_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#30, i_brand#31, cc_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(49) Project [codegen id : 12] +(41) Project [codegen id : 12] Output [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] Input [7]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35, rn#36] -(50) BroadcastExchange +(42) BroadcastExchange Input [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(51) BroadcastHashJoin [codegen id : 13] +(43) BroadcastHashJoin [codegen id : 13] Left keys [4]: [i_category#3, i_brand#2, cc_name#13, rn#19] Right keys [4]: [i_category#30, i_brand#31, cc_name#32, (rn#36 - 1)] Join type: Inner Join condition: None -(52) Project [codegen id : 13] +(44) Project [codegen id : 13] Output [9]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, avg_monthly_sales#20, sum_sales#17, sum_sales#28 AS psum#37, sum_sales#35 AS nsum#38] Input [14]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28, i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] -(53) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [9]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, cc_name#13 ASC NULLS FIRST], [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = cs_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (57) -+- * ColumnarToRow (56) - +- CometFilter (55) - +- CometScan parquet spark_catalog.default.date_dim (54) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(54) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(55) CometFilter +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) -(56) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -(57) BroadcastExchange +(49) BroadcastExchange Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt index aca14449c..b64c2efdc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q57/simplified.txt @@ -13,45 +13,41 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (3) - ColumnarToRow + Sort [i_category,i_brand,cc_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,cc_name] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 - CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #6 - CometFilter [cc_call_center_sk,cc_name] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + Exchange [i_category,i_brand,cc_name] #1 + WholeStageCodegen (2) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] + InputAdapter + Exchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometFilter [i_item_sk,i_brand,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 + CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cc_call_center_sk,cc_name] #6 + CometFilter [cc_call_center_sk,cc_name] + CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) @@ -59,16 +55,13 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (7) - ColumnarToRow + Sort [i_category,i_brand,cc_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - CometColumnarExchange [i_category,i_brand,cc_name] #8 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 + Exchange [i_category,i_brand,cc_name] #8 + WholeStageCodegen (6) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] + InputAdapter + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #9 WholeStageCodegen (12) @@ -76,7 +69,6 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (11) - ColumnarToRow + Sort [i_category,i_brand,cc_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt index f00bd9e4b..2c82f91e9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/explain.txt @@ -1,63 +1,57 @@ == Physical Plan == -TakeOrderedAndProject (59) -+- * Project (58) - +- * BroadcastHashJoin Inner BuildRight (57) - :- * Project (40) - : +- * BroadcastHashJoin Inner BuildRight (39) - : :- * Filter (22) - : : +- * HashAggregate (21) - : : +- * ColumnarToRow (20) - : : +- CometColumnarExchange (19) - : : +- RowToColumnar (18) - : : +- * HashAggregate (17) - : : +- * ColumnarToRow (16) - : : +- CometProject (15) - : : +- CometBroadcastHashJoin (14) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (13) - : : +- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- ReusedExchange (10) - : +- BroadcastExchange (38) - : +- * Filter (37) - : +- * HashAggregate (36) - : +- * ColumnarToRow (35) - : +- CometColumnarExchange (34) - : +- RowToColumnar (33) - : +- * HashAggregate (32) - : +- * ColumnarToRow (31) - : +- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometFilter (24) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) - : : +- ReusedExchange (25) - : +- ReusedExchange (28) - +- BroadcastExchange (56) - +- * Filter (55) - +- * HashAggregate (54) - +- * ColumnarToRow (53) - +- CometColumnarExchange (52) - +- RowToColumnar (51) - +- * HashAggregate (50) - +- * ColumnarToRow (49) - +- CometProject (48) - +- CometBroadcastHashJoin (47) - :- CometProject (45) - : +- CometBroadcastHashJoin (44) - : :- CometFilter (42) - : : +- CometScan parquet spark_catalog.default.web_sales (41) - : +- ReusedExchange (43) - +- ReusedExchange (46) +TakeOrderedAndProject (53) ++- * Project (52) + +- * BroadcastHashJoin Inner BuildRight (51) + :- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * Filter (20) + : : +- * HashAggregate (19) + : : +- Exchange (18) + : : +- * HashAggregate (17) + : : +- * ColumnarToRow (16) + : : +- CometProject (15) + : : +- CometBroadcastHashJoin (14) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (13) + : : +- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- ReusedExchange (10) + : +- BroadcastExchange (34) + : +- * Filter (33) + : +- * HashAggregate (32) + : +- Exchange (31) + : +- * HashAggregate (30) + : +- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometFilter (22) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (21) + : : +- ReusedExchange (23) + : +- ReusedExchange (26) + +- BroadcastExchange (50) + +- * Filter (49) + +- * HashAggregate (48) + +- Exchange (47) + +- * HashAggregate (46) + +- * ColumnarToRow (45) + +- CometProject (44) + +- CometBroadcastHashJoin (43) + :- CometProject (41) + : +- CometBroadcastHashJoin (40) + : :- CometFilter (38) + : : +- CometScan parquet spark_catalog.default.web_sales (37) + : +- ReusedExchange (39) + +- ReusedExchange (42) (1) Scan parquet spark_catalog.default.store_sales @@ -107,7 +101,7 @@ ReadSchema: struct Input [2]: [d_date_sk#7, d_date#8] Condition : isnotnull(d_date_sk#7) -(10) ReusedExchange [Reuses operator id: 65] +(10) ReusedExchange [Reuses operator id: 59] Output [1]: [d_date#9] (11) CometBroadcastHashJoin @@ -142,28 +136,22 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] Aggregate Attributes [1]: [sum#10] Results [2]: [i_item_id#6, sum#11] -(18) RowToColumnar +(18) Exchange Input [2]: [i_item_id#6, sum#11] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(19) CometColumnarExchange -Input [2]: [i_item_id#6, sum#11] -Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(20) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_id#6, sum#11] - -(21) HashAggregate [codegen id : 6] +(19) HashAggregate [codegen id : 6] Input [2]: [i_item_id#6, sum#11] Keys [1]: [i_item_id#6] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#12] Results [2]: [i_item_id#6 AS item_id#13, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#12,17,2) AS ss_item_rev#14] -(22) Filter [codegen id : 6] +(20) Filter [codegen id : 6] Input [2]: [item_id#13, ss_item_rev#14] Condition : isnotnull(ss_item_rev#14) -(23) Scan parquet spark_catalog.default.catalog_sales +(21) Scan parquet spark_catalog.default.catalog_sales Output [3]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] Batched: true Location: InMemoryFileIndex [] @@ -171,80 +159,74 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#17), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(24) CometFilter +(22) CometFilter Input [3]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] Condition : isnotnull(cs_item_sk#15) -(25) ReusedExchange [Reuses operator id: 5] +(23) ReusedExchange [Reuses operator id: 5] Output [2]: [i_item_sk#19, i_item_id#20] -(26) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [3]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17] Right output [2]: [i_item_sk#19, i_item_id#20] Arguments: [cs_item_sk#15], [i_item_sk#19], Inner, BuildRight -(27) CometProject +(25) CometProject Input [5]: [cs_item_sk#15, cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_sk#19, i_item_id#20] Arguments: [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#20], [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#20] -(28) ReusedExchange [Reuses operator id: 13] +(26) ReusedExchange [Reuses operator id: 13] Output [1]: [d_date_sk#21] -(29) CometBroadcastHashJoin +(27) CometBroadcastHashJoin Left output [3]: [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#20] Right output [1]: [d_date_sk#21] Arguments: [cs_sold_date_sk#17], [d_date_sk#21], Inner, BuildRight -(30) CometProject +(28) CometProject Input [4]: [cs_ext_sales_price#16, cs_sold_date_sk#17, i_item_id#20, d_date_sk#21] Arguments: [cs_ext_sales_price#16, i_item_id#20], [cs_ext_sales_price#16, i_item_id#20] -(31) ColumnarToRow [codegen id : 2] +(29) ColumnarToRow [codegen id : 2] Input [2]: [cs_ext_sales_price#16, i_item_id#20] -(32) HashAggregate [codegen id : 2] +(30) HashAggregate [codegen id : 2] Input [2]: [cs_ext_sales_price#16, i_item_id#20] Keys [1]: [i_item_id#20] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#16))] Aggregate Attributes [1]: [sum#22] Results [2]: [i_item_id#20, sum#23] -(33) RowToColumnar -Input [2]: [i_item_id#20, sum#23] - -(34) CometColumnarExchange +(31) Exchange Input [2]: [i_item_id#20, sum#23] -Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(35) ColumnarToRow [codegen id : 3] -Input [2]: [i_item_id#20, sum#23] - -(36) HashAggregate [codegen id : 3] +(32) HashAggregate [codegen id : 3] Input [2]: [i_item_id#20, sum#23] Keys [1]: [i_item_id#20] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#16))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#16))#24] Results [2]: [i_item_id#20 AS item_id#25, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#16))#24,17,2) AS cs_item_rev#26] -(37) Filter [codegen id : 3] +(33) Filter [codegen id : 3] Input [2]: [item_id#25, cs_item_rev#26] Condition : isnotnull(cs_item_rev#26) -(38) BroadcastExchange +(34) BroadcastExchange Input [2]: [item_id#25, cs_item_rev#26] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(39) BroadcastHashJoin [codegen id : 6] +(35) BroadcastHashJoin [codegen id : 6] Left keys [1]: [item_id#13] Right keys [1]: [item_id#25] Join type: Inner Join condition: ((((cast(ss_item_rev#14 as decimal(19,3)) >= (0.9 * cs_item_rev#26)) AND (cast(ss_item_rev#14 as decimal(20,3)) <= (1.1 * cs_item_rev#26))) AND (cast(cs_item_rev#26 as decimal(19,3)) >= (0.9 * ss_item_rev#14))) AND (cast(cs_item_rev#26 as decimal(20,3)) <= (1.1 * ss_item_rev#14))) -(40) Project [codegen id : 6] +(36) Project [codegen id : 6] Output [3]: [item_id#13, ss_item_rev#14, cs_item_rev#26] Input [4]: [item_id#13, ss_item_rev#14, item_id#25, cs_item_rev#26] -(41) Scan parquet spark_catalog.default.web_sales +(37) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Batched: true Location: InMemoryFileIndex [] @@ -252,173 +234,167 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#29), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(42) CometFilter +(38) CometFilter Input [3]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Condition : isnotnull(ws_item_sk#27) -(43) ReusedExchange [Reuses operator id: 5] +(39) ReusedExchange [Reuses operator id: 5] Output [2]: [i_item_sk#31, i_item_id#32] -(44) CometBroadcastHashJoin +(40) CometBroadcastHashJoin Left output [3]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29] Right output [2]: [i_item_sk#31, i_item_id#32] Arguments: [ws_item_sk#27], [i_item_sk#31], Inner, BuildRight -(45) CometProject +(41) CometProject Input [5]: [ws_item_sk#27, ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_sk#31, i_item_id#32] Arguments: [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#32], [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#32] -(46) ReusedExchange [Reuses operator id: 13] +(42) ReusedExchange [Reuses operator id: 13] Output [1]: [d_date_sk#33] -(47) CometBroadcastHashJoin +(43) CometBroadcastHashJoin Left output [3]: [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#32] Right output [1]: [d_date_sk#33] Arguments: [ws_sold_date_sk#29], [d_date_sk#33], Inner, BuildRight -(48) CometProject +(44) CometProject Input [4]: [ws_ext_sales_price#28, ws_sold_date_sk#29, i_item_id#32, d_date_sk#33] Arguments: [ws_ext_sales_price#28, i_item_id#32], [ws_ext_sales_price#28, i_item_id#32] -(49) ColumnarToRow [codegen id : 4] +(45) ColumnarToRow [codegen id : 4] Input [2]: [ws_ext_sales_price#28, i_item_id#32] -(50) HashAggregate [codegen id : 4] +(46) HashAggregate [codegen id : 4] Input [2]: [ws_ext_sales_price#28, i_item_id#32] Keys [1]: [i_item_id#32] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] Aggregate Attributes [1]: [sum#34] Results [2]: [i_item_id#32, sum#35] -(51) RowToColumnar -Input [2]: [i_item_id#32, sum#35] - -(52) CometColumnarExchange -Input [2]: [i_item_id#32, sum#35] -Arguments: hashpartitioning(i_item_id#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(53) ColumnarToRow [codegen id : 5] +(47) Exchange Input [2]: [i_item_id#32, sum#35] +Arguments: hashpartitioning(i_item_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(54) HashAggregate [codegen id : 5] +(48) HashAggregate [codegen id : 5] Input [2]: [i_item_id#32, sum#35] Keys [1]: [i_item_id#32] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#28))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#28))#36] Results [2]: [i_item_id#32 AS item_id#37, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#28))#36,17,2) AS ws_item_rev#38] -(55) Filter [codegen id : 5] +(49) Filter [codegen id : 5] Input [2]: [item_id#37, ws_item_rev#38] Condition : isnotnull(ws_item_rev#38) -(56) BroadcastExchange +(50) BroadcastExchange Input [2]: [item_id#37, ws_item_rev#38] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(57) BroadcastHashJoin [codegen id : 6] +(51) BroadcastHashJoin [codegen id : 6] Left keys [1]: [item_id#13] Right keys [1]: [item_id#37] Join type: Inner Join condition: ((((((((cast(ss_item_rev#14 as decimal(19,3)) >= (0.9 * ws_item_rev#38)) AND (cast(ss_item_rev#14 as decimal(20,3)) <= (1.1 * ws_item_rev#38))) AND (cast(cs_item_rev#26 as decimal(19,3)) >= (0.9 * ws_item_rev#38))) AND (cast(cs_item_rev#26 as decimal(20,3)) <= (1.1 * ws_item_rev#38))) AND (cast(ws_item_rev#38 as decimal(19,3)) >= (0.9 * ss_item_rev#14))) AND (cast(ws_item_rev#38 as decimal(20,3)) <= (1.1 * ss_item_rev#14))) AND (cast(ws_item_rev#38 as decimal(19,3)) >= (0.9 * cs_item_rev#26))) AND (cast(ws_item_rev#38 as decimal(20,3)) <= (1.1 * cs_item_rev#26))) -(58) Project [codegen id : 6] +(52) Project [codegen id : 6] Output [8]: [item_id#13, ss_item_rev#14, (((ss_item_rev#14 / ((ss_item_rev#14 + cs_item_rev#26) + ws_item_rev#38)) / 3) * 100) AS ss_dev#39, cs_item_rev#26, (((cs_item_rev#26 / ((ss_item_rev#14 + cs_item_rev#26) + ws_item_rev#38)) / 3) * 100) AS cs_dev#40, ws_item_rev#38, (((ws_item_rev#38 / ((ss_item_rev#14 + cs_item_rev#26) + ws_item_rev#38)) / 3) * 100) AS ws_dev#41, (((ss_item_rev#14 + cs_item_rev#26) + ws_item_rev#38) / 3) AS average#42] Input [5]: [item_id#13, ss_item_rev#14, cs_item_rev#26, item_id#37, ws_item_rev#38] -(59) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [8]: [item_id#13, ss_item_rev#14, ss_dev#39, cs_item_rev#26, cs_dev#40, ws_item_rev#38, ws_dev#41, average#42] Arguments: 100, [item_id#13 ASC NULLS FIRST, ss_item_rev#14 ASC NULLS FIRST], [item_id#13, ss_item_rev#14, ss_dev#39, cs_item_rev#26, cs_dev#40, ws_item_rev#38, ws_dev#41, average#42] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (69) -+- * ColumnarToRow (68) - +- CometProject (67) - +- CometBroadcastHashJoin (66) - :- CometFilter (61) - : +- CometScan parquet spark_catalog.default.date_dim (60) - +- CometBroadcastExchange (65) - +- CometProject (64) - +- CometFilter (63) - +- CometScan parquet spark_catalog.default.date_dim (62) - - -(60) Scan parquet spark_catalog.default.date_dim +BroadcastExchange (63) ++- * ColumnarToRow (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometFilter (55) + : +- CometScan parquet spark_catalog.default.date_dim (54) + +- CometBroadcastExchange (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.date_dim (56) + + +(54) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(61) CometFilter +(55) CometFilter Input [2]: [d_date_sk#7, d_date#8] Condition : isnotnull(d_date_sk#7) -(62) Scan parquet spark_catalog.default.date_dim +(56) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#43] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#44)] ReadSchema: struct -(63) CometFilter +(57) CometFilter Input [2]: [d_date#9, d_week_seq#43] Condition : (isnotnull(d_week_seq#43) AND (d_week_seq#43 = ReusedSubquery Subquery scalar-subquery#44, [id=#45])) -(64) CometProject +(58) CometProject Input [2]: [d_date#9, d_week_seq#43] Arguments: [d_date#9], [d_date#9] -(65) CometBroadcastExchange +(59) CometBroadcastExchange Input [1]: [d_date#9] Arguments: [d_date#9] -(66) CometBroadcastHashJoin +(60) CometBroadcastHashJoin Left output [2]: [d_date_sk#7, d_date#8] Right output [1]: [d_date#9] Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight -(67) CometProject +(61) CometProject Input [2]: [d_date_sk#7, d_date#8] Arguments: [d_date_sk#7], [d_date_sk#7] -(68) ColumnarToRow [codegen id : 1] +(62) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#7] -(69) BroadcastExchange +(63) BroadcastExchange Input [1]: [d_date_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -Subquery:2 Hosting operator id = 63 Hosting Expression = ReusedSubquery Subquery scalar-subquery#44, [id=#45] +Subquery:2 Hosting operator id = 57 Hosting Expression = ReusedSubquery Subquery scalar-subquery#44, [id=#45] -Subquery:3 Hosting operator id = 62 Hosting Expression = Subquery scalar-subquery#44, [id=#45] -* ColumnarToRow (73) -+- CometProject (72) - +- CometFilter (71) - +- CometScan parquet spark_catalog.default.date_dim (70) +Subquery:3 Hosting operator id = 56 Hosting Expression = Subquery scalar-subquery#44, [id=#45] +* ColumnarToRow (67) ++- CometProject (66) + +- CometFilter (65) + +- CometScan parquet spark_catalog.default.date_dim (64) -(70) Scan parquet spark_catalog.default.date_dim +(64) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date#46, d_week_seq#47] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), EqualTo(d_date,2000-01-03)] ReadSchema: struct -(71) CometFilter +(65) CometFilter Input [2]: [d_date#46, d_week_seq#47] Condition : (isnotnull(d_date#46) AND (d_date#46 = 2000-01-03)) -(72) CometProject +(66) CometProject Input [2]: [d_date#46, d_week_seq#47] Arguments: [d_week_seq#47], [d_week_seq#47] -(73) ColumnarToRow [codegen id : 1] +(67) ColumnarToRow [codegen id : 1] Input [1]: [d_week_seq#47] -Subquery:4 Hosting operator id = 23 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#4 +Subquery:4 Hosting operator id = 21 Hosting Expression = cs_sold_date_sk#17 IN dynamicpruning#4 -Subquery:5 Hosting operator id = 41 Hosting Expression = ws_sold_date_sk#29 IN dynamicpruning#4 +Subquery:5 Hosting operator id = 37 Hosting Expression = ws_sold_date_sk#29 IN dynamicpruning#4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt index f82461dba..0f5a783b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q58/simplified.txt @@ -6,91 +6,85 @@ TakeOrderedAndProject [item_id,ss_item_rev,ss_dev,cs_item_rev,cs_dev,ws_item_rev BroadcastHashJoin [item_id,item_id,ss_item_rev,cs_item_rev] Filter [ss_item_rev] HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),item_id,ss_item_rev,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,d_date_sk] - CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date] #3 - CometProject [d_date] - CometFilter [d_date,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [i_item_sk,i_item_id] #4 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date] #3 + InputAdapter + Exchange [i_item_id] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #3 + CometProject [d_date] + CometFilter [d_date,d_week_seq] + ReusedSubquery [d_week_seq] #2 + CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + Subquery #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [i_item_sk,i_item_id] #4 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + ReusedExchange [d_date] #3 InputAdapter BroadcastExchange #6 WholeStageCodegen (3) Filter [cs_item_rev] HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),item_id,cs_item_rev,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #7 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id] - CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] - CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] - CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_item_id] #4 - ReusedExchange [d_date_sk] #5 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (5) - Filter [ws_item_rev] - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),item_id,ws_item_rev,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #9 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] + InputAdapter + Exchange [i_item_id] #7 + WholeStageCodegen (2) + HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] ColumnarToRow InputAdapter - CometProject [ws_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] - CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] - CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [i_item_sk,i_item_id] #4 ReusedExchange [d_date_sk] #5 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (5) + Filter [ws_item_rev] + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),item_id,ws_item_rev,sum] + InputAdapter + Exchange [i_item_id] #9 + WholeStageCodegen (4) + HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [i_item_sk,i_item_id] #4 + ReusedExchange [d_date_sk] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt index 8829b74e0..afa9351c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/explain.txt @@ -1,50 +1,47 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * Project (45) - +- * BroadcastHashJoin Inner BuildRight (44) - :- * Project (26) - : +- * BroadcastHashJoin Inner BuildRight (25) - : :- * Project (19) - : : +- * BroadcastHashJoin Inner BuildRight (18) - : : :- * HashAggregate (13) - : : : +- * ColumnarToRow (12) - : : : +- CometColumnarExchange (11) - : : : +- RowToColumnar (10) - : : : +- * HashAggregate (9) - : : : +- * ColumnarToRow (8) - : : : +- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- BroadcastExchange (17) - : : +- * ColumnarToRow (16) - : : +- CometFilter (15) - : : +- CometScan parquet spark_catalog.default.store (14) - : +- BroadcastExchange (24) - : +- * ColumnarToRow (23) - : +- CometProject (22) - : +- CometFilter (21) - : +- CometScan parquet spark_catalog.default.date_dim (20) - +- BroadcastExchange (43) - +- * Project (42) - +- * BroadcastHashJoin Inner BuildRight (41) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * HashAggregate (29) - : : +- * ColumnarToRow (28) - : : +- ReusedExchange (27) - : +- BroadcastExchange (33) - : +- * ColumnarToRow (32) - : +- CometFilter (31) - : +- CometScan parquet spark_catalog.default.store (30) - +- BroadcastExchange (40) - +- * ColumnarToRow (39) - +- CometProject (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.date_dim (36) +TakeOrderedAndProject (43) ++- * Project (42) + +- * BroadcastHashJoin Inner BuildRight (41) + :- * Project (24) + : +- * BroadcastHashJoin Inner BuildRight (23) + : :- * Project (17) + : : +- * BroadcastHashJoin Inner BuildRight (16) + : : :- * HashAggregate (11) + : : : +- Exchange (10) + : : : +- * HashAggregate (9) + : : : +- * ColumnarToRow (8) + : : : +- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (15) + : : +- * ColumnarToRow (14) + : : +- CometFilter (13) + : : +- CometScan parquet spark_catalog.default.store (12) + : +- BroadcastExchange (22) + : +- * ColumnarToRow (21) + : +- CometProject (20) + : +- CometFilter (19) + : +- CometScan parquet spark_catalog.default.date_dim (18) + +- BroadcastExchange (40) + +- * Project (39) + +- * BroadcastHashJoin Inner BuildRight (38) + :- * Project (32) + : +- * BroadcastHashJoin Inner BuildRight (31) + : :- * HashAggregate (26) + : : +- ReusedExchange (25) + : +- BroadcastExchange (30) + : +- * ColumnarToRow (29) + : +- CometFilter (28) + : +- CometScan parquet spark_catalog.default.store (27) + +- BroadcastExchange (37) + +- * ColumnarToRow (36) + +- CometProject (35) + +- CometFilter (34) + +- CometScan parquet spark_catalog.default.date_dim (33) (1) Scan parquet spark_catalog.default.store_sales @@ -93,171 +90,162 @@ Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) T Aggregate Attributes [7]: [sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] Results [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] -(10) RowToColumnar +(10) Exchange Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] +Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(11) CometColumnarExchange -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] -Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(12) ColumnarToRow [codegen id : 8] -Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] - -(13) HashAggregate [codegen id : 8] +(11) HashAggregate [codegen id : 8] Input [9]: [d_week_seq#5, ss_store_sk#1, sum#14, sum#15, sum#16, sum#17, sum#18, sum#19, sum#20] Keys [2]: [d_week_seq#5, ss_store_sk#1] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27] Results [9]: [d_week_seq#5, ss_store_sk#1, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END))#21,17,2) AS sun_sales#28, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END))#22,17,2) AS mon_sales#29, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END))#23,17,2) AS tue_sales#30, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END))#24,17,2) AS wed_sales#31, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END))#25,17,2) AS thu_sales#32, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END))#26,17,2) AS fri_sales#33, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))#27,17,2) AS sat_sales#34] -(14) Scan parquet spark_catalog.default.store +(12) Scan parquet spark_catalog.default.store Output [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] ReadSchema: struct -(15) CometFilter +(13) CometFilter Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] Condition : (isnotnull(s_store_sk#35) AND isnotnull(s_store_id#36)) -(16) ColumnarToRow [codegen id : 2] +(14) ColumnarToRow [codegen id : 2] Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] -(17) BroadcastExchange +(15) BroadcastExchange Input [3]: [s_store_sk#35, s_store_id#36, s_store_name#37] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] -(18) BroadcastHashJoin [codegen id : 8] +(16) BroadcastHashJoin [codegen id : 8] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#35] Join type: Inner Join condition: None -(19) Project [codegen id : 8] +(17) Project [codegen id : 8] Output [10]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37] Input [12]: [d_week_seq#5, ss_store_sk#1, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_sk#35, s_store_id#36, s_store_name#37] -(20) Scan parquet spark_catalog.default.date_dim +(18) Scan parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#38, d_week_seq#39] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_week_seq)] ReadSchema: struct -(21) CometFilter +(19) CometFilter Input [2]: [d_month_seq#38, d_week_seq#39] Condition : (((isnotnull(d_month_seq#38) AND (d_month_seq#38 >= 1212)) AND (d_month_seq#38 <= 1223)) AND isnotnull(d_week_seq#39)) -(22) CometProject +(20) CometProject Input [2]: [d_month_seq#38, d_week_seq#39] Arguments: [d_week_seq#39], [d_week_seq#39] -(23) ColumnarToRow [codegen id : 3] +(21) ColumnarToRow [codegen id : 3] Input [1]: [d_week_seq#39] -(24) BroadcastExchange +(22) BroadcastExchange Input [1]: [d_week_seq#39] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(25) BroadcastHashJoin [codegen id : 8] +(23) BroadcastHashJoin [codegen id : 8] Left keys [1]: [d_week_seq#5] Right keys [1]: [d_week_seq#39] Join type: Inner Join condition: None -(26) Project [codegen id : 8] +(24) Project [codegen id : 8] Output [10]: [s_store_name#37 AS s_store_name1#40, d_week_seq#5 AS d_week_seq1#41, s_store_id#36 AS s_store_id1#42, sun_sales#28 AS sun_sales1#43, mon_sales#29 AS mon_sales1#44, tue_sales#30 AS tue_sales1#45, wed_sales#31 AS wed_sales1#46, thu_sales#32 AS thu_sales1#47, fri_sales#33 AS fri_sales1#48, sat_sales#34 AS sat_sales1#49] Input [11]: [d_week_seq#5, sun_sales#28, mon_sales#29, tue_sales#30, wed_sales#31, thu_sales#32, fri_sales#33, sat_sales#34, s_store_id#36, s_store_name#37, d_week_seq#39] -(27) ReusedExchange [Reuses operator id: 11] +(25) ReusedExchange [Reuses operator id: 10] Output [9]: [d_week_seq#50, ss_store_sk#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] -(28) ColumnarToRow [codegen id : 7] -Input [9]: [d_week_seq#50, ss_store_sk#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] - -(29) HashAggregate [codegen id : 7] +(26) HashAggregate [codegen id : 7] Input [9]: [d_week_seq#50, ss_store_sk#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] Keys [2]: [d_week_seq#50, ss_store_sk#51] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END)), sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))] Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END))#21, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END))#22, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END))#23, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END))#24, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END))#25, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END))#26, sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))#27] Results [9]: [d_week_seq#50, ss_store_sk#51, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Sunday ) THEN ss_sales_price#60 END))#21,17,2) AS sun_sales#61, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Monday ) THEN ss_sales_price#60 END))#22,17,2) AS mon_sales#62, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Tuesday ) THEN ss_sales_price#60 END))#23,17,2) AS tue_sales#63, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Wednesday) THEN ss_sales_price#60 END))#24,17,2) AS wed_sales#64, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Thursday ) THEN ss_sales_price#60 END))#25,17,2) AS thu_sales#65, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Friday ) THEN ss_sales_price#60 END))#26,17,2) AS fri_sales#66, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#59 = Saturday ) THEN ss_sales_price#60 END))#27,17,2) AS sat_sales#67] -(30) Scan parquet spark_catalog.default.store +(27) Scan parquet spark_catalog.default.store Output [2]: [s_store_sk#68, s_store_id#69] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_id)] ReadSchema: struct -(31) CometFilter +(28) CometFilter Input [2]: [s_store_sk#68, s_store_id#69] Condition : (isnotnull(s_store_sk#68) AND isnotnull(s_store_id#69)) -(32) ColumnarToRow [codegen id : 5] +(29) ColumnarToRow [codegen id : 5] Input [2]: [s_store_sk#68, s_store_id#69] -(33) BroadcastExchange +(30) BroadcastExchange Input [2]: [s_store_sk#68, s_store_id#69] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] -(34) BroadcastHashJoin [codegen id : 7] +(31) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ss_store_sk#51] Right keys [1]: [s_store_sk#68] Join type: Inner Join condition: None -(35) Project [codegen id : 7] +(32) Project [codegen id : 7] Output [9]: [d_week_seq#50, sun_sales#61, mon_sales#62, tue_sales#63, wed_sales#64, thu_sales#65, fri_sales#66, sat_sales#67, s_store_id#69] Input [11]: [d_week_seq#50, ss_store_sk#51, sun_sales#61, mon_sales#62, tue_sales#63, wed_sales#64, thu_sales#65, fri_sales#66, sat_sales#67, s_store_sk#68, s_store_id#69] -(36) Scan parquet spark_catalog.default.date_dim +(33) Scan parquet spark_catalog.default.date_dim Output [2]: [d_month_seq#70, d_week_seq#71] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1224), LessThanOrEqual(d_month_seq,1235), IsNotNull(d_week_seq)] ReadSchema: struct -(37) CometFilter +(34) CometFilter Input [2]: [d_month_seq#70, d_week_seq#71] Condition : (((isnotnull(d_month_seq#70) AND (d_month_seq#70 >= 1224)) AND (d_month_seq#70 <= 1235)) AND isnotnull(d_week_seq#71)) -(38) CometProject +(35) CometProject Input [2]: [d_month_seq#70, d_week_seq#71] Arguments: [d_week_seq#71], [d_week_seq#71] -(39) ColumnarToRow [codegen id : 6] +(36) ColumnarToRow [codegen id : 6] Input [1]: [d_week_seq#71] -(40) BroadcastExchange +(37) BroadcastExchange Input [1]: [d_week_seq#71] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -(41) BroadcastHashJoin [codegen id : 7] +(38) BroadcastHashJoin [codegen id : 7] Left keys [1]: [d_week_seq#50] Right keys [1]: [d_week_seq#71] Join type: Inner Join condition: None -(42) Project [codegen id : 7] +(39) Project [codegen id : 7] Output [9]: [d_week_seq#50 AS d_week_seq2#72, s_store_id#69 AS s_store_id2#73, sun_sales#61 AS sun_sales2#74, mon_sales#62 AS mon_sales2#75, tue_sales#63 AS tue_sales2#76, wed_sales#64 AS wed_sales2#77, thu_sales#65 AS thu_sales2#78, fri_sales#66 AS fri_sales2#79, sat_sales#67 AS sat_sales2#80] Input [10]: [d_week_seq#50, sun_sales#61, mon_sales#62, tue_sales#63, wed_sales#64, thu_sales#65, fri_sales#66, sat_sales#67, s_store_id#69, d_week_seq#71] -(43) BroadcastExchange +(40) BroadcastExchange Input [9]: [d_week_seq2#72, s_store_id2#73, sun_sales2#74, mon_sales2#75, tue_sales2#76, wed_sales2#77, thu_sales2#78, fri_sales2#79, sat_sales2#80] Arguments: HashedRelationBroadcastMode(List(input[1, string, true], (input[0, int, true] - 52)),false), [plan_id=6] -(44) BroadcastHashJoin [codegen id : 8] +(41) BroadcastHashJoin [codegen id : 8] Left keys [2]: [s_store_id1#42, d_week_seq1#41] Right keys [2]: [s_store_id2#73, (d_week_seq2#72 - 52)] Join type: Inner Join condition: None -(45) Project [codegen id : 8] +(42) Project [codegen id : 8] Output [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1#43 / sun_sales2#74) AS (sun_sales1 / sun_sales2)#81, (mon_sales1#44 / mon_sales2#75) AS (mon_sales1 / mon_sales2)#82, (tue_sales1#45 / tue_sales2#76) AS (tue_sales1 / tue_sales2)#83, (wed_sales1#46 / wed_sales2#77) AS (wed_sales1 / wed_sales2)#84, (thu_sales1#47 / thu_sales2#78) AS (thu_sales1 / thu_sales2)#85, (fri_sales1#48 / fri_sales2#79) AS (fri_sales1 / fri_sales2)#86, (sat_sales1#49 / sat_sales2#80) AS (sat_sales1 / sat_sales2)#87] Input [19]: [s_store_name1#40, d_week_seq1#41, s_store_id1#42, sun_sales1#43, mon_sales1#44, tue_sales1#45, wed_sales1#46, thu_sales1#47, fri_sales1#48, sat_sales1#49, d_week_seq2#72, s_store_id2#73, sun_sales2#74, mon_sales2#75, tue_sales2#76, wed_sales2#77, thu_sales2#78, fri_sales2#79, sat_sales2#80] -(46) TakeOrderedAndProject +(43) TakeOrderedAndProject Input [10]: [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#81, (mon_sales1 / mon_sales2)#82, (tue_sales1 / tue_sales2)#83, (wed_sales1 / wed_sales2)#84, (thu_sales1 / thu_sales2)#85, (fri_sales1 / fri_sales2)#86, (sat_sales1 / sat_sales2)#87] Arguments: 100, [s_store_name1#40 ASC NULLS FIRST, s_store_id1#42 ASC NULLS FIRST, d_week_seq1#41 ASC NULLS FIRST], [s_store_name1#40, s_store_id1#42, d_week_seq1#41, (sun_sales1 / sun_sales2)#81, (mon_sales1 / mon_sales2)#82, (tue_sales1 / tue_sales2)#83, (wed_sales1 / wed_sales2)#84, (thu_sales1 / thu_sales2)#85, (fri_sales1 / fri_sales2)#86, (sat_sales1 / sat_sales2)#87] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt index 43f903e19..58bae46d1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q59/simplified.txt @@ -7,21 +7,19 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] BroadcastHashJoin [ss_store_sk,s_store_sk] HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_week_seq,ss_store_sk] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] - CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] - CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 - CometFilter [d_date_sk,d_week_seq,d_day_name] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] + InputAdapter + Exchange [d_week_seq,ss_store_sk] #1 + WholeStageCodegen (1) + HashAggregate [d_week_seq,ss_store_sk,d_day_name,ss_sales_price] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] + CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] + CometFilter [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastExchange [d_date_sk,d_week_seq,d_day_name] #2 + CometFilter [d_date_sk,d_week_seq,d_day_name] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq,d_day_name] InputAdapter BroadcastExchange #3 WholeStageCodegen (2) @@ -45,9 +43,8 @@ TakeOrderedAndProject [s_store_name1,s_store_id1,d_week_seq1,(sun_sales1 / sun_s Project [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id] BroadcastHashJoin [ss_store_sk,s_store_sk] HashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] [sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END)),sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 + InputAdapter + ReusedExchange [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum] #1 InputAdapter BroadcastExchange #6 WholeStageCodegen (5) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt index f8552c4f8..7c380571e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/explain.txt @@ -1,48 +1,44 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * Filter (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometColumnarExchange (40) - +- RowToColumnar (39) - +- * HashAggregate (38) - +- * Project (37) - +- * BroadcastHashJoin Inner BuildRight (36) - :- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.customer_address (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.customer (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.store_sales (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.date_dim (13) - +- BroadcastExchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * ColumnarToRow (22) - : +- CometFilter (21) - : +- CometScan parquet spark_catalog.default.item (20) - +- BroadcastExchange (32) - +- * Filter (31) - +- * HashAggregate (30) - +- * ColumnarToRow (29) - +- CometColumnarExchange (28) - +- RowToColumnar (27) - +- * HashAggregate (26) - +- * ColumnarToRow (25) - +- CometFilter (24) - +- CometScan parquet spark_catalog.default.item (23) +TakeOrderedAndProject (40) ++- * Filter (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.customer (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.store_sales (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometScan parquet spark_catalog.default.date_dim (13) + +- BroadcastExchange (33) + +- * Project (32) + +- * BroadcastHashJoin Inner BuildRight (31) + :- * ColumnarToRow (22) + : +- CometFilter (21) + : +- CometScan parquet spark_catalog.default.item (20) + +- BroadcastExchange (30) + +- * Filter (29) + +- * HashAggregate (28) + +- Exchange (27) + +- * HashAggregate (26) + +- * ColumnarToRow (25) + +- CometFilter (24) + +- CometScan parquet spark_catalog.default.item (23) (1) Scan parquet spark_catalog.default.customer_address @@ -171,161 +167,149 @@ Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] Aggregate Attributes [2]: [sum#18, count#19] Results [3]: [i_category#17, sum#20, count#21] -(27) RowToColumnar +(27) Exchange Input [3]: [i_category#17, sum#20, count#21] +Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(28) CometColumnarExchange -Input [3]: [i_category#17, sum#20, count#21] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(29) ColumnarToRow [codegen id : 2] -Input [3]: [i_category#17, sum#20, count#21] - -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [3]: [i_category#17, sum#20, count#21] Keys [1]: [i_category#17] Functions [1]: [avg(UnscaledValue(i_current_price#16))] Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#16))#22] Results [2]: [cast((avg(UnscaledValue(i_current_price#16))#22 / 100.0) as decimal(11,6)) AS avg(i_current_price)#23, i_category#17] -(31) Filter [codegen id : 2] +(29) Filter [codegen id : 2] Input [2]: [avg(i_current_price)#23, i_category#17] Condition : isnotnull(avg(i_current_price)#23) -(32) BroadcastExchange +(30) BroadcastExchange Input [2]: [avg(i_current_price)#23, i_category#17] Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=2] -(33) BroadcastHashJoin [codegen id : 3] +(31) BroadcastHashJoin [codegen id : 3] Left keys [1]: [i_category#15] Right keys [1]: [i_category#17] Join type: Inner Join condition: (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#23)) -(34) Project [codegen id : 3] +(32) Project [codegen id : 3] Output [1]: [i_item_sk#13] Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#23, i_category#17] -(35) BroadcastExchange +(33) BroadcastExchange Input [1]: [i_item_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(36) BroadcastHashJoin [codegen id : 4] +(34) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_item_sk#5] Right keys [1]: [i_item_sk#13] Join type: Inner Join condition: None -(37) Project [codegen id : 4] +(35) Project [codegen id : 4] Output [1]: [ca_state#2] Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#13] -(38) HashAggregate [codegen id : 4] +(36) HashAggregate [codegen id : 4] Input [1]: [ca_state#2] Keys [1]: [ca_state#2] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#24] Results [2]: [ca_state#2, count#25] -(39) RowToColumnar -Input [2]: [ca_state#2, count#25] - -(40) CometColumnarExchange -Input [2]: [ca_state#2, count#25] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(41) ColumnarToRow [codegen id : 5] +(37) Exchange Input [2]: [ca_state#2, count#25] +Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(42) HashAggregate [codegen id : 5] +(38) HashAggregate [codegen id : 5] Input [2]: [ca_state#2, count#25] Keys [1]: [ca_state#2] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#26] Results [2]: [ca_state#2 AS state#27, count(1)#26 AS cnt#28] -(43) Filter [codegen id : 5] +(39) Filter [codegen id : 5] Input [2]: [state#27, cnt#28] Condition : (cnt#28 >= 10) -(44) TakeOrderedAndProject +(40) TakeOrderedAndProject Input [2]: [state#27, cnt#28] Arguments: 100, [cnt#28 ASC NULLS FIRST], [state#27, cnt#28] ===== Subqueries ===== Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (49) -+- * ColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan parquet spark_catalog.default.date_dim (45) +BroadcastExchange (45) ++- * ColumnarToRow (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan parquet spark_catalog.default.date_dim (41) -(45) Scan parquet spark_catalog.default.date_dim +(41) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#11), IsNotNull(d_date_sk)] ReadSchema: struct -(46) CometFilter +(42) CometFilter Input [2]: [d_date_sk#9, d_month_seq#10] Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = ReusedSubquery Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) -(47) CometProject +(43) CometProject Input [2]: [d_date_sk#9, d_month_seq#10] Arguments: [d_date_sk#9], [d_date_sk#9] -(48) ColumnarToRow [codegen id : 1] +(44) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(49) BroadcastExchange +(45) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -Subquery:2 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] +Subquery:2 Hosting operator id = 42 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] -Subquery:3 Hosting operator id = 45 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* ColumnarToRow (56) -+- CometHashAggregate (55) - +- CometColumnarExchange (54) - +- CometHashAggregate (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.date_dim (50) +Subquery:3 Hosting operator id = 41 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* ColumnarToRow (52) ++- CometHashAggregate (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(50) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#29, d_year#30, d_moy#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(51) CometFilter +(47) CometFilter Input [3]: [d_month_seq#29, d_year#30, d_moy#31] Condition : (((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2000)) AND (d_moy#31 = 1)) -(52) CometProject +(48) CometProject Input [3]: [d_month_seq#29, d_year#30, d_moy#31] Arguments: [d_month_seq#29], [d_month_seq#29] -(53) CometHashAggregate +(49) CometHashAggregate Input [1]: [d_month_seq#29] Keys [1]: [d_month_seq#29] Functions: [] -(54) CometColumnarExchange +(50) CometExchange Input [1]: [d_month_seq#29] -Arguments: hashpartitioning(d_month_seq#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(d_month_seq#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(55) CometHashAggregate +(51) CometHashAggregate Input [1]: [d_month_seq#29] Keys [1]: [d_month_seq#29] Functions: [] -(56) ColumnarToRow [codegen id : 1] +(52) ColumnarToRow [codegen id : 1] Input [1]: [d_month_seq#29] Subquery:4 Hosting operator id = 14 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt index e28908700..c5bbfc7d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q6/simplified.txt @@ -2,76 +2,72 @@ TakeOrderedAndProject [cnt,state] WholeStageCodegen (5) Filter [cnt] HashAggregate [ca_state,count] [count(1),state,cnt,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_state] #1 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [ca_state] [count,count] - Project [ca_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - ColumnarToRow - InputAdapter - CometProject [ca_state,ss_item_sk] - CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] - CometProject [ca_state,ss_item_sk,ss_sold_date_sk] - CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometProject [ca_state,c_customer_sk] - CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [d_month_seq] - CometColumnarExchange [d_month_seq] #5 - CometHashAggregate [d_month_seq] - CometProject [d_month_seq] - CometFilter [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - Project [i_item_sk] - BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] - ColumnarToRow - InputAdapter - CometFilter [i_item_sk,i_current_price,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Filter [avg(i_current_price)] - HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count] + InputAdapter + Exchange [ca_state] #1 + WholeStageCodegen (4) + HashAggregate [ca_state] [count,count] + Project [ca_state] + BroadcastHashJoin [ss_item_sk,i_item_sk] + ColumnarToRow + InputAdapter + CometProject [ca_state,ss_item_sk] + CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] + CometProject [ca_state,ss_item_sk,ss_sold_date_sk] + CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometProject [ca_state,c_customer_sk] + CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) ColumnarToRow InputAdapter - CometColumnarExchange [i_category] #9 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_current_price] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometFilter [i_current_price,i_category] - CometScan parquet spark_catalog.default.item [i_current_price,i_category] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + ReusedSubquery [d_month_seq] #2 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + Subquery #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + ReusedSubquery [d_month_seq] #2 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + ReusedSubquery [d_month_seq] #2 + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + Project [i_item_sk] + BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_current_price,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) + Filter [avg(i_current_price)] + HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count] + InputAdapter + Exchange [i_category] #9 + WholeStageCodegen (1) + HashAggregate [i_category,i_current_price] [sum,count,sum,count] + ColumnarToRow + InputAdapter + CometFilter [i_current_price,i_category] + CometScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt index 1f6d378e8..342a3deb8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/explain.txt @@ -1,75 +1,67 @@ == Physical Plan == -TakeOrderedAndProject (71) -+- * HashAggregate (70) - +- * ColumnarToRow (69) - +- CometColumnarExchange (68) - +- RowToColumnar (67) - +- * HashAggregate (66) - +- Union (65) - :- * HashAggregate (30) - : +- * ColumnarToRow (29) - : +- CometColumnarExchange (28) - : +- RowToColumnar (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer_address (9) - : +- CometBroadcastExchange (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.item (15) - : +- CometBroadcastExchange (20) - : +- CometProject (19) - : +- CometFilter (18) - : +- CometScan parquet spark_catalog.default.item (17) - :- * HashAggregate (47) - : +- * ColumnarToRow (46) - : +- CometColumnarExchange (45) - : +- RowToColumnar (44) - : +- * HashAggregate (43) - : +- * ColumnarToRow (42) - : +- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (38) - : : +- CometBroadcastHashJoin (37) - : : :- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometFilter (32) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (31) - : : : +- ReusedExchange (33) - : : +- ReusedExchange (36) - : +- ReusedExchange (39) - +- * HashAggregate (64) - +- * ColumnarToRow (63) - +- CometColumnarExchange (62) - +- RowToColumnar (61) - +- * HashAggregate (60) - +- * ColumnarToRow (59) - +- CometProject (58) - +- CometBroadcastHashJoin (57) - :- CometProject (55) - : +- CometBroadcastHashJoin (54) - : :- CometProject (52) - : : +- CometBroadcastHashJoin (51) - : : :- CometFilter (49) - : : : +- CometScan parquet spark_catalog.default.web_sales (48) - : : +- ReusedExchange (50) - : +- ReusedExchange (53) - +- ReusedExchange (56) +TakeOrderedAndProject (63) ++- * HashAggregate (62) + +- Exchange (61) + +- * HashAggregate (60) + +- Union (59) + :- * HashAggregate (28) + : +- Exchange (27) + : +- * HashAggregate (26) + : +- * ColumnarToRow (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.customer_address (9) + : +- CometBroadcastExchange (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (16) + : : +- CometScan parquet spark_catalog.default.item (15) + : +- CometBroadcastExchange (20) + : +- CometProject (19) + : +- CometFilter (18) + : +- CometScan parquet spark_catalog.default.item (17) + :- * HashAggregate (43) + : +- Exchange (42) + : +- * HashAggregate (41) + : +- * ColumnarToRow (40) + : +- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- * HashAggregate (58) + +- Exchange (57) + +- * HashAggregate (56) + +- * ColumnarToRow (55) + +- CometProject (54) + +- CometBroadcastHashJoin (53) + :- CometProject (51) + : +- CometBroadcastHashJoin (50) + : :- CometProject (48) + : : +- CometBroadcastHashJoin (47) + : : :- CometFilter (45) + : : : +- CometScan parquet spark_catalog.default.web_sales (44) + : : +- ReusedExchange (46) + : +- ReusedExchange (49) + +- ReusedExchange (52) (1) Scan parquet spark_catalog.default.store_sales @@ -198,24 +190,18 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum#15] Results [2]: [i_item_id#12, sum#16] -(27) RowToColumnar +(27) Exchange Input [2]: [i_item_id#12, sum#16] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(28) CometColumnarExchange -Input [2]: [i_item_id#12, sum#16] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(29) ColumnarToRow [codegen id : 2] -Input [2]: [i_item_id#12, sum#16] - -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [2]: [i_item_id#12, sum#16] Keys [1]: [i_item_id#12] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#3))#17] Results [2]: [i_item_id#12, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#17,17,2) AS total_sales#18] -(31) Scan parquet spark_catalog.default.catalog_sales +(29) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22] Batched: true Location: InMemoryFileIndex [] @@ -223,74 +209,68 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#22), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_addr_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(32) CometFilter +(30) CometFilter Input [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22] Condition : (isnotnull(cs_bill_addr_sk#19) AND isnotnull(cs_item_sk#20)) -(33) ReusedExchange [Reuses operator id: 6] +(31) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#24] -(34) CometBroadcastHashJoin +(32) CometBroadcastHashJoin Left output [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22] Right output [1]: [d_date_sk#24] Arguments: [cs_sold_date_sk#22], [d_date_sk#24], Inner, BuildRight -(35) CometProject +(33) CometProject Input [5]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, cs_sold_date_sk#22, d_date_sk#24] Arguments: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21], [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21] -(36) ReusedExchange [Reuses operator id: 12] +(34) ReusedExchange [Reuses operator id: 12] Output [1]: [ca_address_sk#25] -(37) CometBroadcastHashJoin +(35) CometBroadcastHashJoin Left output [3]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21] Right output [1]: [ca_address_sk#25] Arguments: [cs_bill_addr_sk#19], [ca_address_sk#25], Inner, BuildRight -(38) CometProject +(36) CometProject Input [4]: [cs_bill_addr_sk#19, cs_item_sk#20, cs_ext_sales_price#21, ca_address_sk#25] Arguments: [cs_item_sk#20, cs_ext_sales_price#21], [cs_item_sk#20, cs_ext_sales_price#21] -(39) ReusedExchange [Reuses operator id: 22] +(37) ReusedExchange [Reuses operator id: 22] Output [2]: [i_item_sk#26, i_item_id#27] -(40) CometBroadcastHashJoin +(38) CometBroadcastHashJoin Left output [2]: [cs_item_sk#20, cs_ext_sales_price#21] Right output [2]: [i_item_sk#26, i_item_id#27] Arguments: [cs_item_sk#20], [i_item_sk#26], Inner, BuildRight -(41) CometProject +(39) CometProject Input [4]: [cs_item_sk#20, cs_ext_sales_price#21, i_item_sk#26, i_item_id#27] Arguments: [cs_ext_sales_price#21, i_item_id#27], [cs_ext_sales_price#21, i_item_id#27] -(42) ColumnarToRow [codegen id : 3] +(40) ColumnarToRow [codegen id : 3] Input [2]: [cs_ext_sales_price#21, i_item_id#27] -(43) HashAggregate [codegen id : 3] +(41) HashAggregate [codegen id : 3] Input [2]: [cs_ext_sales_price#21, i_item_id#27] Keys [1]: [i_item_id#27] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#21))] Aggregate Attributes [1]: [sum#28] Results [2]: [i_item_id#27, sum#29] -(44) RowToColumnar -Input [2]: [i_item_id#27, sum#29] - -(45) CometColumnarExchange -Input [2]: [i_item_id#27, sum#29] -Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(46) ColumnarToRow [codegen id : 4] +(42) Exchange Input [2]: [i_item_id#27, sum#29] +Arguments: hashpartitioning(i_item_id#27, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(47) HashAggregate [codegen id : 4] +(43) HashAggregate [codegen id : 4] Input [2]: [i_item_id#27, sum#29] Keys [1]: [i_item_id#27] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#21))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#21))#30] Results [2]: [i_item_id#27, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#21))#30,17,2) AS total_sales#31] -(48) Scan parquet spark_catalog.default.web_sales +(44) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35] Batched: true Location: InMemoryFileIndex [] @@ -298,137 +278,125 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#35), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_addr_sk), IsNotNull(ws_item_sk)] ReadSchema: struct -(49) CometFilter +(45) CometFilter Input [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35] Condition : (isnotnull(ws_bill_addr_sk#33) AND isnotnull(ws_item_sk#32)) -(50) ReusedExchange [Reuses operator id: 6] +(46) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#37] -(51) CometBroadcastHashJoin +(47) CometBroadcastHashJoin Left output [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35] Right output [1]: [d_date_sk#37] Arguments: [ws_sold_date_sk#35], [d_date_sk#37], Inner, BuildRight -(52) CometProject +(48) CometProject Input [5]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ws_sold_date_sk#35, d_date_sk#37] Arguments: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34], [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34] -(53) ReusedExchange [Reuses operator id: 12] +(49) ReusedExchange [Reuses operator id: 12] Output [1]: [ca_address_sk#38] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [3]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34] Right output [1]: [ca_address_sk#38] Arguments: [ws_bill_addr_sk#33], [ca_address_sk#38], Inner, BuildRight -(55) CometProject +(51) CometProject Input [4]: [ws_item_sk#32, ws_bill_addr_sk#33, ws_ext_sales_price#34, ca_address_sk#38] Arguments: [ws_item_sk#32, ws_ext_sales_price#34], [ws_item_sk#32, ws_ext_sales_price#34] -(56) ReusedExchange [Reuses operator id: 22] +(52) ReusedExchange [Reuses operator id: 22] Output [2]: [i_item_sk#39, i_item_id#40] -(57) CometBroadcastHashJoin +(53) CometBroadcastHashJoin Left output [2]: [ws_item_sk#32, ws_ext_sales_price#34] Right output [2]: [i_item_sk#39, i_item_id#40] Arguments: [ws_item_sk#32], [i_item_sk#39], Inner, BuildRight -(58) CometProject +(54) CometProject Input [4]: [ws_item_sk#32, ws_ext_sales_price#34, i_item_sk#39, i_item_id#40] Arguments: [ws_ext_sales_price#34, i_item_id#40], [ws_ext_sales_price#34, i_item_id#40] -(59) ColumnarToRow [codegen id : 5] +(55) ColumnarToRow [codegen id : 5] Input [2]: [ws_ext_sales_price#34, i_item_id#40] -(60) HashAggregate [codegen id : 5] +(56) HashAggregate [codegen id : 5] Input [2]: [ws_ext_sales_price#34, i_item_id#40] Keys [1]: [i_item_id#40] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#34))] Aggregate Attributes [1]: [sum#41] Results [2]: [i_item_id#40, sum#42] -(61) RowToColumnar +(57) Exchange Input [2]: [i_item_id#40, sum#42] +Arguments: hashpartitioning(i_item_id#40, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(62) CometColumnarExchange -Input [2]: [i_item_id#40, sum#42] -Arguments: hashpartitioning(i_item_id#40, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(63) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_id#40, sum#42] - -(64) HashAggregate [codegen id : 6] +(58) HashAggregate [codegen id : 6] Input [2]: [i_item_id#40, sum#42] Keys [1]: [i_item_id#40] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#34))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#34))#43] Results [2]: [i_item_id#40, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#34))#43,17,2) AS total_sales#44] -(65) Union +(59) Union -(66) HashAggregate [codegen id : 7] +(60) HashAggregate [codegen id : 7] Input [2]: [i_item_id#12, total_sales#18] Keys [1]: [i_item_id#12] Functions [1]: [partial_sum(total_sales#18)] Aggregate Attributes [2]: [sum#45, isEmpty#46] Results [3]: [i_item_id#12, sum#47, isEmpty#48] -(67) RowToColumnar -Input [3]: [i_item_id#12, sum#47, isEmpty#48] - -(68) CometColumnarExchange -Input [3]: [i_item_id#12, sum#47, isEmpty#48] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(69) ColumnarToRow [codegen id : 8] +(61) Exchange Input [3]: [i_item_id#12, sum#47, isEmpty#48] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(70) HashAggregate [codegen id : 8] +(62) HashAggregate [codegen id : 8] Input [3]: [i_item_id#12, sum#47, isEmpty#48] Keys [1]: [i_item_id#12] Functions [1]: [sum(total_sales#18)] Aggregate Attributes [1]: [sum(total_sales#18)#49] Results [2]: [i_item_id#12, sum(total_sales#18)#49 AS total_sales#50] -(71) TakeOrderedAndProject +(63) TakeOrderedAndProject Input [2]: [i_item_id#12, total_sales#50] Arguments: 100, [i_item_id#12 ASC NULLS FIRST, total_sales#50 ASC NULLS FIRST], [i_item_id#12, total_sales#50] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (76) -+- * ColumnarToRow (75) - +- CometProject (74) - +- CometFilter (73) - +- CometScan parquet spark_catalog.default.date_dim (72) +BroadcastExchange (68) ++- * ColumnarToRow (67) + +- CometProject (66) + +- CometFilter (65) + +- CometScan parquet spark_catalog.default.date_dim (64) -(72) Scan parquet spark_catalog.default.date_dim +(64) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#6, d_year#7, d_moy#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,9), IsNotNull(d_date_sk)] ReadSchema: struct -(73) CometFilter +(65) CometFilter Input [3]: [d_date_sk#6, d_year#7, d_moy#8] Condition : ((((isnotnull(d_year#7) AND isnotnull(d_moy#8)) AND (d_year#7 = 1998)) AND (d_moy#8 = 9)) AND isnotnull(d_date_sk#6)) -(74) CometProject +(66) CometProject Input [3]: [d_date_sk#6, d_year#7, d_moy#8] Arguments: [d_date_sk#6], [d_date_sk#6] -(75) ColumnarToRow [codegen id : 1] +(67) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(76) BroadcastExchange +(68) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -Subquery:2 Hosting operator id = 31 Hosting Expression = cs_sold_date_sk#22 IN dynamicpruning#5 +Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#22 IN dynamicpruning#5 -Subquery:3 Hosting operator id = 48 Hosting Expression = ws_sold_date_sk#35 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 44 Hosting Expression = ws_sold_date_sk#35 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt index 6c079c1e4..51025cf34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q60/simplified.txt @@ -1,97 +1,89 @@ TakeOrderedAndProject [i_item_id,total_sales] WholeStageCodegen (8) HashAggregate [i_item_id,sum,isEmpty] [sum(total_sales),total_sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #1 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] - CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ca_address_sk] #5 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk,i_item_id] #6 - CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [i_item_id] #7 - CometProject [i_item_id] - CometFilter [i_item_id,i_category] - CometScan parquet spark_catalog.default.item [i_item_id,i_category] - WholeStageCodegen (4) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #8 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] - CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] - CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 - WholeStageCodegen (6) - HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #9 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] - CometProject [ws_item_sk,ws_ext_sales_price] - CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] - CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] - CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [ca_address_sk] #5 - ReusedExchange [i_item_sk,i_item_id] #6 + InputAdapter + Exchange [i_item_id] #1 + WholeStageCodegen (7) + HashAggregate [i_item_id,total_sales] [sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (2) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ss_ext_sales_price)),total_sales,sum] + InputAdapter + Exchange [i_item_id] #2 + WholeStageCodegen (1) + HashAggregate [i_item_id,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ca_address_sk] + CometProject [ss_item_sk,ss_addr_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_addr_sk,ss_ext_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ca_address_sk] #5 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [i_item_id] #7 + CometProject [i_item_id] + CometFilter [i_item_id,i_category] + CometScan parquet spark_catalog.default.item [i_item_id,i_category] + WholeStageCodegen (4) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(cs_ext_sales_price)),total_sales,sum] + InputAdapter + Exchange [i_item_id] #8 + WholeStageCodegen (3) + HashAggregate [i_item_id,cs_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [cs_ext_sales_price,i_item_id] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,ca_address_sk] + CometProject [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price] + CometBroadcastHashJoin [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [ca_address_sk] #5 + ReusedExchange [i_item_sk,i_item_id] #6 + WholeStageCodegen (6) + HashAggregate [i_item_id,sum] [sum(UnscaledValue(ws_ext_sales_price)),total_sales,sum] + InputAdapter + Exchange [i_item_id] #9 + WholeStageCodegen (5) + HashAggregate [i_item_id,ws_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_ext_sales_price,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ca_address_sk] + CometProject [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price] + CometBroadcastHashJoin [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_addr_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [ca_address_sk] #5 + ReusedExchange [i_item_sk,i_item_id] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt index 61610b6ee..6d0d03e13 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/explain.txt @@ -1,73 +1,69 @@ == Physical Plan == -* Project (69) -+- * BroadcastNestedLoopJoin Inner BuildRight (68) - :- * HashAggregate (43) - : +- * ColumnarToRow (42) - : +- CometColumnarExchange (41) - : +- RowToColumnar (40) - : +- * HashAggregate (39) - : +- * ColumnarToRow (38) - : +- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) - : : : :- CometProject (20) - : : : : +- CometBroadcastHashJoin (19) - : : : : :- CometProject (14) - : : : : : +- CometBroadcastHashJoin (13) - : : : : : :- CometProject (8) - : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : +- CometBroadcastExchange (6) - : : : : : : +- CometProject (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.store (3) - : : : : : +- CometBroadcastExchange (12) - : : : : : +- CometProject (11) - : : : : : +- CometFilter (10) - : : : : : +- CometScan parquet spark_catalog.default.promotion (9) - : : : : +- CometBroadcastExchange (18) - : : : : +- CometProject (17) - : : : : +- CometFilter (16) - : : : : +- CometScan parquet spark_catalog.default.date_dim (15) - : : : +- CometBroadcastExchange (23) - : : : +- CometFilter (22) - : : : +- CometScan parquet spark_catalog.default.customer (21) - : : +- CometBroadcastExchange (29) - : : +- CometProject (28) - : : +- CometFilter (27) - : : +- CometScan parquet spark_catalog.default.customer_address (26) - : +- CometBroadcastExchange (35) - : +- CometProject (34) - : +- CometFilter (33) - : +- CometScan parquet spark_catalog.default.item (32) - +- BroadcastExchange (67) - +- * HashAggregate (66) - +- * ColumnarToRow (65) - +- CometColumnarExchange (64) - +- RowToColumnar (63) - +- * HashAggregate (62) - +- * ColumnarToRow (61) - +- CometProject (60) - +- CometBroadcastHashJoin (59) - :- CometProject (57) - : +- CometBroadcastHashJoin (56) - : :- CometProject (54) - : : +- CometBroadcastHashJoin (53) - : : :- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometProject (48) - : : : : +- CometBroadcastHashJoin (47) - : : : : :- CometFilter (45) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (44) - : : : : +- ReusedExchange (46) - : : : +- ReusedExchange (49) - : : +- ReusedExchange (52) - : +- ReusedExchange (55) - +- ReusedExchange (58) +* Project (65) ++- * BroadcastNestedLoopJoin Inner BuildRight (64) + :- * HashAggregate (41) + : +- Exchange (40) + : +- * HashAggregate (39) + : +- * ColumnarToRow (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (31) + : : +- CometBroadcastHashJoin (30) + : : :- CometProject (25) + : : : +- CometBroadcastHashJoin (24) + : : : :- CometProject (20) + : : : : +- CometBroadcastHashJoin (19) + : : : : :- CometProject (14) + : : : : : +- CometBroadcastHashJoin (13) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometProject (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan parquet spark_catalog.default.store (3) + : : : : : +- CometBroadcastExchange (12) + : : : : : +- CometProject (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan parquet spark_catalog.default.promotion (9) + : : : : +- CometBroadcastExchange (18) + : : : : +- CometProject (17) + : : : : +- CometFilter (16) + : : : : +- CometScan parquet spark_catalog.default.date_dim (15) + : : : +- CometBroadcastExchange (23) + : : : +- CometFilter (22) + : : : +- CometScan parquet spark_catalog.default.customer (21) + : : +- CometBroadcastExchange (29) + : : +- CometProject (28) + : : +- CometFilter (27) + : : +- CometScan parquet spark_catalog.default.customer_address (26) + : +- CometBroadcastExchange (35) + : +- CometProject (34) + : +- CometFilter (33) + : +- CometScan parquet spark_catalog.default.item (32) + +- BroadcastExchange (63) + +- * HashAggregate (62) + +- Exchange (61) + +- * HashAggregate (60) + +- * ColumnarToRow (59) + +- CometProject (58) + +- CometBroadcastHashJoin (57) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometProject (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometProject (49) + : : : +- CometBroadcastHashJoin (48) + : : : :- CometProject (46) + : : : : +- CometBroadcastHashJoin (45) + : : : : :- CometFilter (43) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (42) + : : : : +- ReusedExchange (44) + : : : +- ReusedExchange (47) + : : +- ReusedExchange (50) + : +- ReusedExchange (53) + +- ReusedExchange (56) (1) Scan parquet spark_catalog.default.store_sales @@ -256,24 +252,18 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum#23] Results [1]: [sum#24] -(40) RowToColumnar +(40) Exchange Input [1]: [sum#24] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] -(41) CometColumnarExchange -Input [1]: [sum#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(42) ColumnarToRow [codegen id : 4] -Input [1]: [sum#24] - -(43) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [1]: [sum#24] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#5))#25] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#5))#25,17,2) AS promotions#26] -(44) Scan parquet spark_catalog.default.store_sales +(42) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#27, ss_customer_sk#28, ss_store_sk#29, ss_ext_sales_price#30, ss_sold_date_sk#31] Batched: true Location: InMemoryFileIndex [] @@ -281,141 +271,135 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#31), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(45) CometFilter +(43) CometFilter Input [5]: [ss_item_sk#27, ss_customer_sk#28, ss_store_sk#29, ss_ext_sales_price#30, ss_sold_date_sk#31] Condition : ((isnotnull(ss_store_sk#29) AND isnotnull(ss_customer_sk#28)) AND isnotnull(ss_item_sk#27)) -(46) ReusedExchange [Reuses operator id: 6] +(44) ReusedExchange [Reuses operator id: 6] Output [1]: [s_store_sk#33] -(47) CometBroadcastHashJoin +(45) CometBroadcastHashJoin Left output [5]: [ss_item_sk#27, ss_customer_sk#28, ss_store_sk#29, ss_ext_sales_price#30, ss_sold_date_sk#31] Right output [1]: [s_store_sk#33] Arguments: [ss_store_sk#29], [s_store_sk#33], Inner, BuildRight -(48) CometProject +(46) CometProject Input [6]: [ss_item_sk#27, ss_customer_sk#28, ss_store_sk#29, ss_ext_sales_price#30, ss_sold_date_sk#31, s_store_sk#33] Arguments: [ss_item_sk#27, ss_customer_sk#28, ss_ext_sales_price#30, ss_sold_date_sk#31], [ss_item_sk#27, ss_customer_sk#28, ss_ext_sales_price#30, ss_sold_date_sk#31] -(49) ReusedExchange [Reuses operator id: 18] +(47) ReusedExchange [Reuses operator id: 18] Output [1]: [d_date_sk#34] -(50) CometBroadcastHashJoin +(48) CometBroadcastHashJoin Left output [4]: [ss_item_sk#27, ss_customer_sk#28, ss_ext_sales_price#30, ss_sold_date_sk#31] Right output [1]: [d_date_sk#34] Arguments: [ss_sold_date_sk#31], [d_date_sk#34], Inner, BuildRight -(51) CometProject +(49) CometProject Input [5]: [ss_item_sk#27, ss_customer_sk#28, ss_ext_sales_price#30, ss_sold_date_sk#31, d_date_sk#34] Arguments: [ss_item_sk#27, ss_customer_sk#28, ss_ext_sales_price#30], [ss_item_sk#27, ss_customer_sk#28, ss_ext_sales_price#30] -(52) ReusedExchange [Reuses operator id: 23] +(50) ReusedExchange [Reuses operator id: 23] Output [2]: [c_customer_sk#35, c_current_addr_sk#36] -(53) CometBroadcastHashJoin +(51) CometBroadcastHashJoin Left output [3]: [ss_item_sk#27, ss_customer_sk#28, ss_ext_sales_price#30] Right output [2]: [c_customer_sk#35, c_current_addr_sk#36] Arguments: [ss_customer_sk#28], [c_customer_sk#35], Inner, BuildRight -(54) CometProject +(52) CometProject Input [5]: [ss_item_sk#27, ss_customer_sk#28, ss_ext_sales_price#30, c_customer_sk#35, c_current_addr_sk#36] Arguments: [ss_item_sk#27, ss_ext_sales_price#30, c_current_addr_sk#36], [ss_item_sk#27, ss_ext_sales_price#30, c_current_addr_sk#36] -(55) ReusedExchange [Reuses operator id: 29] +(53) ReusedExchange [Reuses operator id: 29] Output [1]: [ca_address_sk#37] -(56) CometBroadcastHashJoin +(54) CometBroadcastHashJoin Left output [3]: [ss_item_sk#27, ss_ext_sales_price#30, c_current_addr_sk#36] Right output [1]: [ca_address_sk#37] Arguments: [c_current_addr_sk#36], [ca_address_sk#37], Inner, BuildRight -(57) CometProject +(55) CometProject Input [4]: [ss_item_sk#27, ss_ext_sales_price#30, c_current_addr_sk#36, ca_address_sk#37] Arguments: [ss_item_sk#27, ss_ext_sales_price#30], [ss_item_sk#27, ss_ext_sales_price#30] -(58) ReusedExchange [Reuses operator id: 35] +(56) ReusedExchange [Reuses operator id: 35] Output [1]: [i_item_sk#38] -(59) CometBroadcastHashJoin +(57) CometBroadcastHashJoin Left output [2]: [ss_item_sk#27, ss_ext_sales_price#30] Right output [1]: [i_item_sk#38] Arguments: [ss_item_sk#27], [i_item_sk#38], Inner, BuildRight -(60) CometProject +(58) CometProject Input [3]: [ss_item_sk#27, ss_ext_sales_price#30, i_item_sk#38] Arguments: [ss_ext_sales_price#30], [ss_ext_sales_price#30] -(61) ColumnarToRow [codegen id : 2] +(59) ColumnarToRow [codegen id : 2] Input [1]: [ss_ext_sales_price#30] -(62) HashAggregate [codegen id : 2] +(60) HashAggregate [codegen id : 2] Input [1]: [ss_ext_sales_price#30] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#30))] Aggregate Attributes [1]: [sum#39] Results [1]: [sum#40] -(63) RowToColumnar -Input [1]: [sum#40] - -(64) CometColumnarExchange -Input [1]: [sum#40] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(65) ColumnarToRow [codegen id : 3] +(61) Exchange Input [1]: [sum#40] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(66) HashAggregate [codegen id : 3] +(62) HashAggregate [codegen id : 3] Input [1]: [sum#40] Keys: [] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#30))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#30))#41] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#30))#41,17,2) AS total#42] -(67) BroadcastExchange +(63) BroadcastExchange Input [1]: [total#42] Arguments: IdentityBroadcastMode, [plan_id=3] -(68) BroadcastNestedLoopJoin [codegen id : 4] +(64) BroadcastNestedLoopJoin [codegen id : 4] Join type: Inner Join condition: None -(69) Project [codegen id : 4] +(65) Project [codegen id : 4] Output [3]: [promotions#26, total#42, ((cast(promotions#26 as decimal(15,4)) / cast(total#42 as decimal(15,4))) * 100) AS ((CAST(promotions AS DECIMAL(15,4)) / CAST(total AS DECIMAL(15,4))) * 100)#43] Input [2]: [promotions#26, total#42] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (74) -+- * ColumnarToRow (73) - +- CometProject (72) - +- CometFilter (71) - +- CometScan parquet spark_catalog.default.date_dim (70) +BroadcastExchange (70) ++- * ColumnarToRow (69) + +- CometProject (68) + +- CometFilter (67) + +- CometScan parquet spark_catalog.default.date_dim (66) -(70) Scan parquet spark_catalog.default.date_dim +(66) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#14, d_year#15, d_moy#16] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(71) CometFilter +(67) CometFilter Input [3]: [d_date_sk#14, d_year#15, d_moy#16] Condition : ((((isnotnull(d_year#15) AND isnotnull(d_moy#16)) AND (d_year#15 = 1998)) AND (d_moy#16 = 11)) AND isnotnull(d_date_sk#14)) -(72) CometProject +(68) CometProject Input [3]: [d_date_sk#14, d_year#15, d_moy#16] Arguments: [d_date_sk#14], [d_date_sk#14] -(73) ColumnarToRow [codegen id : 1] +(69) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(74) BroadcastExchange +(70) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -Subquery:2 Hosting operator id = 44 Hosting Expression = ss_sold_date_sk#31 IN dynamicpruning#7 +Subquery:2 Hosting operator id = 42 Hosting Expression = ss_sold_date_sk#31 IN dynamicpruning#7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt index e38989b41..8d39d9e4e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q61/simplified.txt @@ -2,86 +2,82 @@ WholeStageCodegen (4) Project [promotions,total] BroadcastNestedLoopJoin HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),promotions,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] - CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,p_promo_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk] #3 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_gmt_offset] - CometScan parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] - CometBroadcastExchange [p_promo_sk] #4 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 - CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #7 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [i_item_sk] #8 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_category] + InputAdapter + Exchange #1 + WholeStageCodegen (1) + HashAggregate [ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] + CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,p_promo_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk] #3 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_gmt_offset] + CometScan parquet spark_catalog.default.store [s_store_sk,s_gmt_offset] + CometBroadcastExchange [p_promo_sk] #4 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_dmail,p_channel_email,p_channel_tv] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #6 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #7 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [i_item_sk] #8 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_category] InputAdapter BroadcastExchange #9 WholeStageCodegen (3) HashAggregate [sum] [sum(UnscaledValue(ss_ext_sales_price)),total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange #10 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] - CometProject [ss_item_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] - CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [s_store_sk] #3 - ReusedExchange [d_date_sk] #5 - ReusedExchange [c_customer_sk,c_current_addr_sk] #6 - ReusedExchange [ca_address_sk] #7 - ReusedExchange [i_item_sk] #8 + InputAdapter + Exchange #10 + WholeStageCodegen (2) + HashAggregate [ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] + CometProject [ss_item_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,c_current_addr_sk,ca_address_sk] + CometProject [ss_item_sk,ss_ext_sales_price,c_current_addr_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,c_customer_sk,c_current_addr_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,s_store_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [s_store_sk] #3 + ReusedExchange [d_date_sk] #5 + ReusedExchange [c_customer_sk,c_current_addr_sk] #6 + ReusedExchange [ca_address_sk] #7 + ReusedExchange [i_item_sk] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/explain.txt index e804b1dd3..cba9fc4b9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/explain.txt @@ -1,34 +1,32 @@ == Physical Plan == -TakeOrderedAndProject (30) -+- * HashAggregate (29) - +- * ColumnarToRow (28) - +- CometColumnarExchange (27) - +- RowToColumnar (26) - +- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.warehouse (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.ship_mode (8) - : +- CometBroadcastExchange (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.web_site (13) - +- CometBroadcastExchange (21) - +- CometProject (20) - +- CometFilter (19) - +- CometScan parquet spark_catalog.default.date_dim (18) +TakeOrderedAndProject (28) ++- * HashAggregate (27) + +- Exchange (26) + +- * HashAggregate (25) + +- * ColumnarToRow (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometProject (17) + : +- CometBroadcastHashJoin (16) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (15) + : +- CometFilter (14) + : +- CometScan parquet spark_catalog.default.web_site (13) + +- CometBroadcastExchange (21) + +- CometProject (20) + +- CometFilter (19) + +- CometScan parquet spark_catalog.default.date_dim (18) (1) Scan parquet spark_catalog.default.web_sales @@ -152,24 +150,18 @@ Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) < Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] Results [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -(26) RowToColumnar +(26) Exchange Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] +Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(27) CometColumnarExchange -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(28) ColumnarToRow [codegen id : 2] -Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] - -(29) HashAggregate [codegen id : 2] +(27) HashAggregate [codegen id : 2] Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] Functions [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] Aggregate Attributes [5]: [sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] -(30) TakeOrderedAndProject +(28) TakeOrderedAndProject Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, web_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, web_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/simplified.txt index 9cd3afd85..b9e35e282 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q62/simplified.txt @@ -1,34 +1,32 @@ TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] WholeStageCodegen (2) HashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [_groupingexpression,sm_type,web_name] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [_groupingexpression,sm_type,web_name,ws_ship_date_sk,ws_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] - CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] - CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_site_sk,web_name] - CometProject [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] - CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] - CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 - CometFilter [sm_ship_mode_sk,sm_type] - CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] - CometBroadcastExchange [web_site_sk,web_name] #4 - CometFilter [web_site_sk,web_name] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_name] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + Exchange [_groupingexpression,sm_type,web_name] #1 + WholeStageCodegen (1) + HashAggregate [_groupingexpression,sm_type,web_name,ws_ship_date_sk,ws_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] + CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] + CometProject [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_site_sk,web_name] + CometProject [ws_ship_date_sk,ws_web_site_sk,ws_sold_date_sk,w_warehouse_name,sm_type] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] + CometProject [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_sold_date_sk,w_warehouse_name] + CometBroadcastHashJoin [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_web_site_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometFilter [sm_ship_mode_sk,sm_type] + CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [web_site_sk,web_name] #4 + CometFilter [web_site_sk,web_name] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt index 53b396c29..c10d628ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/explain.txt @@ -1,37 +1,33 @@ == Physical Plan == -TakeOrderedAndProject (33) -+- * Project (32) - +- * Filter (31) - +- Window (30) - +- * ColumnarToRow (29) - +- CometSort (28) - +- CometColumnarExchange (27) - +- RowToColumnar (26) - +- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometColumnarExchange (23) - +- RowToColumnar (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometProject (19) - +- CometBroadcastHashJoin (18) - :- CometProject (14) - : +- CometBroadcastHashJoin (13) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.item (1) - : : +- CometBroadcastExchange (6) - : : +- CometFilter (5) - : : +- CometScan parquet spark_catalog.default.store_sales (4) - : +- CometBroadcastExchange (12) - : +- CometProject (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.date_dim (9) - +- CometBroadcastExchange (17) - +- CometFilter (16) - +- CometScan parquet spark_catalog.default.store (15) +TakeOrderedAndProject (29) ++- * Project (28) + +- * Filter (27) + +- Window (26) + +- * Sort (25) + +- Exchange (24) + +- * HashAggregate (23) + +- Exchange (22) + +- * HashAggregate (21) + +- * ColumnarToRow (20) + +- CometProject (19) + +- CometBroadcastHashJoin (18) + :- CometProject (14) + : +- CometBroadcastHashJoin (13) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (6) + : : +- CometFilter (5) + : : +- CometScan parquet spark_catalog.default.store_sales (4) + : +- CometBroadcastExchange (12) + : +- CometProject (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.date_dim (9) + +- CometBroadcastExchange (17) + +- CometFilter (16) + +- CometScan parquet spark_catalog.default.store (15) (1) Scan parquet spark_catalog.default.item @@ -136,82 +132,70 @@ Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] Aggregate Attributes [1]: [sum#19] Results [3]: [i_manager_id#5, d_moy#17, sum#20] -(22) RowToColumnar +(22) Exchange Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(23) CometColumnarExchange -Input [3]: [i_manager_id#5, d_moy#17, sum#20] -Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(24) ColumnarToRow [codegen id : 2] -Input [3]: [i_manager_id#5, d_moy#17, sum#20] - -(25) HashAggregate [codegen id : 2] +(23) HashAggregate [codegen id : 2] Input [3]: [i_manager_id#5, d_moy#17, sum#20] Keys [2]: [i_manager_id#5, d_moy#17] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#21] Results [3]: [i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#21,17,2) AS sum_sales#22, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#21,17,2) AS _w0#23] -(26) RowToColumnar -Input [3]: [i_manager_id#5, sum_sales#22, _w0#23] - -(27) CometColumnarExchange -Input [3]: [i_manager_id#5, sum_sales#22, _w0#23] -Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(28) CometSort +(24) Exchange Input [3]: [i_manager_id#5, sum_sales#22, _w0#23] -Arguments: [i_manager_id#5, sum_sales#22, _w0#23], [i_manager_id#5 ASC NULLS FIRST] +Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(29) ColumnarToRow [codegen id : 3] +(25) Sort [codegen id : 3] Input [3]: [i_manager_id#5, sum_sales#22, _w0#23] +Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 -(30) Window +(26) Window Input [3]: [i_manager_id#5, sum_sales#22, _w0#23] Arguments: [avg(_w0#23) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#24], [i_manager_id#5] -(31) Filter [codegen id : 4] +(27) Filter [codegen id : 4] Input [4]: [i_manager_id#5, sum_sales#22, _w0#23, avg_monthly_sales#24] Condition : CASE WHEN (avg_monthly_sales#24 > 0.000000) THEN ((abs((sum_sales#22 - avg_monthly_sales#24)) / avg_monthly_sales#24) > 0.1000000000000000) ELSE false END -(32) Project [codegen id : 4] +(28) Project [codegen id : 4] Output [3]: [i_manager_id#5, sum_sales#22, avg_monthly_sales#24] Input [4]: [i_manager_id#5, sum_sales#22, _w0#23, avg_monthly_sales#24] -(33) TakeOrderedAndProject +(29) TakeOrderedAndProject Input [3]: [i_manager_id#5, sum_sales#22, avg_monthly_sales#24] Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#24 ASC NULLS FIRST, sum_sales#22 ASC NULLS FIRST], [i_manager_id#5, sum_sales#22, avg_monthly_sales#24] ===== Subqueries ===== Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 -BroadcastExchange (38) -+- * ColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan parquet spark_catalog.default.date_dim (34) +BroadcastExchange (34) ++- * ColumnarToRow (33) + +- CometProject (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.date_dim (30) -(34) Scan parquet spark_catalog.default.date_dim +(30) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct -(35) CometFilter +(31) CometFilter Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] Condition : (d_month_seq#16 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) -(36) CometProject +(32) CometProject Input [3]: [d_date_sk#15, d_month_seq#16, d_moy#17] Arguments: [d_date_sk#15, d_moy#17], [d_date_sk#15, d_moy#17] -(37) ColumnarToRow [codegen id : 1] +(33) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#15, d_moy#17] -(38) BroadcastExchange +(34) BroadcastExchange Input [2]: [d_date_sk#15, d_moy#17] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt index 34a22f010..e84d3c2a0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q63/simplified.txt @@ -5,45 +5,41 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] InputAdapter Window [_w0,i_manager_id] WholeStageCodegen (3) - ColumnarToRow + Sort [i_manager_id] InputAdapter - CometSort [i_manager_id,sum_sales,_w0] - CometColumnarExchange [i_manager_id] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_manager_id,d_moy] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_manager_id,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] - CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] - CometProject [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometProject [i_item_sk,i_manager_id] - CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_month_seq,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [d_date_sk,d_moy] #5 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_month_seq,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] - CometBroadcastExchange [s_store_sk] #6 - CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] + Exchange [i_manager_id] #1 + WholeStageCodegen (2) + HashAggregate [i_manager_id,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + InputAdapter + Exchange [i_manager_id,d_moy] #2 + WholeStageCodegen (1) + HashAggregate [i_manager_id,d_moy,ss_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [i_manager_id,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] + CometProject [i_manager_id,ss_store_sk,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] + CometProject [i_manager_id,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_manager_id,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometProject [i_item_sk,i_manager_id] + CometFilter [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_manager_id] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_month_seq,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] + CometBroadcastExchange [d_date_sk,d_moy] #5 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_month_seq,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_moy] + CometBroadcastExchange [s_store_sk] #6 + CometFilter [s_store_sk] + CometScan parquet spark_catalog.default.store [s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt index cb87fc6fc..d4e450675 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/explain.txt @@ -1,193 +1,186 @@ == Physical Plan == -* ColumnarToRow (189) -+- CometSort (188) - +- CometColumnarExchange (187) - +- RowToColumnar (186) - +- * Project (185) - +- * SortMergeJoin Inner (184) - :- * ColumnarToRow (113) - : +- CometSort (112) - : +- CometColumnarExchange (111) - : +- RowToColumnar (110) - : +- * HashAggregate (109) - : +- * HashAggregate (108) - : +- * Project (107) - : +- * BroadcastHashJoin Inner BuildRight (106) - : :- * Project (100) - : : +- * BroadcastHashJoin Inner BuildRight (99) - : : :- * Project (97) - : : : +- * BroadcastHashJoin Inner BuildRight (96) - : : : :- * Project (91) - : : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : : :- * Project (88) - : : : : : +- * BroadcastHashJoin Inner BuildRight (87) - : : : : : :- * Project (82) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : :- * Project (79) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (78) - : : : : : : : :- * Project (73) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (72) - : : : : : : : : :- * Project (67) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : :- * Project (64) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (63) - : : : : : : : : : : :- * Project (58) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (57) - : : : : : : : : : : : :- * Project (55) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : : : : : : : : : : :- * Project (49) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (48) - : : : : : : : : : : : : : :- * Project (43) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (42) - : : : : : : : : : : : : : : :- * Project (37) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (36) - : : : : : : : : : : : : : : : :- * Project (34) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (33) - : : : : : : : : : : : : : : : : :- * ColumnarToRow (11) - : : : : : : : : : : : : : : : : : +- CometSort (10) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (9) - : : : : : : : : : : : : : : : : : +- CometProject (8) - : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) - : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : : : : : : : : : : : : +- CometProject (6) - : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) - : : : : : : : : : : : : : : : : +- * Sort (32) - : : : : : : : : : : : : : : : : +- * Project (31) - : : : : : : : : : : : : : : : : +- * Filter (30) - : : : : : : : : : : : : : : : : +- * HashAggregate (29) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (28) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (27) - : : : : : : : : : : : : : : : : +- RowToColumnar (26) - : : : : : : : : : : : : : : : : +- * HashAggregate (25) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (24) - : : : : : : : : : : : : : : : : +- CometProject (23) - : : : : : : : : : : : : : : : : +- CometSortMergeJoin (22) - : : : : : : : : : : : : : : : : :- CometSort (16) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (15) - : : : : : : : : : : : : : : : : : +- CometProject (14) - : : : : : : : : : : : : : : : : : +- CometFilter (13) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (12) - : : : : : : : : : : : : : : : : +- CometSort (21) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (20) - : : : : : : : : : : : : : : : : +- CometProject (19) - : : : : : : : : : : : : : : : : +- CometFilter (18) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (17) - : : : : : : : : : : : : : : : +- ReusedExchange (35) - : : : : : : : : : : : : : : +- BroadcastExchange (41) - : : : : : : : : : : : : : : +- * ColumnarToRow (40) - : : : : : : : : : : : : : : +- CometFilter (39) - : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store (38) - : : : : : : : : : : : : : +- BroadcastExchange (47) - : : : : : : : : : : : : : +- * ColumnarToRow (46) - : : : : : : : : : : : : : +- CometFilter (45) - : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer (44) - : : : : : : : : : : : : +- BroadcastExchange (53) - : : : : : : : : : : : : +- * ColumnarToRow (52) - : : : : : : : : : : : : +- CometFilter (51) - : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (50) - : : : : : : : : : : : +- ReusedExchange (56) - : : : : : : : : : : +- BroadcastExchange (62) - : : : : : : : : : : +- * ColumnarToRow (61) - : : : : : : : : : : +- CometFilter (60) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (59) - : : : : : : : : : +- ReusedExchange (65) - : : : : : : : : +- BroadcastExchange (71) - : : : : : : : : +- * ColumnarToRow (70) - : : : : : : : : +- CometFilter (69) - : : : : : : : : +- CometScan parquet spark_catalog.default.promotion (68) - : : : : : : : +- BroadcastExchange (77) - : : : : : : : +- * ColumnarToRow (76) - : : : : : : : +- CometFilter (75) - : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (74) - : : : : : : +- ReusedExchange (80) - : : : : : +- BroadcastExchange (86) - : : : : : +- * ColumnarToRow (85) - : : : : : +- CometFilter (84) - : : : : : +- CometScan parquet spark_catalog.default.customer_address (83) - : : : : +- ReusedExchange (89) - : : : +- BroadcastExchange (95) - : : : +- * ColumnarToRow (94) - : : : +- CometFilter (93) - : : : +- CometScan parquet spark_catalog.default.income_band (92) - : : +- ReusedExchange (98) - : +- BroadcastExchange (105) - : +- * ColumnarToRow (104) - : +- CometProject (103) - : +- CometFilter (102) - : +- CometScan parquet spark_catalog.default.item (101) - +- * ColumnarToRow (183) - +- CometSort (182) - +- CometColumnarExchange (181) - +- RowToColumnar (180) - +- * HashAggregate (179) - +- * HashAggregate (178) - +- * Project (177) - +- * BroadcastHashJoin Inner BuildRight (176) - :- * Project (174) - : +- * BroadcastHashJoin Inner BuildRight (173) - : :- * Project (171) - : : +- * BroadcastHashJoin Inner BuildRight (170) - : : :- * Project (168) - : : : +- * BroadcastHashJoin Inner BuildRight (167) - : : : :- * Project (165) - : : : : +- * BroadcastHashJoin Inner BuildRight (164) - : : : : :- * Project (162) - : : : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : : : :- * Project (159) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (158) - : : : : : : :- * Project (156) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (155) - : : : : : : : :- * Project (153) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) - : : : : : : : : :- * Project (150) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) - : : : : : : : : : :- * Project (147) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) - : : : : : : : : : : :- * Project (144) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : : : : : : : : : :- * Project (141) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : : : : : : : : : :- * Project (138) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : : : : : : : : : :- * Project (135) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) - : : : : : : : : : : : : : : :- * Project (132) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (131) - : : : : : : : : : : : : : : : :- * ColumnarToRow (124) - : : : : : : : : : : : : : : : : +- CometSort (123) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (122) - : : : : : : : : : : : : : : : : +- CometProject (121) - : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (120) - : : : : : : : : : : : : : : : : :- CometBroadcastExchange (116) - : : : : : : : : : : : : : : : : : +- CometFilter (115) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (114) - : : : : : : : : : : : : : : : : +- CometProject (119) - : : : : : : : : : : : : : : : : +- CometFilter (118) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (117) - : : : : : : : : : : : : : : : +- * Sort (130) - : : : : : : : : : : : : : : : +- * Project (129) - : : : : : : : : : : : : : : : +- * Filter (128) - : : : : : : : : : : : : : : : +- * HashAggregate (127) - : : : : : : : : : : : : : : : +- * ColumnarToRow (126) - : : : : : : : : : : : : : : : +- ReusedExchange (125) - : : : : : : : : : : : : : : +- ReusedExchange (133) - : : : : : : : : : : : : : +- ReusedExchange (136) - : : : : : : : : : : : : +- ReusedExchange (139) - : : : : : : : : : : : +- ReusedExchange (142) - : : : : : : : : : : +- ReusedExchange (145) - : : : : : : : : : +- ReusedExchange (148) - : : : : : : : : +- ReusedExchange (151) - : : : : : : : +- ReusedExchange (154) - : : : : : : +- ReusedExchange (157) - : : : : : +- ReusedExchange (160) - : : : : +- ReusedExchange (163) - : : : +- ReusedExchange (166) - : : +- ReusedExchange (169) - : +- ReusedExchange (172) - +- ReusedExchange (175) +* ColumnarToRow (182) ++- CometSort (181) + +- CometColumnarExchange (180) + +- RowToColumnar (179) + +- * Project (178) + +- * SortMergeJoin Inner (177) + :- * Sort (109) + : +- Exchange (108) + : +- * HashAggregate (107) + : +- * HashAggregate (106) + : +- * Project (105) + : +- * BroadcastHashJoin Inner BuildRight (104) + : :- * Project (98) + : : +- * BroadcastHashJoin Inner BuildRight (97) + : : :- * Project (95) + : : : +- * BroadcastHashJoin Inner BuildRight (94) + : : : :- * Project (89) + : : : : +- * BroadcastHashJoin Inner BuildRight (88) + : : : : :- * Project (86) + : : : : : +- * BroadcastHashJoin Inner BuildRight (85) + : : : : : :- * Project (80) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (79) + : : : : : : :- * Project (77) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (76) + : : : : : : : :- * Project (71) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (70) + : : : : : : : : :- * Project (65) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (64) + : : : : : : : : : :- * Project (62) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (61) + : : : : : : : : : : :- * Project (56) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (55) + : : : : : : : : : : : :- * Project (53) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (52) + : : : : : : : : : : : : :- * Project (47) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (46) + : : : : : : : : : : : : : :- * Project (41) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (40) + : : : : : : : : : : : : : : :- * Project (35) + : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (34) + : : : : : : : : : : : : : : : :- * Project (32) + : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (31) + : : : : : : : : : : : : : : : : :- * ColumnarToRow (11) + : : : : : : : : : : : : : : : : : +- CometSort (10) + : : : : : : : : : : : : : : : : : +- CometExchange (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- * Sort (30) + : : : : : : : : : : : : : : : : +- * Project (29) + : : : : : : : : : : : : : : : : +- * Filter (28) + : : : : : : : : : : : : : : : : +- * HashAggregate (27) + : : : : : : : : : : : : : : : : +- Exchange (26) + : : : : : : : : : : : : : : : : +- * HashAggregate (25) + : : : : : : : : : : : : : : : : +- * ColumnarToRow (24) + : : : : : : : : : : : : : : : : +- CometProject (23) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (22) + : : : : : : : : : : : : : : : : :- CometSort (16) + : : : : : : : : : : : : : : : : : +- CometExchange (15) + : : : : : : : : : : : : : : : : : +- CometProject (14) + : : : : : : : : : : : : : : : : : +- CometFilter (13) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (12) + : : : : : : : : : : : : : : : : +- CometSort (21) + : : : : : : : : : : : : : : : : +- CometExchange (20) + : : : : : : : : : : : : : : : : +- CometProject (19) + : : : : : : : : : : : : : : : : +- CometFilter (18) + : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (17) + : : : : : : : : : : : : : : : +- ReusedExchange (33) + : : : : : : : : : : : : : : +- BroadcastExchange (39) + : : : : : : : : : : : : : : +- * ColumnarToRow (38) + : : : : : : : : : : : : : : +- CometFilter (37) + : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : +- BroadcastExchange (45) + : : : : : : : : : : : : : +- * ColumnarToRow (44) + : : : : : : : : : : : : : +- CometFilter (43) + : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : +- BroadcastExchange (51) + : : : : : : : : : : : : +- * ColumnarToRow (50) + : : : : : : : : : : : : +- CometFilter (49) + : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (48) + : : : : : : : : : : : +- ReusedExchange (54) + : : : : : : : : : : +- BroadcastExchange (60) + : : : : : : : : : : +- * ColumnarToRow (59) + : : : : : : : : : : +- CometFilter (58) + : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (57) + : : : : : : : : : +- ReusedExchange (63) + : : : : : : : : +- BroadcastExchange (69) + : : : : : : : : +- * ColumnarToRow (68) + : : : : : : : : +- CometFilter (67) + : : : : : : : : +- CometScan parquet spark_catalog.default.promotion (66) + : : : : : : : +- BroadcastExchange (75) + : : : : : : : +- * ColumnarToRow (74) + : : : : : : : +- CometFilter (73) + : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (72) + : : : : : : +- ReusedExchange (78) + : : : : : +- BroadcastExchange (84) + : : : : : +- * ColumnarToRow (83) + : : : : : +- CometFilter (82) + : : : : : +- CometScan parquet spark_catalog.default.customer_address (81) + : : : : +- ReusedExchange (87) + : : : +- BroadcastExchange (93) + : : : +- * ColumnarToRow (92) + : : : +- CometFilter (91) + : : : +- CometScan parquet spark_catalog.default.income_band (90) + : : +- ReusedExchange (96) + : +- BroadcastExchange (103) + : +- * ColumnarToRow (102) + : +- CometProject (101) + : +- CometFilter (100) + : +- CometScan parquet spark_catalog.default.item (99) + +- * Sort (176) + +- Exchange (175) + +- * HashAggregate (174) + +- * HashAggregate (173) + +- * Project (172) + +- * BroadcastHashJoin Inner BuildRight (171) + :- * Project (169) + : +- * BroadcastHashJoin Inner BuildRight (168) + : :- * Project (166) + : : +- * BroadcastHashJoin Inner BuildRight (165) + : : :- * Project (163) + : : : +- * BroadcastHashJoin Inner BuildRight (162) + : : : :- * Project (160) + : : : : +- * BroadcastHashJoin Inner BuildRight (159) + : : : : :- * Project (157) + : : : : : +- * BroadcastHashJoin Inner BuildRight (156) + : : : : : :- * Project (154) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (153) + : : : : : : :- * Project (151) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (150) + : : : : : : : :- * Project (148) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (147) + : : : : : : : : :- * Project (145) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (144) + : : : : : : : : : :- * Project (142) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (141) + : : : : : : : : : : :- * Project (139) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (138) + : : : : : : : : : : : :- * Project (136) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (135) + : : : : : : : : : : : : :- * Project (133) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (132) + : : : : : : : : : : : : : :- * Project (130) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (129) + : : : : : : : : : : : : : : :- * Project (127) + : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (126) + : : : : : : : : : : : : : : : :- * ColumnarToRow (120) + : : : : : : : : : : : : : : : : +- CometSort (119) + : : : : : : : : : : : : : : : : +- CometExchange (118) + : : : : : : : : : : : : : : : : +- CometProject (117) + : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : : : :- CometBroadcastExchange (112) + : : : : : : : : : : : : : : : : : +- CometFilter (111) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (110) + : : : : : : : : : : : : : : : : +- CometProject (115) + : : : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (113) + : : : : : : : : : : : : : : : +- * Sort (125) + : : : : : : : : : : : : : : : +- * Project (124) + : : : : : : : : : : : : : : : +- * Filter (123) + : : : : : : : : : : : : : : : +- * HashAggregate (122) + : : : : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : : : : +- ReusedExchange (128) + : : : : : : : : : : : : : +- ReusedExchange (131) + : : : : : : : : : : : : +- ReusedExchange (134) + : : : : : : : : : : : +- ReusedExchange (137) + : : : : : : : : : : +- ReusedExchange (140) + : : : : : : : : : +- ReusedExchange (143) + : : : : : : : : +- ReusedExchange (146) + : : : : : : : +- ReusedExchange (149) + : : : : : : +- ReusedExchange (152) + : : : : : +- ReusedExchange (155) + : : : : +- ReusedExchange (158) + : : : +- ReusedExchange (161) + : : +- ReusedExchange (164) + : +- ReusedExchange (167) + +- ReusedExchange (170) (1) Scan parquet spark_catalog.default.store_sales @@ -230,9 +223,9 @@ Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number# Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(9) CometColumnarExchange +(9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -256,9 +249,9 @@ Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (16) CometSort Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] @@ -279,9 +272,9 @@ Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -(20) CometColumnarExchange +(20) CometExchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (21) CometSort Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] @@ -306,408 +299,396 @@ Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum((( Aggregate Attributes [3]: [sum#27, sum#28, isEmpty#29] Results [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] -(26) RowToColumnar +(26) Exchange Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] +Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(27) CometColumnarExchange -Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(28) ColumnarToRow [codegen id : 3] -Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] - -(29) HashAggregate [codegen id : 3] +(27) HashAggregate [codegen id : 3] Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] Keys [1]: [cs_item_sk#17] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#19)), sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#19))#33, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#34] Results [3]: [cs_item_sk#17, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#19))#33,17,2) AS sale#35, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#34 AS refund#36] -(30) Filter [codegen id : 3] +(28) Filter [codegen id : 3] Input [3]: [cs_item_sk#17, sale#35, refund#36] Condition : ((isnotnull(sale#35) AND isnotnull(refund#36)) AND (cast(sale#35 as decimal(21,2)) > (2 * refund#36))) -(31) Project [codegen id : 3] +(29) Project [codegen id : 3] Output [1]: [cs_item_sk#17] Input [3]: [cs_item_sk#17, sale#35, refund#36] -(32) Sort [codegen id : 3] +(30) Sort [codegen id : 3] Input [1]: [cs_item_sk#17] Arguments: [cs_item_sk#17 ASC NULLS FIRST], false, 0 -(33) SortMergeJoin [codegen id : 19] +(31) SortMergeJoin [codegen id : 19] Left keys [1]: [ss_item_sk#1] Right keys [1]: [cs_item_sk#17] Join type: Inner Join condition: None -(34) Project [codegen id : 19] +(32) Project [codegen id : 19] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] -(35) ReusedExchange [Reuses operator id: 193] +(33) ReusedExchange [Reuses operator id: 186] Output [2]: [d_date_sk#37, d_year#38] -(36) BroadcastHashJoin [codegen id : 19] +(34) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_sold_date_sk#12] Right keys [1]: [d_date_sk#37] Join type: Inner Join condition: None -(37) Project [codegen id : 19] +(35) Project [codegen id : 19] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38] Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#37, d_year#38] -(38) Scan parquet spark_catalog.default.store +(36) Scan parquet spark_catalog.default.store Output [3]: [s_store_sk#39, s_store_name#40, s_zip#41] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] ReadSchema: struct -(39) CometFilter +(37) CometFilter Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] Condition : ((isnotnull(s_store_sk#39) AND isnotnull(s_store_name#40)) AND isnotnull(s_zip#41)) -(40) ColumnarToRow [codegen id : 5] +(38) ColumnarToRow [codegen id : 5] Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] -(41) BroadcastExchange +(39) BroadcastExchange Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(42) BroadcastHashJoin [codegen id : 19] +(40) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_store_sk#6] Right keys [1]: [s_store_sk#39] Join type: Inner Join condition: None -(43) Project [codegen id : 19] +(41) Project [codegen id : 19] Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41] Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_sk#39, s_store_name#40, s_zip#41] -(44) Scan parquet spark_catalog.default.customer +(42) Scan parquet spark_catalog.default.customer Output [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(45) CometFilter +(43) CometFilter Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Condition : (((((isnotnull(c_customer_sk#42) AND isnotnull(c_first_sales_date_sk#47)) AND isnotnull(c_first_shipto_date_sk#46)) AND isnotnull(c_current_cdemo_sk#43)) AND isnotnull(c_current_hdemo_sk#44)) AND isnotnull(c_current_addr_sk#45)) -(46) ColumnarToRow [codegen id : 6] +(44) ColumnarToRow [codegen id : 6] Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] -(47) BroadcastExchange +(45) BroadcastExchange Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] -(48) BroadcastHashJoin [codegen id : 19] +(46) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_customer_sk#2] Right keys [1]: [c_customer_sk#42] Join type: Inner Join condition: None -(49) Project [codegen id : 19] +(47) Project [codegen id : 19] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] -(50) Scan parquet spark_catalog.default.date_dim +(48) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#48, d_year#49] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(51) CometFilter +(49) CometFilter Input [2]: [d_date_sk#48, d_year#49] Condition : isnotnull(d_date_sk#48) -(52) ColumnarToRow [codegen id : 7] +(50) ColumnarToRow [codegen id : 7] Input [2]: [d_date_sk#48, d_year#49] -(53) BroadcastExchange +(51) BroadcastExchange Input [2]: [d_date_sk#48, d_year#49] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] -(54) BroadcastHashJoin [codegen id : 19] +(52) BroadcastHashJoin [codegen id : 19] Left keys [1]: [c_first_sales_date_sk#47] Right keys [1]: [d_date_sk#48] Join type: Inner Join condition: None -(55) Project [codegen id : 19] +(53) Project [codegen id : 19] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, d_year#49] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47, d_date_sk#48, d_year#49] -(56) ReusedExchange [Reuses operator id: 53] +(54) ReusedExchange [Reuses operator id: 51] Output [2]: [d_date_sk#50, d_year#51] -(57) BroadcastHashJoin [codegen id : 19] +(55) BroadcastHashJoin [codegen id : 19] Left keys [1]: [c_first_shipto_date_sk#46] Right keys [1]: [d_date_sk#50] Join type: Inner Join condition: None -(58) Project [codegen id : 19] +(56) Project [codegen id : 19] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, d_year#49, d_date_sk#50, d_year#51] -(59) Scan parquet spark_catalog.default.customer_demographics +(57) Scan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#52, cd_marital_status#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] ReadSchema: struct -(60) CometFilter +(58) CometFilter Input [2]: [cd_demo_sk#52, cd_marital_status#53] Condition : (isnotnull(cd_demo_sk#52) AND isnotnull(cd_marital_status#53)) -(61) ColumnarToRow [codegen id : 9] +(59) ColumnarToRow [codegen id : 9] Input [2]: [cd_demo_sk#52, cd_marital_status#53] -(62) BroadcastExchange +(60) BroadcastExchange Input [2]: [cd_demo_sk#52, cd_marital_status#53] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -(63) BroadcastHashJoin [codegen id : 19] +(61) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_cdemo_sk#3] Right keys [1]: [cd_demo_sk#52] Join type: Inner Join condition: None -(64) Project [codegen id : 19] +(62) Project [codegen id : 19] Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_marital_status#53] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_demo_sk#52, cd_marital_status#53] -(65) ReusedExchange [Reuses operator id: 62] +(63) ReusedExchange [Reuses operator id: 60] Output [2]: [cd_demo_sk#54, cd_marital_status#55] -(66) BroadcastHashJoin [codegen id : 19] +(64) BroadcastHashJoin [codegen id : 19] Left keys [1]: [c_current_cdemo_sk#43] Right keys [1]: [cd_demo_sk#54] Join type: Inner Join condition: NOT (cd_marital_status#53 = cd_marital_status#55) -(67) Project [codegen id : 19] +(65) Project [codegen id : 19] Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_marital_status#53, cd_demo_sk#54, cd_marital_status#55] -(68) Scan parquet spark_catalog.default.promotion +(66) Scan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(69) CometFilter +(67) CometFilter Input [1]: [p_promo_sk#56] Condition : isnotnull(p_promo_sk#56) -(70) ColumnarToRow [codegen id : 11] +(68) ColumnarToRow [codegen id : 11] Input [1]: [p_promo_sk#56] -(71) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_promo_sk#56] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] -(72) BroadcastHashJoin [codegen id : 19] +(70) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_promo_sk#7] Right keys [1]: [p_promo_sk#56] Join type: Inner Join condition: None -(73) Project [codegen id : 19] +(71) Project [codegen id : 19] Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, p_promo_sk#56] -(74) Scan parquet spark_catalog.default.household_demographics +(72) Scan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#57, hd_income_band_sk#58] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(75) CometFilter +(73) CometFilter Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] Condition : (isnotnull(hd_demo_sk#57) AND isnotnull(hd_income_band_sk#58)) -(76) ColumnarToRow [codegen id : 12] +(74) ColumnarToRow [codegen id : 12] Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] -(77) BroadcastExchange +(75) BroadcastExchange Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] -(78) BroadcastHashJoin [codegen id : 19] +(76) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_hdemo_sk#4] Right keys [1]: [hd_demo_sk#57] Join type: Inner Join condition: None -(79) Project [codegen id : 19] +(77) Project [codegen id : 19] Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58] Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_demo_sk#57, hd_income_band_sk#58] -(80) ReusedExchange [Reuses operator id: 77] +(78) ReusedExchange [Reuses operator id: 75] Output [2]: [hd_demo_sk#59, hd_income_band_sk#60] -(81) BroadcastHashJoin [codegen id : 19] +(79) BroadcastHashJoin [codegen id : 19] Left keys [1]: [c_current_hdemo_sk#44] Right keys [1]: [hd_demo_sk#59] Join type: Inner Join condition: None -(82) Project [codegen id : 19] +(80) Project [codegen id : 19] Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60] Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_demo_sk#59, hd_income_band_sk#60] -(83) Scan parquet spark_catalog.default.customer_address +(81) Scan parquet spark_catalog.default.customer_address Output [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(84) CometFilter +(82) CometFilter Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Condition : isnotnull(ca_address_sk#61) -(85) ColumnarToRow [codegen id : 14] +(83) ColumnarToRow [codegen id : 14] Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] -(86) BroadcastExchange +(84) BroadcastExchange Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] -(87) BroadcastHashJoin [codegen id : 19] +(85) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_addr_sk#5] Right keys [1]: [ca_address_sk#61] Join type: Inner Join condition: None -(88) Project [codegen id : 19] +(86) Project [codegen id : 19] Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] -(89) ReusedExchange [Reuses operator id: 86] +(87) ReusedExchange [Reuses operator id: 84] Output [5]: [ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(90) BroadcastHashJoin [codegen id : 19] +(88) BroadcastHashJoin [codegen id : 19] Left keys [1]: [c_current_addr_sk#45] Right keys [1]: [ca_address_sk#66] Join type: Inner Join condition: None -(91) Project [codegen id : 19] +(89) Project [codegen id : 19] Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(92) Scan parquet spark_catalog.default.income_band +(90) Scan parquet spark_catalog.default.income_band Output [1]: [ib_income_band_sk#71] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(93) CometFilter +(91) CometFilter Input [1]: [ib_income_band_sk#71] Condition : isnotnull(ib_income_band_sk#71) -(94) ColumnarToRow [codegen id : 16] +(92) ColumnarToRow [codegen id : 16] Input [1]: [ib_income_band_sk#71] -(95) BroadcastExchange +(93) BroadcastExchange Input [1]: [ib_income_band_sk#71] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] -(96) BroadcastHashJoin [codegen id : 19] +(94) BroadcastHashJoin [codegen id : 19] Left keys [1]: [hd_income_band_sk#58] Right keys [1]: [ib_income_band_sk#71] Join type: Inner Join condition: None -(97) Project [codegen id : 19] +(95) Project [codegen id : 19] Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#71] -(98) ReusedExchange [Reuses operator id: 95] +(96) ReusedExchange [Reuses operator id: 93] Output [1]: [ib_income_band_sk#72] -(99) BroadcastHashJoin [codegen id : 19] +(97) BroadcastHashJoin [codegen id : 19] Left keys [1]: [hd_income_band_sk#60] Right keys [1]: [ib_income_band_sk#72] Join type: Inner Join condition: None -(100) Project [codegen id : 19] +(98) Project [codegen id : 19] Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#72] -(101) Scan parquet spark_catalog.default.item +(99) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(102) CometFilter +(100) CometFilter Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Condition : ((((((isnotnull(i_current_price#74) AND i_color#75 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(103) CometProject +(101) CometProject Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Arguments: [i_item_sk#73, i_product_name#76], [i_item_sk#73, i_product_name#76] -(104) ColumnarToRow [codegen id : 18] +(102) ColumnarToRow [codegen id : 18] Input [2]: [i_item_sk#73, i_product_name#76] -(105) BroadcastExchange +(103) BroadcastExchange Input [2]: [i_item_sk#73, i_product_name#76] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] -(106) BroadcastHashJoin [codegen id : 19] +(104) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_item_sk#1] Right keys [1]: [i_item_sk#73] Join type: Inner Join condition: None -(107) Project [codegen id : 19] +(105) Project [codegen id : 19] Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, d_year#49, d_year#51, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] -(108) HashAggregate [codegen id : 19] +(106) HashAggregate [codegen id : 19] Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, d_year#49, d_year#51, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] Keys [15]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] Aggregate Attributes [4]: [count#77, sum#78, sum#79, sum#80] Results [19]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51, count#81, sum#82, sum#83, sum#84] -(109) HashAggregate [codegen id : 19] +(107) HashAggregate [codegen id : 19] Input [19]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51, count#81, sum#82, sum#83, sum#84] Keys [15]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] Aggregate Attributes [4]: [count(1)#85, sum(UnscaledValue(ss_wholesale_cost#9))#86, sum(UnscaledValue(ss_list_price#10))#87, sum(UnscaledValue(ss_coupon_amt#11))#88] Results [17]: [i_product_name#76 AS product_name#89, i_item_sk#73 AS item_sk#90, s_store_name#40 AS store_name#91, s_zip#41 AS store_zip#92, ca_street_number#62 AS b_street_number#93, ca_street_name#63 AS b_streen_name#94, ca_city#64 AS b_city#95, ca_zip#65 AS b_zip#96, ca_street_number#67 AS c_street_number#97, ca_street_name#68 AS c_street_name#98, ca_city#69 AS c_city#99, ca_zip#70 AS c_zip#100, d_year#38 AS syear#101, count(1)#85 AS cnt#102, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#86,17,2) AS s1#103, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#87,17,2) AS s2#104, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#88,17,2) AS s3#105] -(110) RowToColumnar -Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] - -(111) CometColumnarExchange -Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] -Arguments: hashpartitioning(item_sk#90, store_name#91, store_zip#92, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] - -(112) CometSort +(108) Exchange Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] -Arguments: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105], [item_sk#90 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, store_zip#92 ASC NULLS FIRST] +Arguments: hashpartitioning(item_sk#90, store_name#91, store_zip#92, 5), ENSURE_REQUIREMENTS, [plan_id=14] -(113) ColumnarToRow [codegen id : 20] +(109) Sort [codegen id : 20] Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] +Arguments: [item_sk#90 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, store_zip#92 ASC NULLS FIRST], false, 0 -(114) Scan parquet spark_catalog.default.store_sales +(110) Scan parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Batched: true Location: InMemoryFileIndex [] @@ -715,380 +696,371 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#117), dynamicpruningexpression(ss_s PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(115) CometFilter +(111) CometFilter Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Condition : (((((((isnotnull(ss_item_sk#106) AND isnotnull(ss_ticket_number#113)) AND isnotnull(ss_store_sk#111)) AND isnotnull(ss_customer_sk#107)) AND isnotnull(ss_cdemo_sk#108)) AND isnotnull(ss_promo_sk#112)) AND isnotnull(ss_hdemo_sk#109)) AND isnotnull(ss_addr_sk#110)) -(116) CometBroadcastExchange +(112) CometBroadcastExchange Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Arguments: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(117) Scan parquet spark_catalog.default.store_returns +(113) Scan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(118) CometFilter +(114) CometFilter Input [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] Condition : (isnotnull(sr_item_sk#119) AND isnotnull(sr_ticket_number#120)) -(119) CometProject +(115) CometProject Input [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] Arguments: [sr_item_sk#119, sr_ticket_number#120], [sr_item_sk#119, sr_ticket_number#120] -(120) CometBroadcastHashJoin +(116) CometBroadcastHashJoin Left output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Right output [2]: [sr_item_sk#119, sr_ticket_number#120] Arguments: [ss_item_sk#106, ss_ticket_number#113], [sr_item_sk#119, sr_ticket_number#120], Inner, BuildLeft -(121) CometProject +(117) CometProject Input [14]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, sr_item_sk#119, sr_ticket_number#120] Arguments: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(122) CometColumnarExchange +(118) CometExchange Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -Arguments: hashpartitioning(ss_item_sk#106, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] +Arguments: hashpartitioning(ss_item_sk#106, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(123) CometSort +(119) CometSort Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Arguments: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#106 ASC NULLS FIRST] -(124) ColumnarToRow [codegen id : 21] +(120) ColumnarToRow [codegen id : 21] Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(125) ReusedExchange [Reuses operator id: 27] +(121) ReusedExchange [Reuses operator id: 26] Output [4]: [cs_item_sk#122, sum#123, sum#124, isEmpty#125] -(126) ColumnarToRow [codegen id : 23] -Input [4]: [cs_item_sk#122, sum#123, sum#124, isEmpty#125] - -(127) HashAggregate [codegen id : 23] +(122) HashAggregate [codegen id : 23] Input [4]: [cs_item_sk#122, sum#123, sum#124, isEmpty#125] Keys [1]: [cs_item_sk#122] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#126)), sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))] Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#126))#33, sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))#34] Results [3]: [cs_item_sk#122, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#126))#33,17,2) AS sale#130, sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))#34 AS refund#131] -(128) Filter [codegen id : 23] +(123) Filter [codegen id : 23] Input [3]: [cs_item_sk#122, sale#130, refund#131] Condition : ((isnotnull(sale#130) AND isnotnull(refund#131)) AND (cast(sale#130 as decimal(21,2)) > (2 * refund#131))) -(129) Project [codegen id : 23] +(124) Project [codegen id : 23] Output [1]: [cs_item_sk#122] Input [3]: [cs_item_sk#122, sale#130, refund#131] -(130) Sort [codegen id : 23] +(125) Sort [codegen id : 23] Input [1]: [cs_item_sk#122] Arguments: [cs_item_sk#122 ASC NULLS FIRST], false, 0 -(131) SortMergeJoin [codegen id : 39] +(126) SortMergeJoin [codegen id : 39] Left keys [1]: [ss_item_sk#106] Right keys [1]: [cs_item_sk#122] Join type: Inner Join condition: None -(132) Project [codegen id : 39] +(127) Project [codegen id : 39] Output [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, cs_item_sk#122] -(133) ReusedExchange [Reuses operator id: 197] +(128) ReusedExchange [Reuses operator id: 190] Output [2]: [d_date_sk#132, d_year#133] -(134) BroadcastHashJoin [codegen id : 39] +(129) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_sold_date_sk#117] Right keys [1]: [d_date_sk#132] Join type: Inner Join condition: None -(135) Project [codegen id : 39] +(130) Project [codegen id : 39] Output [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133] Input [13]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, d_date_sk#132, d_year#133] -(136) ReusedExchange [Reuses operator id: 41] +(131) ReusedExchange [Reuses operator id: 39] Output [3]: [s_store_sk#134, s_store_name#135, s_zip#136] -(137) BroadcastHashJoin [codegen id : 39] +(132) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_store_sk#111] Right keys [1]: [s_store_sk#134] Join type: Inner Join condition: None -(138) Project [codegen id : 39] +(133) Project [codegen id : 39] Output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136] Input [14]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_sk#134, s_store_name#135, s_zip#136] -(139) ReusedExchange [Reuses operator id: 47] +(134) ReusedExchange [Reuses operator id: 45] Output [6]: [c_customer_sk#137, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] -(140) BroadcastHashJoin [codegen id : 39] +(135) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_customer_sk#107] Right keys [1]: [c_customer_sk#137] Join type: Inner Join condition: None -(141) Project [codegen id : 39] +(136) Project [codegen id : 39] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] Input [18]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_customer_sk#137, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] -(142) ReusedExchange [Reuses operator id: 53] +(137) ReusedExchange [Reuses operator id: 51] Output [2]: [d_date_sk#143, d_year#144] -(143) BroadcastHashJoin [codegen id : 39] +(138) BroadcastHashJoin [codegen id : 39] Left keys [1]: [c_first_sales_date_sk#142] Right keys [1]: [d_date_sk#143] Join type: Inner Join condition: None -(144) Project [codegen id : 39] +(139) Project [codegen id : 39] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, d_year#144] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142, d_date_sk#143, d_year#144] -(145) ReusedExchange [Reuses operator id: 53] +(140) ReusedExchange [Reuses operator id: 51] Output [2]: [d_date_sk#145, d_year#146] -(146) BroadcastHashJoin [codegen id : 39] +(141) BroadcastHashJoin [codegen id : 39] Left keys [1]: [c_first_shipto_date_sk#141] Right keys [1]: [d_date_sk#145] Join type: Inner Join condition: None -(147) Project [codegen id : 39] +(142) Project [codegen id : 39] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, d_year#144, d_date_sk#145, d_year#146] -(148) ReusedExchange [Reuses operator id: 62] +(143) ReusedExchange [Reuses operator id: 60] Output [2]: [cd_demo_sk#147, cd_marital_status#148] -(149) BroadcastHashJoin [codegen id : 39] +(144) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_cdemo_sk#108] Right keys [1]: [cd_demo_sk#147] Join type: Inner Join condition: None -(150) Project [codegen id : 39] +(145) Project [codegen id : 39] Output [16]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_marital_status#148] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_demo_sk#147, cd_marital_status#148] -(151) ReusedExchange [Reuses operator id: 62] +(146) ReusedExchange [Reuses operator id: 60] Output [2]: [cd_demo_sk#149, cd_marital_status#150] -(152) BroadcastHashJoin [codegen id : 39] +(147) BroadcastHashJoin [codegen id : 39] Left keys [1]: [c_current_cdemo_sk#138] Right keys [1]: [cd_demo_sk#149] Join type: Inner Join condition: NOT (cd_marital_status#148 = cd_marital_status#150) -(153) Project [codegen id : 39] +(148) Project [codegen id : 39] Output [14]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [18]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_marital_status#148, cd_demo_sk#149, cd_marital_status#150] -(154) ReusedExchange [Reuses operator id: 71] +(149) ReusedExchange [Reuses operator id: 69] Output [1]: [p_promo_sk#151] -(155) BroadcastHashJoin [codegen id : 39] +(150) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_promo_sk#112] Right keys [1]: [p_promo_sk#151] Join type: Inner Join condition: None -(156) Project [codegen id : 39] +(151) Project [codegen id : 39] Output [13]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [15]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, p_promo_sk#151] -(157) ReusedExchange [Reuses operator id: 77] +(152) ReusedExchange [Reuses operator id: 75] Output [2]: [hd_demo_sk#152, hd_income_band_sk#153] -(158) BroadcastHashJoin [codegen id : 39] +(153) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_hdemo_sk#109] Right keys [1]: [hd_demo_sk#152] Join type: Inner Join condition: None -(159) Project [codegen id : 39] +(154) Project [codegen id : 39] Output [13]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153] Input [15]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_demo_sk#152, hd_income_band_sk#153] -(160) ReusedExchange [Reuses operator id: 77] +(155) ReusedExchange [Reuses operator id: 75] Output [2]: [hd_demo_sk#154, hd_income_band_sk#155] -(161) BroadcastHashJoin [codegen id : 39] +(156) BroadcastHashJoin [codegen id : 39] Left keys [1]: [c_current_hdemo_sk#139] Right keys [1]: [hd_demo_sk#154] Join type: Inner Join condition: None -(162) Project [codegen id : 39] +(157) Project [codegen id : 39] Output [13]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155] Input [15]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_demo_sk#154, hd_income_band_sk#155] -(163) ReusedExchange [Reuses operator id: 86] +(158) ReusedExchange [Reuses operator id: 84] Output [5]: [ca_address_sk#156, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] -(164) BroadcastHashJoin [codegen id : 39] +(159) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_addr_sk#110] Right keys [1]: [ca_address_sk#156] Join type: Inner Join condition: None -(165) Project [codegen id : 39] +(160) Project [codegen id : 39] Output [16]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] Input [18]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_address_sk#156, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] -(166) ReusedExchange [Reuses operator id: 86] +(161) ReusedExchange [Reuses operator id: 84] Output [5]: [ca_address_sk#161, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] -(167) BroadcastHashJoin [codegen id : 39] +(162) BroadcastHashJoin [codegen id : 39] Left keys [1]: [c_current_addr_sk#140] Right keys [1]: [ca_address_sk#161] Join type: Inner Join condition: None -(168) Project [codegen id : 39] +(163) Project [codegen id : 39] Output [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [21]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_address_sk#161, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] -(169) ReusedExchange [Reuses operator id: 95] +(164) ReusedExchange [Reuses operator id: 93] Output [1]: [ib_income_band_sk#166] -(170) BroadcastHashJoin [codegen id : 39] +(165) BroadcastHashJoin [codegen id : 39] Left keys [1]: [hd_income_band_sk#153] Right keys [1]: [ib_income_band_sk#166] Join type: Inner Join condition: None -(171) Project [codegen id : 39] +(166) Project [codegen id : 39] Output [18]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [20]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, ib_income_band_sk#166] -(172) ReusedExchange [Reuses operator id: 95] +(167) ReusedExchange [Reuses operator id: 93] Output [1]: [ib_income_band_sk#167] -(173) BroadcastHashJoin [codegen id : 39] +(168) BroadcastHashJoin [codegen id : 39] Left keys [1]: [hd_income_band_sk#155] Right keys [1]: [ib_income_band_sk#167] Join type: Inner Join condition: None -(174) Project [codegen id : 39] +(169) Project [codegen id : 39] Output [17]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, ib_income_band_sk#167] -(175) ReusedExchange [Reuses operator id: 105] +(170) ReusedExchange [Reuses operator id: 103] Output [2]: [i_item_sk#168, i_product_name#169] -(176) BroadcastHashJoin [codegen id : 39] +(171) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_item_sk#106] Right keys [1]: [i_item_sk#168] Join type: Inner Join condition: None -(177) Project [codegen id : 39] +(172) Project [codegen id : 39] Output [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, d_year#144, d_year#146, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] Input [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] -(178) HashAggregate [codegen id : 39] +(173) HashAggregate [codegen id : 39] Input [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, d_year#144, d_year#146, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] Keys [15]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#114)), partial_sum(UnscaledValue(ss_list_price#115)), partial_sum(UnscaledValue(ss_coupon_amt#116))] Aggregate Attributes [4]: [count#77, sum#170, sum#171, sum#172] Results [19]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146, count#81, sum#173, sum#174, sum#175] -(179) HashAggregate [codegen id : 39] +(174) HashAggregate [codegen id : 39] Input [19]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146, count#81, sum#173, sum#174, sum#175] Keys [15]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#114)), sum(UnscaledValue(ss_list_price#115)), sum(UnscaledValue(ss_coupon_amt#116))] Aggregate Attributes [4]: [count(1)#85, sum(UnscaledValue(ss_wholesale_cost#114))#86, sum(UnscaledValue(ss_list_price#115))#87, sum(UnscaledValue(ss_coupon_amt#116))#88] Results [8]: [i_item_sk#168 AS item_sk#176, s_store_name#135 AS store_name#177, s_zip#136 AS store_zip#178, d_year#133 AS syear#179, count(1)#85 AS cnt#180, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#114))#86,17,2) AS s1#181, MakeDecimal(sum(UnscaledValue(ss_list_price#115))#87,17,2) AS s2#182, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#116))#88,17,2) AS s3#183] -(180) RowToColumnar -Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] - -(181) CometColumnarExchange -Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] -Arguments: hashpartitioning(item_sk#176, store_name#177, store_zip#178, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] - -(182) CometSort +(175) Exchange Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] -Arguments: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183], [item_sk#176 ASC NULLS FIRST, store_name#177 ASC NULLS FIRST, store_zip#178 ASC NULLS FIRST] +Arguments: hashpartitioning(item_sk#176, store_name#177, store_zip#178, 5), ENSURE_REQUIREMENTS, [plan_id=16] -(183) ColumnarToRow [codegen id : 40] +(176) Sort [codegen id : 40] Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] +Arguments: [item_sk#176 ASC NULLS FIRST, store_name#177 ASC NULLS FIRST, store_zip#178 ASC NULLS FIRST], false, 0 -(184) SortMergeJoin [codegen id : 41] +(177) SortMergeJoin [codegen id : 41] Left keys [3]: [item_sk#90, store_name#91, store_zip#92] Right keys [3]: [item_sk#176, store_name#177, store_zip#178] Join type: Inner Join condition: (cnt#180 <= cnt#102) -(185) Project [codegen id : 41] +(178) Project [codegen id : 41] Output [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] Input [25]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] -(186) RowToColumnar +(179) RowToColumnar Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] -(187) CometColumnarExchange +(180) CometColumnarExchange Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] Arguments: rangepartitioning(product_name#89 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, cnt#180 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=17] -(188) CometSort +(181) CometSort Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] Arguments: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180], [product_name#89 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, cnt#180 ASC NULLS FIRST] -(189) ColumnarToRow [codegen id : 42] +(182) ColumnarToRow [codegen id : 42] Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (193) -+- * ColumnarToRow (192) - +- CometFilter (191) - +- CometScan parquet spark_catalog.default.date_dim (190) +BroadcastExchange (186) ++- * ColumnarToRow (185) + +- CometFilter (184) + +- CometScan parquet spark_catalog.default.date_dim (183) -(190) Scan parquet spark_catalog.default.date_dim +(183) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#37, d_year#38] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(191) CometFilter +(184) CometFilter Input [2]: [d_date_sk#37, d_year#38] Condition : ((isnotnull(d_year#38) AND (d_year#38 = 1999)) AND isnotnull(d_date_sk#37)) -(192) ColumnarToRow [codegen id : 1] +(185) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#37, d_year#38] -(193) BroadcastExchange +(186) BroadcastExchange Input [2]: [d_date_sk#37, d_year#38] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] -Subquery:2 Hosting operator id = 114 Hosting Expression = ss_sold_date_sk#117 IN dynamicpruning#118 -BroadcastExchange (197) -+- * ColumnarToRow (196) - +- CometFilter (195) - +- CometScan parquet spark_catalog.default.date_dim (194) +Subquery:2 Hosting operator id = 110 Hosting Expression = ss_sold_date_sk#117 IN dynamicpruning#118 +BroadcastExchange (190) ++- * ColumnarToRow (189) + +- CometFilter (188) + +- CometScan parquet spark_catalog.default.date_dim (187) -(194) Scan parquet spark_catalog.default.date_dim +(187) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#132, d_year#133] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(195) CometFilter +(188) CometFilter Input [2]: [d_date_sk#132, d_year#133] Condition : ((isnotnull(d_year#133) AND (d_year#133 = 2000)) AND isnotnull(d_date_sk#132)) -(196) ColumnarToRow [codegen id : 1] +(189) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#132, d_year#133] -(197) BroadcastExchange +(190) BroadcastExchange Input [2]: [d_date_sk#132, d_year#133] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt index 7209b28f9..c472a785d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q64/simplified.txt @@ -9,268 +9,261 @@ WholeStageCodegen (42) SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] InputAdapter WholeStageCodegen (20) - ColumnarToRow + Sort [item_sk,store_name,store_zip] InputAdapter - CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #2 - RowToColumnar - WholeStageCodegen (19) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + Exchange [item_sk,store_name,store_zip] #2 + WholeStageCodegen (19) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (3) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [cs_item_sk] #6 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) ColumnarToRow InputAdapter - CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometColumnarExchange [cs_item_sk,cs_order_number] #7 - CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] - CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometColumnarExchange [cr_item_sk,cr_order_number] #8 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter - ReusedExchange [d_date_sk,d_year] #5 + WholeStageCodegen (3) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + InputAdapter + Exchange [cs_item_sk] #6 + WholeStageCodegen (2) + HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] + CometExchange [cs_item_sk,cs_order_number] #7 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometExchange [cr_item_sk,cr_order_number] #8 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] InputAdapter - BroadcastExchange #9 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_store_name,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + ReusedExchange [d_date_sk,d_year] #5 InputAdapter - BroadcastExchange #10 - WholeStageCodegen (6) + BroadcastExchange #9 + WholeStageCodegen (5) ColumnarToRow InputAdapter - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] InputAdapter - BroadcastExchange #11 - WholeStageCodegen (7) + BroadcastExchange #10 + WholeStageCodegen (6) ColumnarToRow InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] InputAdapter - ReusedExchange [d_date_sk,d_year] #11 + BroadcastExchange #11 + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #12 - WholeStageCodegen (9) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + ReusedExchange [d_date_sk,d_year] #11 InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 + BroadcastExchange #12 + WholeStageCodegen (9) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_marital_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] InputAdapter - BroadcastExchange #13 - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] + ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter - BroadcastExchange #14 - WholeStageCodegen (12) + BroadcastExchange #13 + WholeStageCodegen (11) ColumnarToRow InputAdapter - CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometFilter [p_promo_sk] + CometScan parquet spark_catalog.default.promotion [p_promo_sk] InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + BroadcastExchange #14 + WholeStageCodegen (12) + ColumnarToRow + InputAdapter + CometFilter [hd_demo_sk,hd_income_band_sk] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] InputAdapter - BroadcastExchange #15 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + BroadcastExchange #15 + WholeStageCodegen (14) + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] InputAdapter - BroadcastExchange #16 - WholeStageCodegen (16) - ColumnarToRow - InputAdapter - CometFilter [ib_income_band_sk] - CometScan parquet spark_catalog.default.income_band [ib_income_band_sk] + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter - ReusedExchange [ib_income_band_sk] #16 + BroadcastExchange #16 + WholeStageCodegen (16) + ColumnarToRow + InputAdapter + CometFilter [ib_income_band_sk] + CometScan parquet spark_catalog.default.income_band [ib_income_band_sk] InputAdapter - BroadcastExchange #17 - WholeStageCodegen (18) - ColumnarToRow - InputAdapter - CometProject [i_item_sk,i_product_name] - CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + BroadcastExchange #17 + WholeStageCodegen (18) + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_current_price,i_color,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] InputAdapter WholeStageCodegen (40) - ColumnarToRow + Sort [item_sk,store_name,store_zip] InputAdapter - CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #18 - RowToColumnar - WholeStageCodegen (39) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + Exchange [item_sk,store_name,store_zip] #18 + WholeStageCodegen (39) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (21) - ColumnarToRow - InputAdapter - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #19 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #20 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #21 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (23) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] + InputAdapter + WholeStageCodegen (21) + ColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #19 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #20 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #21 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter - ReusedExchange [d_date_sk,d_year] #21 + WholeStageCodegen (23) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + InputAdapter + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + ReusedExchange [d_date_sk,d_year] #21 InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 InputAdapter - ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 InputAdapter ReusedExchange [d_date_sk,d_year] #11 InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [d_date_sk,d_year] #11 InputAdapter ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter - ReusedExchange [p_promo_sk] #13 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [p_promo_sk] #13 InputAdapter ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter - ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter ReusedExchange [ib_income_band_sk] #16 InputAdapter - ReusedExchange [i_item_sk,i_product_name] #17 + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt index 9b2797d93..ea7975366 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/explain.txt @@ -1,52 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (48) -+- * Project (47) - +- * BroadcastHashJoin Inner BuildRight (46) - :- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * Project (21) - : : +- * BroadcastHashJoin Inner BuildRight (20) +TakeOrderedAndProject (42) ++- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (25) + : +- * BroadcastHashJoin Inner BuildRight (24) + : :- * Project (19) + : : +- * BroadcastHashJoin Inner BuildRight (18) : : :- * ColumnarToRow (3) : : : +- CometFilter (2) : : : +- CometScan parquet spark_catalog.default.store (1) - : : +- BroadcastExchange (19) - : : +- * Filter (18) - : : +- * HashAggregate (17) - : : +- * ColumnarToRow (16) - : : +- CometColumnarExchange (15) - : : +- RowToColumnar (14) - : : +- * HashAggregate (13) - : : +- * ColumnarToRow (12) - : : +- CometProject (11) - : : +- CometBroadcastHashJoin (10) - : : :- CometFilter (5) - : : : +- CometScan parquet spark_catalog.default.store_sales (4) - : : +- CometBroadcastExchange (9) - : : +- CometProject (8) - : : +- CometFilter (7) - : : +- CometScan parquet spark_catalog.default.date_dim (6) - : +- BroadcastExchange (25) - : +- * ColumnarToRow (24) - : +- CometFilter (23) - : +- CometScan parquet spark_catalog.default.item (22) - +- BroadcastExchange (45) - +- * Filter (44) - +- * HashAggregate (43) - +- * ColumnarToRow (42) - +- CometColumnarExchange (41) - +- RowToColumnar (40) - +- * HashAggregate (39) - +- * HashAggregate (38) - +- * ColumnarToRow (37) - +- CometColumnarExchange (36) - +- RowToColumnar (35) - +- * HashAggregate (34) - +- * ColumnarToRow (33) - +- CometProject (32) - +- CometBroadcastHashJoin (31) - :- CometFilter (29) - : +- CometScan parquet spark_catalog.default.store_sales (28) - +- ReusedExchange (30) + : : +- BroadcastExchange (17) + : : +- * Filter (16) + : : +- * HashAggregate (15) + : : +- Exchange (14) + : : +- * HashAggregate (13) + : : +- * ColumnarToRow (12) + : : +- CometProject (11) + : : +- CometBroadcastHashJoin (10) + : : :- CometFilter (5) + : : : +- CometScan parquet spark_catalog.default.store_sales (4) + : : +- CometBroadcastExchange (9) + : : +- CometProject (8) + : : +- CometFilter (7) + : : +- CometScan parquet spark_catalog.default.date_dim (6) + : +- BroadcastExchange (23) + : +- * ColumnarToRow (22) + : +- CometFilter (21) + : +- CometScan parquet spark_catalog.default.item (20) + +- BroadcastExchange (39) + +- * Filter (38) + +- * HashAggregate (37) + +- Exchange (36) + +- * HashAggregate (35) + +- * HashAggregate (34) + +- Exchange (33) + +- * HashAggregate (32) + +- * ColumnarToRow (31) + +- CometProject (30) + +- CometBroadcastHashJoin (29) + :- CometFilter (27) + : +- CometScan parquet spark_catalog.default.store_sales (26) + +- ReusedExchange (28) (1) Scan parquet spark_catalog.default.store @@ -113,70 +107,64 @@ Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] Aggregate Attributes [1]: [sum#10] Results [3]: [ss_store_sk#4, ss_item_sk#3, sum#11] -(14) RowToColumnar +(14) Exchange Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#11] +Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(15) CometColumnarExchange -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#11] -Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(16) ColumnarToRow [codegen id : 2] -Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#11] - -(17) HashAggregate [codegen id : 2] +(15) HashAggregate [codegen id : 2] Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#11] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [sum(UnscaledValue(ss_sales_price#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#5))#12] Results [3]: [ss_store_sk#4, ss_item_sk#3, MakeDecimal(sum(UnscaledValue(ss_sales_price#5))#12,17,2) AS revenue#13] -(18) Filter [codegen id : 2] +(16) Filter [codegen id : 2] Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#13] Condition : isnotnull(revenue#13) -(19) BroadcastExchange +(17) BroadcastExchange Input [3]: [ss_store_sk#4, ss_item_sk#3, revenue#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] -(20) BroadcastHashJoin [codegen id : 7] +(18) BroadcastHashJoin [codegen id : 7] Left keys [1]: [s_store_sk#1] Right keys [1]: [ss_store_sk#4] Join type: Inner Join condition: None -(21) Project [codegen id : 7] +(19) Project [codegen id : 7] Output [4]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#13] Input [5]: [s_store_sk#1, s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#13] -(22) Scan parquet spark_catalog.default.item +(20) Scan parquet spark_catalog.default.item Output [5]: [i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(23) CometFilter +(21) CometFilter Input [5]: [i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Condition : isnotnull(i_item_sk#14) -(24) ColumnarToRow [codegen id : 3] +(22) ColumnarToRow [codegen id : 3] Input [5]: [i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] -(25) BroadcastExchange +(23) BroadcastExchange Input [5]: [i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(26) BroadcastHashJoin [codegen id : 7] +(24) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ss_item_sk#3] Right keys [1]: [i_item_sk#14] Join type: Inner Join condition: None -(27) Project [codegen id : 7] +(25) Project [codegen id : 7] Output [7]: [s_store_name#2, ss_store_sk#4, revenue#13, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] Input [9]: [s_store_name#2, ss_store_sk#4, ss_item_sk#3, revenue#13, i_item_sk#14, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18] -(28) Scan parquet spark_catalog.default.store_sales +(26) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21, ss_sold_date_sk#22] Batched: true Location: InMemoryFileIndex [] @@ -184,127 +172,115 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#22), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_store_sk)] ReadSchema: struct -(29) CometFilter +(27) CometFilter Input [4]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21, ss_sold_date_sk#22] Condition : isnotnull(ss_store_sk#20) -(30) ReusedExchange [Reuses operator id: 9] +(28) ReusedExchange [Reuses operator id: 9] Output [1]: [d_date_sk#24] -(31) CometBroadcastHashJoin +(29) CometBroadcastHashJoin Left output [4]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21, ss_sold_date_sk#22] Right output [1]: [d_date_sk#24] Arguments: [ss_sold_date_sk#22], [d_date_sk#24], Inner, BuildRight -(32) CometProject +(30) CometProject Input [5]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21, ss_sold_date_sk#22, d_date_sk#24] Arguments: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21], [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21] -(33) ColumnarToRow [codegen id : 4] +(31) ColumnarToRow [codegen id : 4] Input [3]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21] -(34) HashAggregate [codegen id : 4] +(32) HashAggregate [codegen id : 4] Input [3]: [ss_item_sk#19, ss_store_sk#20, ss_sales_price#21] Keys [2]: [ss_store_sk#20, ss_item_sk#19] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#21))] Aggregate Attributes [1]: [sum#25] Results [3]: [ss_store_sk#20, ss_item_sk#19, sum#26] -(35) RowToColumnar -Input [3]: [ss_store_sk#20, ss_item_sk#19, sum#26] - -(36) CometColumnarExchange +(33) Exchange Input [3]: [ss_store_sk#20, ss_item_sk#19, sum#26] -Arguments: hashpartitioning(ss_store_sk#20, ss_item_sk#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ss_store_sk#20, ss_item_sk#19, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(37) ColumnarToRow [codegen id : 5] -Input [3]: [ss_store_sk#20, ss_item_sk#19, sum#26] - -(38) HashAggregate [codegen id : 5] +(34) HashAggregate [codegen id : 5] Input [3]: [ss_store_sk#20, ss_item_sk#19, sum#26] Keys [2]: [ss_store_sk#20, ss_item_sk#19] Functions [1]: [sum(UnscaledValue(ss_sales_price#21))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#21))#27] Results [2]: [ss_store_sk#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#21))#27,17,2) AS revenue#28] -(39) HashAggregate [codegen id : 5] +(35) HashAggregate [codegen id : 5] Input [2]: [ss_store_sk#20, revenue#28] Keys [1]: [ss_store_sk#20] Functions [1]: [partial_avg(revenue#28)] Aggregate Attributes [2]: [sum#29, count#30] Results [3]: [ss_store_sk#20, sum#31, count#32] -(40) RowToColumnar -Input [3]: [ss_store_sk#20, sum#31, count#32] - -(41) CometColumnarExchange -Input [3]: [ss_store_sk#20, sum#31, count#32] -Arguments: hashpartitioning(ss_store_sk#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(42) ColumnarToRow [codegen id : 6] +(36) Exchange Input [3]: [ss_store_sk#20, sum#31, count#32] +Arguments: hashpartitioning(ss_store_sk#20, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) HashAggregate [codegen id : 6] +(37) HashAggregate [codegen id : 6] Input [3]: [ss_store_sk#20, sum#31, count#32] Keys [1]: [ss_store_sk#20] Functions [1]: [avg(revenue#28)] Aggregate Attributes [1]: [avg(revenue#28)#33] Results [2]: [ss_store_sk#20, avg(revenue#28)#33 AS ave#34] -(44) Filter [codegen id : 6] +(38) Filter [codegen id : 6] Input [2]: [ss_store_sk#20, ave#34] Condition : isnotnull(ave#34) -(45) BroadcastExchange +(39) BroadcastExchange Input [2]: [ss_store_sk#20, ave#34] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -(46) BroadcastHashJoin [codegen id : 7] +(40) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ss_store_sk#4] Right keys [1]: [ss_store_sk#20] Join type: Inner Join condition: (cast(revenue#13 as decimal(23,7)) <= (0.1 * ave#34)) -(47) Project [codegen id : 7] +(41) Project [codegen id : 7] Output [6]: [s_store_name#2, i_item_desc#15, revenue#13, i_current_price#16, i_wholesale_cost#17, i_brand#18] Input [9]: [s_store_name#2, ss_store_sk#4, revenue#13, i_item_desc#15, i_current_price#16, i_wholesale_cost#17, i_brand#18, ss_store_sk#20, ave#34] -(48) TakeOrderedAndProject +(42) TakeOrderedAndProject Input [6]: [s_store_name#2, i_item_desc#15, revenue#13, i_current_price#16, i_wholesale_cost#17, i_brand#18] Arguments: 100, [s_store_name#2 ASC NULLS FIRST, i_item_desc#15 ASC NULLS FIRST], [s_store_name#2, i_item_desc#15, revenue#13, i_current_price#16, i_wholesale_cost#17, i_brand#18] ===== Subqueries ===== Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (53) -+- * ColumnarToRow (52) - +- CometProject (51) - +- CometFilter (50) - +- CometScan parquet spark_catalog.default.date_dim (49) +BroadcastExchange (47) ++- * ColumnarToRow (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan parquet spark_catalog.default.date_dim (43) -(49) Scan parquet spark_catalog.default.date_dim +(43) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#8, d_month_seq#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1176), LessThanOrEqual(d_month_seq,1187), IsNotNull(d_date_sk)] ReadSchema: struct -(50) CometFilter +(44) CometFilter Input [2]: [d_date_sk#8, d_month_seq#9] Condition : (((isnotnull(d_month_seq#9) AND (d_month_seq#9 >= 1176)) AND (d_month_seq#9 <= 1187)) AND isnotnull(d_date_sk#8)) -(51) CometProject +(45) CometProject Input [2]: [d_date_sk#8, d_month_seq#9] Arguments: [d_date_sk#8], [d_date_sk#8] -(52) ColumnarToRow [codegen id : 1] +(46) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#8] -(53) BroadcastExchange +(47) BroadcastExchange Input [1]: [d_date_sk#8] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] -Subquery:2 Hosting operator id = 28 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#7 +Subquery:2 Hosting operator id = 26 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt index 9480a4131..84d23af42 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q65/simplified.txt @@ -15,30 +15,28 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes WholeStageCodegen (2) Filter [revenue] HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_store_sk,ss_item_sk] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + Exchange [ss_store_sk,ss_item_sk] #2 + WholeStageCodegen (1) + HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] InputAdapter BroadcastExchange #5 WholeStageCodegen (3) @@ -51,24 +49,20 @@ TakeOrderedAndProject [s_store_name,i_item_desc,revenue,i_current_price,i_wholes WholeStageCodegen (6) Filter [ave] HashAggregate [ss_store_sk,sum,count] [avg(revenue),ave,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_store_sk] #7 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [ss_store_sk,revenue] [sum,count,sum,count] - HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_store_sk,ss_item_sk] #8 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_store_sk,ss_sales_price] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + InputAdapter + Exchange [ss_store_sk] #7 + WholeStageCodegen (5) + HashAggregate [ss_store_sk,revenue] [sum,count,sum,count] + HashAggregate [ss_store_sk,ss_item_sk,sum] [sum(UnscaledValue(ss_sales_price)),revenue,sum] + InputAdapter + Exchange [ss_store_sk,ss_item_sk] #8 + WholeStageCodegen (4) + HashAggregate [ss_store_sk,ss_item_sk,ss_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_store_sk,ss_sales_price] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt index 2c6e3a331..ea11cbb74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/explain.txt @@ -1,61 +1,55 @@ == Physical Plan == -TakeOrderedAndProject (57) -+- * HashAggregate (56) - +- * ColumnarToRow (55) - +- CometColumnarExchange (54) - +- RowToColumnar (53) - +- * HashAggregate (52) - +- Union (51) - :- * HashAggregate (30) - : +- * ColumnarToRow (29) - : +- CometColumnarExchange (28) - : +- RowToColumnar (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.warehouse (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.time_dim (13) - : +- CometBroadcastExchange (22) - : +- CometProject (21) - : +- CometFilter (20) - : +- CometScan parquet spark_catalog.default.ship_mode (19) - +- * HashAggregate (50) - +- * ColumnarToRow (49) - +- CometColumnarExchange (48) - +- RowToColumnar (47) - +- * HashAggregate (46) - +- * ColumnarToRow (45) - +- CometProject (44) - +- CometBroadcastHashJoin (43) - :- CometProject (41) - : +- CometBroadcastHashJoin (40) - : :- CometProject (38) - : : +- CometBroadcastHashJoin (37) - : : :- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometFilter (32) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (31) - : : : +- ReusedExchange (33) - : : +- ReusedExchange (36) - : +- ReusedExchange (39) - +- ReusedExchange (42) +TakeOrderedAndProject (51) ++- * HashAggregate (50) + +- Exchange (49) + +- * HashAggregate (48) + +- Union (47) + :- * HashAggregate (28) + : +- Exchange (27) + : +- * HashAggregate (26) + : +- * ColumnarToRow (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.warehouse (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (16) + : : +- CometProject (15) + : : +- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.time_dim (13) + : +- CometBroadcastExchange (22) + : +- CometProject (21) + : +- CometFilter (20) + : +- CometScan parquet spark_catalog.default.ship_mode (19) + +- * HashAggregate (46) + +- Exchange (45) + +- * HashAggregate (44) + +- * ColumnarToRow (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometProject (33) + : : : +- CometBroadcastHashJoin (32) + : : : :- CometFilter (30) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (29) + : : : +- ReusedExchange (31) + : : +- ReusedExchange (34) + : +- ReusedExchange (37) + +- ReusedExchange (40) (1) Scan parquet spark_catalog.default.web_sales @@ -184,24 +178,18 @@ Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 Aggregate Attributes [48]: [sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] Results [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, sum#71, isEmpty#72, sum#73, isEmpty#74, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118] -(27) RowToColumnar +(27) Exchange Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, sum#71, isEmpty#72, sum#73, isEmpty#74, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118] +Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(28) CometColumnarExchange -Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, sum#71, isEmpty#72, sum#73, isEmpty#74, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(29) ColumnarToRow [codegen id : 2] -Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, sum#71, isEmpty#72, sum#73, isEmpty#74, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118] - -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, sum#71, isEmpty#72, sum#73, isEmpty#74, sum#75, isEmpty#76, sum#77, isEmpty#78, sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118] Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17] Functions [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119, sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120, sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121, sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122, sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123, sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124, sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125, sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126, sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127, sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128, sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129, sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130, sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131, sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132, sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133, sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134, sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135, sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136, sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137, sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138, sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139, sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140, sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141, sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#142] Results [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, DHL,BARIAN AS ship_carriers#143, d_year#17 AS year#144, sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#119 AS jan_sales#145, sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#120 AS feb_sales#146, sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#121 AS mar_sales#147, sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#122 AS apr_sales#148, sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#123 AS may_sales#149, sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#124 AS jun_sales#150, sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#125 AS jul_sales#151, sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#126 AS aug_sales#152, sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#127 AS sep_sales#153, sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#128 AS oct_sales#154, sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#129 AS nov_sales#155, sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#130 AS dec_sales#156, sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#131 AS jan_net#157, sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#132 AS feb_net#158, sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#133 AS mar_net#159, sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#134 AS apr_net#160, sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#135 AS may_net#161, sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#136 AS jun_net#162, sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#137 AS jul_net#163, sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#138 AS aug_net#164, sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#139 AS sep_net#165, sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#140 AS oct_net#166, sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#141 AS nov_net#167, sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)#142 AS dec_net#168] -(31) Scan parquet spark_catalog.default.catalog_sales +(29) Scan parquet spark_catalog.default.catalog_sales Output [7]: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_warehouse_sk#171, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, cs_sold_date_sk#175] Batched: true Location: InMemoryFileIndex [] @@ -209,142 +197,130 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#175), dynamicpruningexpression(cs_s PushedFilters: [IsNotNull(cs_warehouse_sk), IsNotNull(cs_sold_time_sk), IsNotNull(cs_ship_mode_sk)] ReadSchema: struct -(32) CometFilter +(30) CometFilter Input [7]: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_warehouse_sk#171, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, cs_sold_date_sk#175] Condition : ((isnotnull(cs_warehouse_sk#171) AND isnotnull(cs_sold_time_sk#169)) AND isnotnull(cs_ship_mode_sk#170)) -(33) ReusedExchange [Reuses operator id: 5] +(31) ReusedExchange [Reuses operator id: 5] Output [7]: [w_warehouse_sk#177, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183] -(34) CometBroadcastHashJoin +(32) CometBroadcastHashJoin Left output [7]: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_warehouse_sk#171, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, cs_sold_date_sk#175] Right output [7]: [w_warehouse_sk#177, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183] Arguments: [cs_warehouse_sk#171], [w_warehouse_sk#177], Inner, BuildRight -(35) CometProject +(33) CometProject Input [14]: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_warehouse_sk#171, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, cs_sold_date_sk#175, w_warehouse_sk#177, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183] Arguments: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, cs_sold_date_sk#175, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183], [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, cs_sold_date_sk#175, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183] -(36) ReusedExchange [Reuses operator id: 10] +(34) ReusedExchange [Reuses operator id: 10] Output [3]: [d_date_sk#184, d_year#185, d_moy#186] -(37) CometBroadcastHashJoin +(35) CometBroadcastHashJoin Left output [12]: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, cs_sold_date_sk#175, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183] Right output [3]: [d_date_sk#184, d_year#185, d_moy#186] Arguments: [cs_sold_date_sk#175], [d_date_sk#184], Inner, BuildRight -(38) CometProject +(36) CometProject Input [15]: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, cs_sold_date_sk#175, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_date_sk#184, d_year#185, d_moy#186] Arguments: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186], [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186] -(39) ReusedExchange [Reuses operator id: 16] +(37) ReusedExchange [Reuses operator id: 16] Output [1]: [t_time_sk#187] -(40) CometBroadcastHashJoin +(38) CometBroadcastHashJoin Left output [13]: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186] Right output [1]: [t_time_sk#187] Arguments: [cs_sold_time_sk#169], [t_time_sk#187], Inner, BuildRight -(41) CometProject +(39) CometProject Input [14]: [cs_sold_time_sk#169, cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186, t_time_sk#187] Arguments: [cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186], [cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186] -(42) ReusedExchange [Reuses operator id: 22] +(40) ReusedExchange [Reuses operator id: 22] Output [1]: [sm_ship_mode_sk#188] -(43) CometBroadcastHashJoin +(41) CometBroadcastHashJoin Left output [12]: [cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186] Right output [1]: [sm_ship_mode_sk#188] Arguments: [cs_ship_mode_sk#170], [sm_ship_mode_sk#188], Inner, BuildRight -(44) CometProject +(42) CometProject Input [13]: [cs_ship_mode_sk#170, cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186, sm_ship_mode_sk#188] Arguments: [cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186], [cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186] -(45) ColumnarToRow [codegen id : 3] +(43) ColumnarToRow [codegen id : 3] Input [11]: [cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186] -(46) HashAggregate [codegen id : 3] +(44) HashAggregate [codegen id : 3] Input [11]: [cs_quantity#172, cs_sales_price#173, cs_net_paid_inc_tax#174, w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, d_moy#186] Keys [7]: [w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185] Functions [24]: [partial_sum(CASE WHEN (d_moy#186 = 1) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 2) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 3) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 4) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 5) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 6) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 7) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 8) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 9) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 10) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 11) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 12) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 1) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 2) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 3) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 4) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 5) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 6) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 7) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 8) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 9) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 10) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 11) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#186 = 12) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)] Aggregate Attributes [48]: [sum#189, isEmpty#190, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236] Results [55]: [w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284] -(47) RowToColumnar -Input [55]: [w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284] - -(48) CometColumnarExchange +(45) Exchange Input [55]: [w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284] -Arguments: hashpartitioning(w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(49) ColumnarToRow [codegen id : 4] -Input [55]: [w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284] - -(50) HashAggregate [codegen id : 4] +(46) HashAggregate [codegen id : 4] Input [55]: [w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262, sum#263, isEmpty#264, sum#265, isEmpty#266, sum#267, isEmpty#268, sum#269, isEmpty#270, sum#271, isEmpty#272, sum#273, isEmpty#274, sum#275, isEmpty#276, sum#277, isEmpty#278, sum#279, isEmpty#280, sum#281, isEmpty#282, sum#283, isEmpty#284] Keys [7]: [w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, d_year#185] Functions [24]: [sum(CASE WHEN (d_moy#186 = 1) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 2) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 3) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 4) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 5) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 6) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 7) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 8) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 9) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 10) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 11) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 12) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 1) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 2) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 3) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 4) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 5) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 6) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 7) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 8) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 9) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 10) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 11) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END), sum(CASE WHEN (d_moy#186 = 12) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)] Aggregate Attributes [24]: [sum(CASE WHEN (d_moy#186 = 1) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#285, sum(CASE WHEN (d_moy#186 = 2) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#286, sum(CASE WHEN (d_moy#186 = 3) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#287, sum(CASE WHEN (d_moy#186 = 4) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#288, sum(CASE WHEN (d_moy#186 = 5) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#289, sum(CASE WHEN (d_moy#186 = 6) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#290, sum(CASE WHEN (d_moy#186 = 7) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#291, sum(CASE WHEN (d_moy#186 = 8) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#292, sum(CASE WHEN (d_moy#186 = 9) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#293, sum(CASE WHEN (d_moy#186 = 10) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#294, sum(CASE WHEN (d_moy#186 = 11) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#295, sum(CASE WHEN (d_moy#186 = 12) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#296, sum(CASE WHEN (d_moy#186 = 1) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#297, sum(CASE WHEN (d_moy#186 = 2) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#298, sum(CASE WHEN (d_moy#186 = 3) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#299, sum(CASE WHEN (d_moy#186 = 4) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#300, sum(CASE WHEN (d_moy#186 = 5) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#301, sum(CASE WHEN (d_moy#186 = 6) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#302, sum(CASE WHEN (d_moy#186 = 7) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#303, sum(CASE WHEN (d_moy#186 = 8) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#304, sum(CASE WHEN (d_moy#186 = 9) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#305, sum(CASE WHEN (d_moy#186 = 10) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#306, sum(CASE WHEN (d_moy#186 = 11) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#307, sum(CASE WHEN (d_moy#186 = 12) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#308] Results [32]: [w_warehouse_name#178, w_warehouse_sq_ft#179, w_city#180, w_county#181, w_state#182, w_country#183, DHL,BARIAN AS ship_carriers#309, d_year#185 AS year#310, sum(CASE WHEN (d_moy#186 = 1) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#285 AS jan_sales#311, sum(CASE WHEN (d_moy#186 = 2) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#286 AS feb_sales#312, sum(CASE WHEN (d_moy#186 = 3) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#287 AS mar_sales#313, sum(CASE WHEN (d_moy#186 = 4) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#288 AS apr_sales#314, sum(CASE WHEN (d_moy#186 = 5) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#289 AS may_sales#315, sum(CASE WHEN (d_moy#186 = 6) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#290 AS jun_sales#316, sum(CASE WHEN (d_moy#186 = 7) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#291 AS jul_sales#317, sum(CASE WHEN (d_moy#186 = 8) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#292 AS aug_sales#318, sum(CASE WHEN (d_moy#186 = 9) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#293 AS sep_sales#319, sum(CASE WHEN (d_moy#186 = 10) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#294 AS oct_sales#320, sum(CASE WHEN (d_moy#186 = 11) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#295 AS nov_sales#321, sum(CASE WHEN (d_moy#186 = 12) THEN (cs_sales_price#173 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#296 AS dec_sales#322, sum(CASE WHEN (d_moy#186 = 1) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#297 AS jan_net#323, sum(CASE WHEN (d_moy#186 = 2) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#298 AS feb_net#324, sum(CASE WHEN (d_moy#186 = 3) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#299 AS mar_net#325, sum(CASE WHEN (d_moy#186 = 4) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#300 AS apr_net#326, sum(CASE WHEN (d_moy#186 = 5) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#301 AS may_net#327, sum(CASE WHEN (d_moy#186 = 6) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#302 AS jun_net#328, sum(CASE WHEN (d_moy#186 = 7) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#303 AS jul_net#329, sum(CASE WHEN (d_moy#186 = 8) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#304 AS aug_net#330, sum(CASE WHEN (d_moy#186 = 9) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#305 AS sep_net#331, sum(CASE WHEN (d_moy#186 = 10) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#306 AS oct_net#332, sum(CASE WHEN (d_moy#186 = 11) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#307 AS nov_net#333, sum(CASE WHEN (d_moy#186 = 12) THEN (cs_net_paid_inc_tax#174 * cast(cs_quantity#172 as decimal(10,0))) ELSE 0.00 END)#308 AS dec_net#334] -(51) Union +(47) Union -(52) HashAggregate [codegen id : 5] +(48) HashAggregate [codegen id : 5] Input [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, jan_sales#145, feb_sales#146, mar_sales#147, apr_sales#148, may_sales#149, jun_sales#150, jul_sales#151, aug_sales#152, sep_sales#153, oct_sales#154, nov_sales#155, dec_sales#156, jan_net#157, feb_net#158, mar_net#159, apr_net#160, may_net#161, jun_net#162, jul_net#163, aug_net#164, sep_net#165, oct_net#166, nov_net#167, dec_net#168] Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144] Functions [36]: [partial_sum(jan_sales#145), partial_sum(feb_sales#146), partial_sum(mar_sales#147), partial_sum(apr_sales#148), partial_sum(may_sales#149), partial_sum(jun_sales#150), partial_sum(jul_sales#151), partial_sum(aug_sales#152), partial_sum(sep_sales#153), partial_sum(oct_sales#154), partial_sum(nov_sales#155), partial_sum(dec_sales#156), partial_sum((jan_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((feb_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((mar_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((apr_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((may_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jun_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jul_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((aug_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((sep_sales#153 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((oct_sales#154 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((nov_sales#155 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((dec_sales#156 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum(jan_net#157), partial_sum(feb_net#158), partial_sum(mar_net#159), partial_sum(apr_net#160), partial_sum(may_net#161), partial_sum(jun_net#162), partial_sum(jul_net#163), partial_sum(aug_net#164), partial_sum(sep_net#165), partial_sum(oct_net#166), partial_sum(nov_net#167), partial_sum(dec_net#168)] Aggregate Attributes [72]: [sum#335, isEmpty#336, sum#337, isEmpty#338, sum#339, isEmpty#340, sum#341, isEmpty#342, sum#343, isEmpty#344, sum#345, isEmpty#346, sum#347, isEmpty#348, sum#349, isEmpty#350, sum#351, isEmpty#352, sum#353, isEmpty#354, sum#355, isEmpty#356, sum#357, isEmpty#358, sum#359, isEmpty#360, sum#361, isEmpty#362, sum#363, isEmpty#364, sum#365, isEmpty#366, sum#367, isEmpty#368, sum#369, isEmpty#370, sum#371, isEmpty#372, sum#373, isEmpty#374, sum#375, isEmpty#376, sum#377, isEmpty#378, sum#379, isEmpty#380, sum#381, isEmpty#382, sum#383, isEmpty#384, sum#385, isEmpty#386, sum#387, isEmpty#388, sum#389, isEmpty#390, sum#391, isEmpty#392, sum#393, isEmpty#394, sum#395, isEmpty#396, sum#397, isEmpty#398, sum#399, isEmpty#400, sum#401, isEmpty#402, sum#403, isEmpty#404, sum#405, isEmpty#406] Results [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476, sum#477, isEmpty#478] -(53) RowToColumnar -Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476, sum#477, isEmpty#478] - -(54) CometColumnarExchange -Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476, sum#477, isEmpty#478] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(55) ColumnarToRow [codegen id : 6] +(49) Exchange Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476, sum#477, isEmpty#478] +Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(56) HashAggregate [codegen id : 6] +(50) HashAggregate [codegen id : 6] Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, sum#407, isEmpty#408, sum#409, isEmpty#410, sum#411, isEmpty#412, sum#413, isEmpty#414, sum#415, isEmpty#416, sum#417, isEmpty#418, sum#419, isEmpty#420, sum#421, isEmpty#422, sum#423, isEmpty#424, sum#425, isEmpty#426, sum#427, isEmpty#428, sum#429, isEmpty#430, sum#431, isEmpty#432, sum#433, isEmpty#434, sum#435, isEmpty#436, sum#437, isEmpty#438, sum#439, isEmpty#440, sum#441, isEmpty#442, sum#443, isEmpty#444, sum#445, isEmpty#446, sum#447, isEmpty#448, sum#449, isEmpty#450, sum#451, isEmpty#452, sum#453, isEmpty#454, sum#455, isEmpty#456, sum#457, isEmpty#458, sum#459, isEmpty#460, sum#461, isEmpty#462, sum#463, isEmpty#464, sum#465, isEmpty#466, sum#467, isEmpty#468, sum#469, isEmpty#470, sum#471, isEmpty#472, sum#473, isEmpty#474, sum#475, isEmpty#476, sum#477, isEmpty#478] Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144] Functions [36]: [sum(jan_sales#145), sum(feb_sales#146), sum(mar_sales#147), sum(apr_sales#148), sum(may_sales#149), sum(jun_sales#150), sum(jul_sales#151), sum(aug_sales#152), sum(sep_sales#153), sum(oct_sales#154), sum(nov_sales#155), sum(dec_sales#156), sum((jan_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((feb_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((mar_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((apr_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((may_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((jun_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((jul_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((aug_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((sep_sales#153 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((oct_sales#154 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((nov_sales#155 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum((dec_sales#156 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), sum(jan_net#157), sum(feb_net#158), sum(mar_net#159), sum(apr_net#160), sum(may_net#161), sum(jun_net#162), sum(jul_net#163), sum(aug_net#164), sum(sep_net#165), sum(oct_net#166), sum(nov_net#167), sum(dec_net#168)] Aggregate Attributes [36]: [sum(jan_sales#145)#479, sum(feb_sales#146)#480, sum(mar_sales#147)#481, sum(apr_sales#148)#482, sum(may_sales#149)#483, sum(jun_sales#150)#484, sum(jul_sales#151)#485, sum(aug_sales#152)#486, sum(sep_sales#153)#487, sum(oct_sales#154)#488, sum(nov_sales#155)#489, sum(dec_sales#156)#490, sum((jan_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#491, sum((feb_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#492, sum((mar_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#493, sum((apr_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#494, sum((may_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#495, sum((jun_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#496, sum((jul_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#497, sum((aug_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#498, sum((sep_sales#153 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#499, sum((oct_sales#154 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#500, sum((nov_sales#155 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#501, sum((dec_sales#156 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#502, sum(jan_net#157)#503, sum(feb_net#158)#504, sum(mar_net#159)#505, sum(apr_net#160)#506, sum(may_net#161)#507, sum(jun_net#162)#508, sum(jul_net#163)#509, sum(aug_net#164)#510, sum(sep_net#165)#511, sum(oct_net#166)#512, sum(nov_net#167)#513, sum(dec_net#168)#514] Results [44]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, sum(jan_sales#145)#479 AS jan_sales#515, sum(feb_sales#146)#480 AS feb_sales#516, sum(mar_sales#147)#481 AS mar_sales#517, sum(apr_sales#148)#482 AS apr_sales#518, sum(may_sales#149)#483 AS may_sales#519, sum(jun_sales#150)#484 AS jun_sales#520, sum(jul_sales#151)#485 AS jul_sales#521, sum(aug_sales#152)#486 AS aug_sales#522, sum(sep_sales#153)#487 AS sep_sales#523, sum(oct_sales#154)#488 AS oct_sales#524, sum(nov_sales#155)#489 AS nov_sales#525, sum(dec_sales#156)#490 AS dec_sales#526, sum((jan_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#491 AS jan_sales_per_sq_foot#527, sum((feb_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#492 AS feb_sales_per_sq_foot#528, sum((mar_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#493 AS mar_sales_per_sq_foot#529, sum((apr_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#494 AS apr_sales_per_sq_foot#530, sum((may_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#495 AS may_sales_per_sq_foot#531, sum((jun_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#496 AS jun_sales_per_sq_foot#532, sum((jul_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#497 AS jul_sales_per_sq_foot#533, sum((aug_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#498 AS aug_sales_per_sq_foot#534, sum((sep_sales#153 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#499 AS sep_sales_per_sq_foot#535, sum((oct_sales#154 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#500 AS oct_sales_per_sq_foot#536, sum((nov_sales#155 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#501 AS nov_sales_per_sq_foot#537, sum((dec_sales#156 / cast(w_warehouse_sq_ft#11 as decimal(10,0))))#502 AS dec_sales_per_sq_foot#538, sum(jan_net#157)#503 AS jan_net#539, sum(feb_net#158)#504 AS feb_net#540, sum(mar_net#159)#505 AS mar_net#541, sum(apr_net#160)#506 AS apr_net#542, sum(may_net#161)#507 AS may_net#543, sum(jun_net#162)#508 AS jun_net#544, sum(jul_net#163)#509 AS jul_net#545, sum(aug_net#164)#510 AS aug_net#546, sum(sep_net#165)#511 AS sep_net#547, sum(oct_net#166)#512 AS oct_net#548, sum(nov_net#167)#513 AS nov_net#549, sum(dec_net#168)#514 AS dec_net#550] -(57) TakeOrderedAndProject +(51) TakeOrderedAndProject Input [44]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, jan_sales#515, feb_sales#516, mar_sales#517, apr_sales#518, may_sales#519, jun_sales#520, jul_sales#521, aug_sales#522, sep_sales#523, oct_sales#524, nov_sales#525, dec_sales#526, jan_sales_per_sq_foot#527, feb_sales_per_sq_foot#528, mar_sales_per_sq_foot#529, apr_sales_per_sq_foot#530, may_sales_per_sq_foot#531, jun_sales_per_sq_foot#532, jul_sales_per_sq_foot#533, aug_sales_per_sq_foot#534, sep_sales_per_sq_foot#535, oct_sales_per_sq_foot#536, nov_sales_per_sq_foot#537, dec_sales_per_sq_foot#538, jan_net#539, feb_net#540, mar_net#541, apr_net#542, may_net#543, jun_net#544, jul_net#545, aug_net#546, sep_net#547, oct_net#548, nov_net#549, dec_net#550] Arguments: 100, [w_warehouse_name#10 ASC NULLS FIRST], [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#143, year#144, jan_sales#515, feb_sales#516, mar_sales#517, apr_sales#518, may_sales#519, jun_sales#520, jul_sales#521, aug_sales#522, sep_sales#523, oct_sales#524, nov_sales#525, dec_sales#526, jan_sales_per_sq_foot#527, feb_sales_per_sq_foot#528, mar_sales_per_sq_foot#529, apr_sales_per_sq_foot#530, may_sales_per_sq_foot#531, jun_sales_per_sq_foot#532, jul_sales_per_sq_foot#533, aug_sales_per_sq_foot#534, sep_sales_per_sq_foot#535, oct_sales_per_sq_foot#536, nov_sales_per_sq_foot#537, dec_sales_per_sq_foot#538, jan_net#539, feb_net#540, mar_net#541, apr_net#542, may_net#543, jun_net#544, jul_net#545, aug_net#546, sep_net#547, oct_net#548, nov_net#549, dec_net#550] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (61) -+- * ColumnarToRow (60) - +- CometFilter (59) - +- CometScan parquet spark_catalog.default.date_dim (58) +BroadcastExchange (55) ++- * ColumnarToRow (54) + +- CometFilter (53) + +- CometScan parquet spark_catalog.default.date_dim (52) -(58) Scan parquet spark_catalog.default.date_dim +(52) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#16, d_year#17, d_moy#18] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(59) CometFilter +(53) CometFilter Input [3]: [d_date_sk#16, d_year#17, d_moy#18] Condition : ((isnotnull(d_year#17) AND (d_year#17 = 2001)) AND isnotnull(d_date_sk#16)) -(60) ColumnarToRow [codegen id : 1] +(54) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#16, d_year#17, d_moy#18] -(61) BroadcastExchange +(55) BroadcastExchange Input [3]: [d_date_sk#16, d_year#17, d_moy#18] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=4] -Subquery:2 Hosting operator id = 31 Hosting Expression = cs_sold_date_sk#175 IN dynamicpruning#8 +Subquery:2 Hosting operator id = 29 Hosting Expression = cs_sold_date_sk#175 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt index 4c60afa90..880c38bbf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q66/simplified.txt @@ -1,77 +1,71 @@ TakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] WholeStageCodegen (6) HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net),jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] - CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] - CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] - CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4 - CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [t_time_sk] #6 - CometProject [t_time_sk] - CometFilter [t_time_sk,t_time] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_time] - CometBroadcastExchange [sm_ship_mode_sk] #7 - CometProject [sm_ship_mode_sk] - CometFilter [sm_ship_mode_sk,sm_carrier] - CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] - WholeStageCodegen (4) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #8 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] - CometProject [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] - CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] - CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] - CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] - CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4 - ReusedExchange [d_date_sk,d_year,d_moy] #5 - ReusedExchange [t_time_sk] #6 - ReusedExchange [sm_ship_mode_sk] #7 + InputAdapter + Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 + WholeStageCodegen (5) + HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (2) + HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 + WholeStageCodegen (1) + HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [ws_sold_time_sk,ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_ship_mode_sk,ws_warehouse_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4 + CometFilter [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [t_time_sk] #6 + CometProject [t_time_sk] + CometFilter [t_time_sk,t_time] + CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_time] + CometBroadcastExchange [sm_ship_mode_sk] #7 + CometProject [sm_ship_mode_sk] + CometFilter [sm_ship_mode_sk,sm_carrier] + CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] + WholeStageCodegen (4) + HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #8 + WholeStageCodegen (3) + HashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] + CometProject [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,t_time_sk] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_date_sk,d_year,d_moy] + CometProject [cs_sold_time_sk,cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometBroadcastHashJoin [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] + CometFilter [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [w_warehouse_sk,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country] #4 + ReusedExchange [d_date_sk,d_year,d_moy] #5 + ReusedExchange [t_time_sk] #6 + ReusedExchange [sm_ship_mode_sk] #7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt index 6213ac10e..a1624de1c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/explain.txt @@ -1,39 +1,35 @@ == Physical Plan == -TakeOrderedAndProject (35) -+- * Filter (34) - +- Window (33) - +- WindowGroupLimit (32) - +- * ColumnarToRow (31) - +- CometSort (30) - +- CometColumnarExchange (29) - +- RowToColumnar (28) - +- WindowGroupLimit (27) - +- * Sort (26) - +- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometColumnarExchange (23) - +- RowToColumnar (22) - +- * HashAggregate (21) - +- * ColumnarToRow (20) - +- CometExpand (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.store (9) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometScan parquet spark_catalog.default.item (14) +TakeOrderedAndProject (31) ++- * Filter (30) + +- Window (29) + +- WindowGroupLimit (28) + +- * Sort (27) + +- Exchange (26) + +- WindowGroupLimit (25) + +- * Sort (24) + +- * HashAggregate (23) + +- Exchange (22) + +- * HashAggregate (21) + +- * ColumnarToRow (20) + +- CometExpand (19) + +- CometProject (18) + +- CometBroadcastHashJoin (17) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (16) + +- CometFilter (15) + +- CometScan parquet spark_catalog.default.item (14) (1) Scan parquet spark_catalog.default.store_sales @@ -138,90 +134,78 @@ Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as d Aggregate Attributes [2]: [sum#28, isEmpty#29] Results [11]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, sum#30, isEmpty#31] -(22) RowToColumnar +(22) Exchange Input [11]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, sum#30, isEmpty#31] +Arguments: hashpartitioning(i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(23) CometColumnarExchange -Input [11]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, sum#30, isEmpty#31] -Arguments: hashpartitioning(i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(24) ColumnarToRow [codegen id : 2] -Input [11]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, sum#30, isEmpty#31] - -(25) HashAggregate [codegen id : 2] +(23) HashAggregate [codegen id : 2] Input [11]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, sum#30, isEmpty#31] Keys [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#32] Results [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#32 AS sumsales#33] -(26) Sort [codegen id : 2] +(24) Sort [codegen id : 2] Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33] Arguments: [i_category#19 ASC NULLS FIRST, sumsales#33 DESC NULLS LAST], false, 0 -(27) WindowGroupLimit +(25) WindowGroupLimit Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33] Arguments: [i_category#19], [sumsales#33 DESC NULLS LAST], rank(sumsales#33), 100, Partial -(28) RowToColumnar +(26) Exchange Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33] +Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(29) CometColumnarExchange -Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33] -Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(30) CometSort -Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33] -Arguments: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33], [i_category#19 ASC NULLS FIRST, sumsales#33 DESC NULLS LAST] - -(31) ColumnarToRow [codegen id : 3] +(27) Sort [codegen id : 3] Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33] +Arguments: [i_category#19 ASC NULLS FIRST, sumsales#33 DESC NULLS LAST], false, 0 -(32) WindowGroupLimit +(28) WindowGroupLimit Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33] Arguments: [i_category#19], [sumsales#33 DESC NULLS LAST], rank(sumsales#33), 100, Final -(33) Window +(29) Window Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33] Arguments: [rank(sumsales#33) windowspecdefinition(i_category#19, sumsales#33 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#34], [i_category#19], [sumsales#33 DESC NULLS LAST] -(34) Filter [codegen id : 4] +(30) Filter [codegen id : 4] Input [10]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33, rk#34] Condition : (rk#34 <= 100) -(35) TakeOrderedAndProject +(31) TakeOrderedAndProject Input [10]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33, rk#34] Arguments: 100, [i_category#19 ASC NULLS FIRST, i_class#20 ASC NULLS FIRST, i_brand#21 ASC NULLS FIRST, i_product_name#22 ASC NULLS FIRST, d_year#23 ASC NULLS FIRST, d_qoy#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, s_store_id#26 ASC NULLS FIRST, sumsales#33 ASC NULLS FIRST, rk#34 ASC NULLS FIRST], [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#33, rk#34] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (40) -+- * ColumnarToRow (39) - +- CometProject (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.date_dim (36) +BroadcastExchange (36) ++- * ColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan parquet spark_catalog.default.date_dim (32) -(36) Scan parquet spark_catalog.default.date_dim +(32) Scan parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(37) CometFilter +(33) CometFilter Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1200)) AND (d_month_seq#8 <= 1211)) AND isnotnull(d_date_sk#7)) -(38) CometProject +(34) CometProject Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -(39) ColumnarToRow [codegen id : 1] +(35) ColumnarToRow [codegen id : 1] Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -(40) BroadcastExchange +(36) BroadcastExchange Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt index 968c7d056..46d9cf4ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q67/simplified.txt @@ -5,47 +5,43 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ Window [sumsales,i_category] WindowGroupLimit [i_category,sumsales] WholeStageCodegen (3) - ColumnarToRow + Sort [i_category,sumsales] InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometColumnarExchange [i_category] #1 - RowToColumnar - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (2) - Sort [i_category,sumsales] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] - CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #5 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + Exchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (2) + Sort [i_category,sumsales] + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] + InputAdapter + Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] + CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt index 054c5eb63..850cd5ccb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/explain.txt @@ -1,48 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * Project (43) - +- * BroadcastHashJoin Inner BuildRight (42) - :- * Project (37) - : +- * BroadcastHashJoin Inner BuildRight (36) - : :- * HashAggregate (31) - : : +- * ColumnarToRow (30) - : : +- CometColumnarExchange (29) - : : +- RowToColumnar (28) - : : +- * HashAggregate (27) - : : +- * ColumnarToRow (26) - : : +- CometProject (25) - : : +- CometBroadcastHashJoin (24) - : : :- CometProject (20) - : : : +- CometBroadcastHashJoin (19) - : : : :- CometProject (14) - : : : : +- CometBroadcastHashJoin (13) - : : : : :- CometProject (8) - : : : : : +- CometBroadcastHashJoin (7) - : : : : : :- CometFilter (2) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : +- CometBroadcastExchange (6) - : : : : : +- CometProject (5) - : : : : : +- CometFilter (4) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : : +- CometBroadcastExchange (12) - : : : : +- CometProject (11) - : : : : +- CometFilter (10) - : : : : +- CometScan parquet spark_catalog.default.store (9) - : : : +- CometBroadcastExchange (18) - : : : +- CometProject (17) - : : : +- CometFilter (16) - : : : +- CometScan parquet spark_catalog.default.household_demographics (15) - : : +- CometBroadcastExchange (23) - : : +- CometFilter (22) - : : +- CometScan parquet spark_catalog.default.customer_address (21) - : +- BroadcastExchange (35) - : +- * ColumnarToRow (34) - : +- CometFilter (33) - : +- CometScan parquet spark_catalog.default.customer (32) - +- BroadcastExchange (41) - +- * ColumnarToRow (40) - +- CometFilter (39) - +- CometScan parquet spark_catalog.default.customer_address (38) +TakeOrderedAndProject (42) ++- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * HashAggregate (29) + : : +- Exchange (28) + : : +- * HashAggregate (27) + : : +- * ColumnarToRow (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (20) + : : : +- CometBroadcastHashJoin (19) + : : : :- CometProject (14) + : : : : +- CometBroadcastHashJoin (13) + : : : : :- CometProject (8) + : : : : : +- CometBroadcastHashJoin (7) + : : : : : :- CometFilter (2) + : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometBroadcastExchange (6) + : : : : : +- CometProject (5) + : : : : : +- CometFilter (4) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : : +- CometBroadcastExchange (12) + : : : : +- CometProject (11) + : : : : +- CometFilter (10) + : : : : +- CometScan parquet spark_catalog.default.store (9) + : : : +- CometBroadcastExchange (18) + : : : +- CometProject (17) + : : : +- CometFilter (16) + : : : +- CometScan parquet spark_catalog.default.household_demographics (15) + : : +- CometBroadcastExchange (23) + : : +- CometFilter (22) + : : +- CometScan parquet spark_catalog.default.customer_address (21) + : +- BroadcastExchange (33) + : +- * ColumnarToRow (32) + : +- CometFilter (31) + : +- CometScan parquet spark_catalog.default.customer (30) + +- BroadcastExchange (39) + +- * ColumnarToRow (38) + +- CometFilter (37) + +- CometScan parquet spark_catalog.default.customer_address (36) (1) Scan parquet spark_catalog.default.store_sales @@ -175,112 +173,106 @@ Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(Un Aggregate Attributes [3]: [sum#21, sum#22, sum#23] Results [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#24, sum#25, sum#26] -(28) RowToColumnar +(28) Exchange Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#24, sum#25, sum#26] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(29) CometColumnarExchange -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#24, sum#25, sum#26] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(30) ColumnarToRow [codegen id : 4] -Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#24, sum#25, sum#26] - -(31) HashAggregate [codegen id : 4] +(29) HashAggregate [codegen id : 4] Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#24, sum#25, sum#26] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20] Functions [3]: [sum(UnscaledValue(ss_ext_sales_price#6)), sum(UnscaledValue(ss_ext_list_price#7)), sum(UnscaledValue(ss_ext_tax#8))] Aggregate Attributes [3]: [sum(UnscaledValue(ss_ext_sales_price#6))#27, sum(UnscaledValue(ss_ext_list_price#7))#28, sum(UnscaledValue(ss_ext_tax#8))#29] Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ca_city#20 AS bought_city#30, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#6))#27,17,2) AS extended_price#31, MakeDecimal(sum(UnscaledValue(ss_ext_list_price#7))#28,17,2) AS list_price#32, MakeDecimal(sum(UnscaledValue(ss_ext_tax#8))#29,17,2) AS extended_tax#33] -(32) Scan parquet spark_catalog.default.customer +(30) Scan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(33) CometFilter +(31) CometFilter Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Condition : (isnotnull(c_customer_sk#34) AND isnotnull(c_current_addr_sk#35)) -(34) ColumnarToRow [codegen id : 2] +(32) ColumnarToRow [codegen id : 2] Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] -(35) BroadcastExchange +(33) BroadcastExchange Input [4]: [c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] -(36) BroadcastHashJoin [codegen id : 4] +(34) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_customer_sk#1] Right keys [1]: [c_customer_sk#34] Join type: Inner Join condition: None -(37) Project [codegen id : 4] +(35) Project [codegen id : 4] Output [8]: [ss_ticket_number#5, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37] Input [10]: [ss_ticket_number#5, ss_customer_sk#1, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_customer_sk#34, c_current_addr_sk#35, c_first_name#36, c_last_name#37] -(38) Scan parquet spark_catalog.default.customer_address +(36) Scan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#38, ca_city#39] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk), IsNotNull(ca_city)] ReadSchema: struct -(39) CometFilter +(37) CometFilter Input [2]: [ca_address_sk#38, ca_city#39] Condition : (isnotnull(ca_address_sk#38) AND isnotnull(ca_city#39)) -(40) ColumnarToRow [codegen id : 3] +(38) ColumnarToRow [codegen id : 3] Input [2]: [ca_address_sk#38, ca_city#39] -(41) BroadcastExchange +(39) BroadcastExchange Input [2]: [ca_address_sk#38, ca_city#39] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(42) BroadcastHashJoin [codegen id : 4] +(40) BroadcastHashJoin [codegen id : 4] Left keys [1]: [c_current_addr_sk#35] Right keys [1]: [ca_address_sk#38] Join type: Inner Join condition: NOT (ca_city#39 = bought_city#30) -(43) Project [codegen id : 4] +(41) Project [codegen id : 4] Output [8]: [c_last_name#37, c_first_name#36, ca_city#39, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] Input [10]: [ss_ticket_number#5, bought_city#30, extended_price#31, list_price#32, extended_tax#33, c_current_addr_sk#35, c_first_name#36, c_last_name#37, ca_address_sk#38, ca_city#39] -(44) TakeOrderedAndProject +(42) TakeOrderedAndProject Input [8]: [c_last_name#37, c_first_name#36, ca_city#39, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] Arguments: 100, [c_last_name#37 ASC NULLS FIRST, ss_ticket_number#5 ASC NULLS FIRST], [c_last_name#37, c_first_name#36, ca_city#39, bought_city#30, ss_ticket_number#5, extended_price#31, extended_tax#33, list_price#32] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (49) -+- * ColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan parquet spark_catalog.default.date_dim (45) +BroadcastExchange (47) ++- * ColumnarToRow (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan parquet spark_catalog.default.date_dim (43) -(45) Scan parquet spark_catalog.default.date_dim +(43) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#11, d_year#12, d_dom#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dom), GreaterThanOrEqual(d_dom,1), LessThanOrEqual(d_dom,2), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(46) CometFilter +(44) CometFilter Input [3]: [d_date_sk#11, d_year#12, d_dom#13] Condition : ((((isnotnull(d_dom#13) AND (d_dom#13 >= 1)) AND (d_dom#13 <= 2)) AND d_year#12 IN (1999,2000,2001)) AND isnotnull(d_date_sk#11)) -(47) CometProject +(45) CometProject Input [3]: [d_date_sk#11, d_year#12, d_dom#13] Arguments: [d_date_sk#11], [d_date_sk#11] -(48) ColumnarToRow [codegen id : 1] +(46) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#11] -(49) BroadcastExchange +(47) BroadcastExchange Input [1]: [d_date_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt index df6e8541d..78e1b1cce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q68/simplified.txt @@ -5,47 +5,45 @@ TakeOrderedAndProject [c_last_name,ss_ticket_number,c_first_name,ca_city,bought_ Project [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] BroadcastHashJoin [ss_customer_sk,c_customer_sk] HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax)),bought_city,extended_price,list_price,extended_tax,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] - CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,s_store_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] - CometBroadcastExchange [s_store_sk] #4 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_city] - CometScan parquet spark_catalog.default.store [s_store_sk,s_city] - CometBroadcastExchange [hd_demo_sk] #5 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometBroadcastExchange [ca_address_sk,ca_city] #6 - CometFilter [ca_address_sk,ca_city] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] + InputAdapter + Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + WholeStageCodegen (1) + HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] [sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] + CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,s_store_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dom] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dom] + CometBroadcastExchange [s_store_sk] #4 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_city] + CometScan parquet spark_catalog.default.store [s_store_sk,s_city] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometBroadcastExchange [ca_address_sk,ca_city] #6 + CometFilter [ca_address_sk,ca_city] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city] InputAdapter BroadcastExchange #7 WholeStageCodegen (2) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt index 4991d569f..b931de292 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/explain.txt @@ -1,50 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (27) - : : +- * BroadcastHashJoin LeftAnti BuildRight (26) - : : :- * BroadcastHashJoin LeftAnti BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometProject (30) - : +- CometFilter (29) - : +- CometScan parquet spark_catalog.default.customer_address (28) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- Exchange (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (27) + : : +- * BroadcastHashJoin LeftAnti BuildRight (26) + : : :- * BroadcastHashJoin LeftAnti BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * ColumnarToRow (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan parquet spark_catalog.default.customer_address (28) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometFilter (36) + +- CometScan parquet spark_catalog.default.customer_demographics (35) (1) Scan parquet spark_catalog.default.customer @@ -240,56 +238,50 @@ Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#26] Results [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] -(42) RowToColumnar +(42) Exchange Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] +Arguments: hashpartitioning(cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) CometColumnarExchange -Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] -Arguments: hashpartitioning(cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 6] -Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] - -(45) HashAggregate [codegen id : 6] +(43) HashAggregate [codegen id : 6] Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] Keys [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#28] Results [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, count(1)#28 AS cnt1#29, cd_purchase_estimate#24, count(1)#28 AS cnt2#30, cd_credit_rating#25, count(1)#28 AS cnt3#31] -(46) TakeOrderedAndProject +(44) TakeOrderedAndProject Input [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#29, cd_purchase_estimate#24, cnt2#30, cd_credit_rating#25, cnt3#31] Arguments: 100, [cd_gender#21 ASC NULLS FIRST, cd_marital_status#22 ASC NULLS FIRST, cd_education_status#23 ASC NULLS FIRST, cd_purchase_estimate#24 ASC NULLS FIRST, cd_credit_rating#25 ASC NULLS FIRST], [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#29, cd_purchase_estimate#24, cnt2#30, cd_credit_rating#25, cnt3#31] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (51) -+- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.date_dim (47) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan parquet spark_catalog.default.date_dim (45) -(47) Scan parquet spark_catalog.default.date_dim +(45) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] ReadSchema: struct -(48) CometFilter +(46) CometFilter Input [3]: [d_date_sk#7, d_year#8, d_moy#9] Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2001)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 6)) AND isnotnull(d_date_sk#7)) -(49) CometProject +(47) CometProject Input [3]: [d_date_sk#7, d_year#8, d_moy#9] Arguments: [d_date_sk#7], [d_date_sk#7] -(50) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#7] -(51) BroadcastExchange +(49) BroadcastExchange Input [1]: [d_date_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt index 9e1e3edb6..c9cc4959b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q69/simplified.txt @@ -1,72 +1,70 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + InputAdapter + Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt index 08ea991ad..5755ed1dc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/explain.txt @@ -1,36 +1,34 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * HashAggregate (31) - +- * ColumnarToRow (30) - +- CometColumnarExchange (29) - +- RowToColumnar (28) - +- * HashAggregate (27) - +- * ColumnarToRow (26) - +- CometProject (25) - +- CometBroadcastHashJoin (24) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.date_dim (9) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.item (15) - +- CometBroadcastExchange (23) - +- CometProject (22) - +- CometFilter (21) - +- CometScan parquet spark_catalog.default.promotion (20) +TakeOrderedAndProject (30) ++- * HashAggregate (29) + +- Exchange (28) + +- * HashAggregate (27) + +- * ColumnarToRow (26) + +- CometProject (25) + +- CometBroadcastHashJoin (24) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.date_dim (9) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometScan parquet spark_catalog.default.item (15) + +- CometBroadcastExchange (23) + +- CometProject (22) + +- CometFilter (21) + +- CometScan parquet spark_catalog.default.promotion (20) (1) Scan parquet spark_catalog.default.store_sales @@ -163,56 +161,50 @@ Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_pr Aggregate Attributes [8]: [sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] Results [9]: [i_item_id#17, sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] -(28) RowToColumnar +(28) Exchange Input [9]: [i_item_id#17, sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(29) CometColumnarExchange -Input [9]: [i_item_id#17, sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] -Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(30) ColumnarToRow [codegen id : 2] -Input [9]: [i_item_id#17, sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] - -(31) HashAggregate [codegen id : 2] +(29) HashAggregate [codegen id : 2] Input [9]: [i_item_id#17, sum#29, count#30, sum#31, count#32, sum#33, count#34, sum#35, count#36] Keys [1]: [i_item_id#17] Functions [4]: [avg(ss_quantity#4), avg(UnscaledValue(ss_list_price#5)), avg(UnscaledValue(ss_coupon_amt#7)), avg(UnscaledValue(ss_sales_price#6))] Aggregate Attributes [4]: [avg(ss_quantity#4)#37, avg(UnscaledValue(ss_list_price#5))#38, avg(UnscaledValue(ss_coupon_amt#7))#39, avg(UnscaledValue(ss_sales_price#6))#40] Results [5]: [i_item_id#17, avg(ss_quantity#4)#37 AS agg1#41, cast((avg(UnscaledValue(ss_list_price#5))#38 / 100.0) as decimal(11,6)) AS agg2#42, cast((avg(UnscaledValue(ss_coupon_amt#7))#39 / 100.0) as decimal(11,6)) AS agg3#43, cast((avg(UnscaledValue(ss_sales_price#6))#40 / 100.0) as decimal(11,6)) AS agg4#44] -(32) TakeOrderedAndProject +(30) TakeOrderedAndProject Input [5]: [i_item_id#17, agg1#41, agg2#42, agg3#43, agg4#44] Arguments: 100, [i_item_id#17 ASC NULLS FIRST], [i_item_id#17, agg1#41, agg2#42, agg3#43, agg4#44] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (37) -+- * ColumnarToRow (36) - +- CometProject (35) - +- CometFilter (34) - +- CometScan parquet spark_catalog.default.date_dim (33) +BroadcastExchange (35) ++- * ColumnarToRow (34) + +- CometProject (33) + +- CometFilter (32) + +- CometScan parquet spark_catalog.default.date_dim (31) -(33) Scan parquet spark_catalog.default.date_dim +(31) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_year#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(34) CometFilter +(32) CometFilter Input [2]: [d_date_sk#14, d_year#15] Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2000)) AND isnotnull(d_date_sk#14)) -(35) CometProject +(33) CometProject Input [2]: [d_date_sk#14, d_year#15] Arguments: [d_date_sk#14], [d_date_sk#14] -(36) ColumnarToRow [codegen id : 1] +(34) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(37) BroadcastExchange +(35) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt index 04db30380..a27daca36 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q7/simplified.txt @@ -1,44 +1,42 @@ TakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] WholeStageCodegen (2) HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price)),agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] - CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [p_promo_sk] #6 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_email,p_channel_event] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] + InputAdapter + Exchange [i_item_id] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] + CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] + CometProject [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk] #3 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [p_promo_sk] #6 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_email,p_channel_event] + CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_email,p_channel_event] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt index b55d8224b..0a6934960 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/explain.txt @@ -1,55 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (51) -+- * Project (50) - +- Window (49) - +- * ColumnarToRow (48) - +- CometSort (47) - +- CometColumnarExchange (46) - +- RowToColumnar (45) - +- * HashAggregate (44) - +- * ColumnarToRow (43) - +- CometColumnarExchange (42) - +- RowToColumnar (41) - +- * HashAggregate (40) - +- * Expand (39) - +- * Project (38) - +- * BroadcastHashJoin Inner BuildRight (37) - :- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.store_sales (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.date_dim (3) - +- BroadcastExchange (36) - +- * BroadcastHashJoin LeftSemi BuildRight (35) - :- * ColumnarToRow (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.store (10) - +- BroadcastExchange (34) - +- * Project (33) - +- * Filter (32) - +- Window (31) - +- WindowGroupLimit (30) - +- * Sort (29) - +- * HashAggregate (28) - +- * ColumnarToRow (27) - +- CometColumnarExchange (26) - +- RowToColumnar (25) - +- * HashAggregate (24) - +- * ColumnarToRow (23) - +- CometProject (22) - +- CometBroadcastHashJoin (21) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.store_sales (13) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.store (15) - +- ReusedExchange (20) +TakeOrderedAndProject (45) ++- * Project (44) + +- Window (43) + +- * Sort (42) + +- Exchange (41) + +- * HashAggregate (40) + +- Exchange (39) + +- * HashAggregate (38) + +- * Expand (37) + +- * Project (36) + +- * BroadcastHashJoin Inner BuildRight (35) + :- * ColumnarToRow (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.date_dim (3) + +- BroadcastExchange (34) + +- * BroadcastHashJoin LeftSemi BuildRight (33) + :- * ColumnarToRow (12) + : +- CometFilter (11) + : +- CometScan parquet spark_catalog.default.store (10) + +- BroadcastExchange (32) + +- * Project (31) + +- * Filter (30) + +- Window (29) + +- WindowGroupLimit (28) + +- * Sort (27) + +- * HashAggregate (26) + +- Exchange (25) + +- * HashAggregate (24) + +- * ColumnarToRow (23) + +- CometProject (22) + +- CometBroadcastHashJoin (21) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.store_sales (13) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometScan parquet spark_catalog.default.store (15) + +- ReusedExchange (20) (1) Scan parquet spark_catalog.default.store_sales @@ -167,150 +161,132 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] Aggregate Attributes [1]: [sum#17] Results [2]: [s_state#15, sum#18] -(25) RowToColumnar +(25) Exchange Input [2]: [s_state#15, sum#18] +Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(26) CometColumnarExchange -Input [2]: [s_state#15, sum#18] -Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(27) ColumnarToRow [codegen id : 2] -Input [2]: [s_state#15, sum#18] - -(28) HashAggregate [codegen id : 2] +(26) HashAggregate [codegen id : 2] Input [2]: [s_state#15, sum#18] Keys [1]: [s_state#15] Functions [1]: [sum(UnscaledValue(ss_net_profit#11))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#11))#19] Results [3]: [s_state#15, MakeDecimal(sum(UnscaledValue(ss_net_profit#11))#19,17,2) AS _w0#20, s_state#15] -(29) Sort [codegen id : 2] +(27) Sort [codegen id : 2] Input [3]: [s_state#15, _w0#20, s_state#15] Arguments: [s_state#15 ASC NULLS FIRST, _w0#20 DESC NULLS LAST], false, 0 -(30) WindowGroupLimit +(28) WindowGroupLimit Input [3]: [s_state#15, _w0#20, s_state#15] Arguments: [s_state#15], [_w0#20 DESC NULLS LAST], rank(_w0#20), 5, Final -(31) Window +(29) Window Input [3]: [s_state#15, _w0#20, s_state#15] Arguments: [rank(_w0#20) windowspecdefinition(s_state#15, _w0#20 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#21], [s_state#15], [_w0#20 DESC NULLS LAST] -(32) Filter [codegen id : 3] +(30) Filter [codegen id : 3] Input [4]: [s_state#15, _w0#20, s_state#15, ranking#21] Condition : (ranking#21 <= 5) -(33) Project [codegen id : 3] +(31) Project [codegen id : 3] Output [1]: [s_state#15] Input [4]: [s_state#15, _w0#20, s_state#15, ranking#21] -(34) BroadcastExchange +(32) BroadcastExchange Input [1]: [s_state#15] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] -(35) BroadcastHashJoin [codegen id : 4] +(33) BroadcastHashJoin [codegen id : 4] Left keys [1]: [s_state#9] Right keys [1]: [s_state#15] Join type: LeftSemi Join condition: None -(36) BroadcastExchange +(34) BroadcastExchange Input [3]: [s_store_sk#7, s_county#8, s_state#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(37) BroadcastHashJoin [codegen id : 5] +(35) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#7] Join type: Inner Join condition: None -(38) Project [codegen id : 5] +(36) Project [codegen id : 5] Output [3]: [ss_net_profit#2, s_state#9, s_county#8] Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_county#8, s_state#9] -(39) Expand [codegen id : 5] +(37) Expand [codegen id : 5] Input [3]: [ss_net_profit#2, s_state#9, s_county#8] Arguments: [[ss_net_profit#2, s_state#9, s_county#8, 0], [ss_net_profit#2, s_state#9, null, 1], [ss_net_profit#2, null, null, 3]], [ss_net_profit#2, s_state#22, s_county#23, spark_grouping_id#24] -(40) HashAggregate [codegen id : 5] +(38) HashAggregate [codegen id : 5] Input [4]: [ss_net_profit#2, s_state#22, s_county#23, spark_grouping_id#24] Keys [3]: [s_state#22, s_county#23, spark_grouping_id#24] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum#25] Results [4]: [s_state#22, s_county#23, spark_grouping_id#24, sum#26] -(41) RowToColumnar -Input [4]: [s_state#22, s_county#23, spark_grouping_id#24, sum#26] - -(42) CometColumnarExchange -Input [4]: [s_state#22, s_county#23, spark_grouping_id#24, sum#26] -Arguments: hashpartitioning(s_state#22, s_county#23, spark_grouping_id#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(43) ColumnarToRow [codegen id : 6] +(39) Exchange Input [4]: [s_state#22, s_county#23, spark_grouping_id#24, sum#26] +Arguments: hashpartitioning(s_state#22, s_county#23, spark_grouping_id#24, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(44) HashAggregate [codegen id : 6] +(40) HashAggregate [codegen id : 6] Input [4]: [s_state#22, s_county#23, spark_grouping_id#24, sum#26] Keys [3]: [s_state#22, s_county#23, spark_grouping_id#24] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#27] Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#27,17,2) AS total_sum#28, s_state#22, s_county#23, (cast((shiftright(spark_grouping_id#24, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#24, 0) & 1) as tinyint)) AS lochierarchy#29, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#27,17,2) AS _w0#30, (cast((shiftright(spark_grouping_id#24, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#24, 0) & 1) as tinyint)) AS _w1#31, CASE WHEN (cast((shiftright(spark_grouping_id#24, 0) & 1) as tinyint) = 0) THEN s_state#22 END AS _w2#32] -(45) RowToColumnar -Input [7]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32] - -(46) CometColumnarExchange -Input [7]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32] -Arguments: hashpartitioning(_w1#31, _w2#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(47) CometSort +(41) Exchange Input [7]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32] -Arguments: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32], [_w1#31 ASC NULLS FIRST, _w2#32 ASC NULLS FIRST, _w0#30 DESC NULLS LAST] +Arguments: hashpartitioning(_w1#31, _w2#32, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(48) ColumnarToRow [codegen id : 7] +(42) Sort [codegen id : 7] Input [7]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32] +Arguments: [_w1#31 ASC NULLS FIRST, _w2#32 ASC NULLS FIRST, _w0#30 DESC NULLS LAST], false, 0 -(49) Window +(43) Window Input [7]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32] Arguments: [rank(_w0#30) windowspecdefinition(_w1#31, _w2#32, _w0#30 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#33], [_w1#31, _w2#32], [_w0#30 DESC NULLS LAST] -(50) Project [codegen id : 8] +(44) Project [codegen id : 8] Output [5]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, rank_within_parent#33] Input [8]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, _w0#30, _w1#31, _w2#32, rank_within_parent#33] -(51) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [5]: [total_sum#28, s_state#22, s_county#23, lochierarchy#29, rank_within_parent#33] Arguments: 100, [lochierarchy#29 DESC NULLS LAST, CASE WHEN (lochierarchy#29 = 0) THEN s_state#22 END ASC NULLS FIRST, rank_within_parent#33 ASC NULLS FIRST], [total_sum#28, s_state#22, s_county#23, lochierarchy#29, rank_within_parent#33] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (56) -+- * ColumnarToRow (55) - +- CometProject (54) - +- CometFilter (53) - +- CometScan parquet spark_catalog.default.date_dim (52) +BroadcastExchange (50) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(52) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(53) CometFilter +(47) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) -(54) CometProject +(48) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(55) ColumnarToRow [codegen id : 1] +(49) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(56) BroadcastExchange +(50) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt index b8e21d9d4..4a6dffc08 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q70/simplified.txt @@ -4,75 +4,69 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (7) - ColumnarToRow + Sort [_w1,_w2,_w0] InputAdapter - CometSort [total_sum,s_state,s_county,lochierarchy,_w0,_w1,_w2] - CometColumnarExchange [_w1,_w2] #1 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_state,s_county,spark_grouping_id] #2 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] + Exchange [_w1,_w2] #1 + WholeStageCodegen (6) + HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + InputAdapter + Exchange [s_state,s_county,spark_grouping_id] #2 + WholeStageCodegen (5) + HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] + Expand [ss_net_profit,s_state,s_county] + Project [ss_net_profit,s_state,s_county] + BroadcastHashJoin [ss_store_sk,s_store_sk] + ColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (4) + BroadcastHashJoin [s_state,s_state] ColumnarToRow InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometFilter [s_store_sk,s_county,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter - BroadcastExchange #5 - WholeStageCodegen (4) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow + BroadcastExchange #6 + WholeStageCodegen (3) + Project [s_state] + Filter [ranking] InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (2) - Sort [s_state,_w0] - HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (2) + Sort [s_state,_w0] + HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] + InputAdapter + Exchange [s_state] #7 + WholeStageCodegen (1) + HashAggregate [s_state,ss_net_profit] [sum,sum] ColumnarToRow InputAdapter - CometColumnarExchange [s_state] #7 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_state,ss_net_profit] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #8 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #4 + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [s_store_sk,s_state] #8 + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt index 137da5e62..286b71984 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/explain.txt @@ -1,45 +1,43 @@ == Physical Plan == -* ColumnarToRow (41) -+- CometSort (40) - +- CometColumnarExchange (39) - +- RowToColumnar (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometColumnarExchange (35) - +- RowToColumnar (34) - +- * HashAggregate (33) - +- * ColumnarToRow (32) - +- CometProject (31) - +- CometBroadcastHashJoin (30) - :- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometBroadcastExchange (4) - : : +- CometProject (3) - : : +- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.item (1) - : +- CometUnion (23) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometFilter (6) - : : : +- CometScan parquet spark_catalog.default.web_sales (5) - : : +- CometBroadcastExchange (10) - : : +- CometProject (9) - : : +- CometFilter (8) - : : +- CometScan parquet spark_catalog.default.date_dim (7) - : :- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (13) - : : +- ReusedExchange (15) - : +- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometFilter (19) - : : +- CometScan parquet spark_catalog.default.store_sales (18) - : +- ReusedExchange (20) - +- CometBroadcastExchange (29) - +- CometProject (28) - +- CometFilter (27) - +- CometScan parquet spark_catalog.default.time_dim (26) +* ColumnarToRow (39) ++- CometSort (38) + +- CometColumnarExchange (37) + +- RowToColumnar (36) + +- * HashAggregate (35) + +- Exchange (34) + +- * HashAggregate (33) + +- * ColumnarToRow (32) + +- CometProject (31) + +- CometBroadcastHashJoin (30) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometBroadcastExchange (4) + : : +- CometProject (3) + : : +- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.item (1) + : +- CometUnion (23) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometFilter (6) + : : : +- CometScan parquet spark_catalog.default.web_sales (5) + : : +- CometBroadcastExchange (10) + : : +- CometProject (9) + : : +- CometFilter (8) + : : +- CometScan parquet spark_catalog.default.date_dim (7) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (13) + : : +- ReusedExchange (15) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometFilter (19) + : : +- CometScan parquet spark_catalog.default.store_sales (18) + : +- ReusedExchange (20) + +- CometBroadcastExchange (29) + +- CometProject (28) + +- CometFilter (27) + +- CometScan parquet spark_catalog.default.time_dim (26) (1) Scan parquet spark_catalog.default.item @@ -201,66 +199,60 @@ Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] Aggregate Attributes [1]: [sum#38] Results [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#39] -(34) RowToColumnar +(34) Exchange Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#39] +Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(35) CometColumnarExchange -Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#39] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(36) ColumnarToRow [codegen id : 2] -Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#39] - -(37) HashAggregate [codegen id : 2] +(35) HashAggregate [codegen id : 2] Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#39] Keys [4]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36] Functions [1]: [sum(UnscaledValue(ext_price#13))] Aggregate Attributes [1]: [sum(UnscaledValue(ext_price#13))#40] Results [5]: [i_brand_id#2 AS brand_id#41, i_brand#3 AS brand#42, t_hour#35, t_minute#36, MakeDecimal(sum(UnscaledValue(ext_price#13))#40,17,2) AS ext_price#43] -(38) RowToColumnar +(36) RowToColumnar Input [5]: [brand_id#41, brand#42, t_hour#35, t_minute#36, ext_price#43] -(39) CometColumnarExchange +(37) CometColumnarExchange Input [5]: [brand_id#41, brand#42, t_hour#35, t_minute#36, ext_price#43] Arguments: rangepartitioning(ext_price#43 DESC NULLS LAST, brand_id#41 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(40) CometSort +(38) CometSort Input [5]: [brand_id#41, brand#42, t_hour#35, t_minute#36, ext_price#43] Arguments: [brand_id#41, brand#42, t_hour#35, t_minute#36, ext_price#43], [ext_price#43 DESC NULLS LAST, brand_id#41 ASC NULLS FIRST] -(41) ColumnarToRow [codegen id : 3] +(39) ColumnarToRow [codegen id : 3] Input [5]: [brand_id#41, brand#42, t_hour#35, t_minute#36, ext_price#43] ===== Subqueries ===== Subquery:1 Hosting operator id = 5 Hosting Expression = ws_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (46) -+- * ColumnarToRow (45) - +- CometProject (44) - +- CometFilter (43) - +- CometScan parquet spark_catalog.default.date_dim (42) +BroadcastExchange (44) ++- * ColumnarToRow (43) + +- CometProject (42) + +- CometFilter (41) + +- CometScan parquet spark_catalog.default.date_dim (40) -(42) Scan parquet spark_catalog.default.date_dim +(40) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_moy), IsNotNull(d_year), EqualTo(d_moy,11), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(43) CometFilter +(41) CometFilter Input [3]: [d_date_sk#10, d_year#11, d_moy#12] Condition : ((((isnotnull(d_moy#12) AND isnotnull(d_year#11)) AND (d_moy#12 = 11)) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) -(44) CometProject +(42) CometProject Input [3]: [d_date_sk#10, d_year#11, d_moy#12] Arguments: [d_date_sk#10], [d_date_sk#10] -(45) ColumnarToRow [codegen id : 1] +(43) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#10] -(46) BroadcastExchange +(44) BroadcastExchange Input [1]: [d_date_sk#10] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt index a4649a1c3..4b19fa36e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q71/simplified.txt @@ -6,52 +6,50 @@ WholeStageCodegen (3) RowToColumnar WholeStageCodegen (2) HashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum] [sum(UnscaledValue(ext_price)),brand_id,brand,ext_price,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_brand,i_brand_id,t_hour,t_minute] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_brand,i_brand_id,t_hour,t_minute,ext_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] - CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] - CometProject [i_brand_id,i_brand,ext_price,time_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] - CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 - CometProject [i_item_sk,i_brand_id,i_brand] - CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] - CometUnion [ext_price,sold_item_sk,time_sk] - CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] - CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] - CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] - CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] - CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [t_time_sk,t_hour,t_minute] #6 - CometProject [t_time_sk,t_hour,t_minute] - CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] - CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] + InputAdapter + Exchange [i_brand,i_brand_id,t_hour,t_minute] #2 + WholeStageCodegen (1) + HashAggregate [i_brand,i_brand_id,t_hour,t_minute,ext_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] + CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] + CometProject [i_brand_id,i_brand,ext_price,time_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_brand,ext_price,sold_item_sk,time_sk] + CometBroadcastExchange [i_item_sk,i_brand_id,i_brand] #3 + CometProject [i_item_sk,i_brand_id,i_brand] + CometFilter [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_brand,i_manager_id] + CometUnion [ext_price,sold_item_sk,time_sk] + CometProject [ws_ext_sales_price,ws_item_sk,ws_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,d_date_sk] + CometFilter [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_sold_time_sk,ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [cs_ext_sales_price,cs_item_sk,cs_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,d_date_sk] + CometFilter [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_sold_time_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + CometProject [ss_ext_sales_price,ss_item_sk,ss_sold_time_sk] [ext_price,sold_item_sk,time_sk] + CometBroadcastHashJoin [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] + CometFilter [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_sold_time_sk,ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [t_time_sk,t_hour,t_minute] #6 + CometProject [t_time_sk,t_hour,t_minute] + CometFilter [t_time_sk,t_hour,t_minute,t_meal_time] + CometScan parquet spark_catalog.default.time_dim [t_time_sk,t_hour,t_minute,t_meal_time] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt index 2f6da7d32..f53b2f84e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/explain.txt @@ -1,72 +1,71 @@ == Physical Plan == -* ColumnarToRow (68) -+- CometTakeOrderedAndProject (67) - +- CometHashAggregate (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometProject (63) - +- CometSortMergeJoin (62) - :- CometSort (56) - : +- CometColumnarExchange (55) - : +- RowToColumnar (54) - : +- * Project (53) - : +- * BroadcastHashJoin LeftOuter BuildRight (52) - : :- * Project (47) - : : +- * BroadcastHashJoin Inner BuildRight (46) - : : :- * ColumnarToRow (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometProject (35) - : : : : +- CometBroadcastHashJoin (34) - : : : : :- CometProject (29) - : : : : : +- CometBroadcastHashJoin (28) - : : : : : :- CometProject (23) - : : : : : : +- CometBroadcastHashJoin (22) - : : : : : : :- CometProject (17) - : : : : : : : +- CometBroadcastHashJoin (16) - : : : : : : : :- CometProject (12) - : : : : : : : : +- CometBroadcastHashJoin (11) - : : : : : : : : :- CometProject (7) - : : : : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- CometBroadcastExchange (5) - : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) - : : : : : : : : +- CometBroadcastExchange (10) - : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) - : : : : : : : +- CometBroadcastExchange (15) - : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan parquet spark_catalog.default.item (13) - : : : : : : +- CometBroadcastExchange (21) - : : : : : : +- CometProject (20) - : : : : : : +- CometFilter (19) - : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) - : : : : : +- CometBroadcastExchange (27) - : : : : : +- CometProject (26) - : : : : : +- CometFilter (25) - : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) - : : : : +- CometBroadcastExchange (33) - : : : : +- CometProject (32) - : : : : +- CometFilter (31) - : : : : +- CometScan parquet spark_catalog.default.date_dim (30) - : : : +- CometBroadcastExchange (38) - : : : +- CometFilter (37) - : : : +- CometScan parquet spark_catalog.default.date_dim (36) - : : +- BroadcastExchange (45) - : : +- * ColumnarToRow (44) - : : +- CometFilter (43) - : : +- CometScan parquet spark_catalog.default.date_dim (42) - : +- BroadcastExchange (51) - : +- * ColumnarToRow (50) - : +- CometFilter (49) - : +- CometScan parquet spark_catalog.default.promotion (48) - +- CometSort (61) - +- CometColumnarExchange (60) - +- CometProject (59) - +- CometFilter (58) - +- CometScan parquet spark_catalog.default.catalog_returns (57) +TakeOrderedAndProject (67) ++- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * SortMergeJoin LeftOuter (62) + :- * Sort (55) + : +- Exchange (54) + : +- * Project (53) + : +- * BroadcastHashJoin LeftOuter BuildRight (52) + : :- * Project (47) + : : +- * BroadcastHashJoin Inner BuildRight (46) + : : :- * ColumnarToRow (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometScan parquet spark_catalog.default.date_dim (36) + : : +- BroadcastExchange (45) + : : +- * ColumnarToRow (44) + : : +- CometFilter (43) + : : +- CometScan parquet spark_catalog.default.date_dim (42) + : +- BroadcastExchange (51) + : +- * ColumnarToRow (50) + : +- CometFilter (49) + : +- CometScan parquet spark_catalog.default.promotion (48) + +- * ColumnarToRow (61) + +- CometSort (60) + +- CometExchange (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.catalog_returns (56) (1) Scan parquet spark_catalog.default.catalog_sales @@ -321,99 +320,101 @@ Join condition: None Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -(54) RowToColumnar +(54) Exchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(55) CometColumnarExchange +(55) Sort [codegen id : 4] Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 -(56) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] - -(57) Scan parquet spark_catalog.default.catalog_returns +(56) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(58) CometFilter +(57) CometFilter Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) -(59) CometProject +(58) CometProject Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] -(60) CometColumnarExchange +(59) CometExchange Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(61) CometSort +(60) CometSort Input [2]: [cr_item_sk#31, cr_order_number#32] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] -(62) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +(61) ColumnarToRow [codegen id : 5] +Input [2]: [cr_item_sk#31, cr_order_number#32] -(63) CometProject +(62) SortMergeJoin [codegen id : 6] +Left keys [2]: [cs_item_sk#4, cs_order_number#6] +Right keys [2]: [cr_item_sk#31, cr_order_number#32] +Join type: LeftOuter +Join condition: None + +(63) Project [codegen id : 6] +Output [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(64) CometHashAggregate +(64) HashAggregate [codegen id : 6] Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#34] +Results [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] -(65) CometColumnarExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +(65) Exchange +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] +Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(66) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] +(66) HashAggregate [codegen id : 7] +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#36] +Results [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count(1)#36 AS no_promo#37, count(1)#36 AS promo#38, count(1)#36 AS total_cnt#39] -(67) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -(68) ColumnarToRow [codegen id : 4] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +(67) TakeOrderedAndProject +Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] +Arguments: 100, [total_cnt#39 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (73) -+- * ColumnarToRow (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan parquet spark_catalog.default.date_dim (69) +BroadcastExchange (72) ++- * ColumnarToRow (71) + +- CometProject (70) + +- CometFilter (69) + +- CometScan parquet spark_catalog.default.date_dim (68) -(69) Scan parquet spark_catalog.default.date_dim +(68) Scan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(70) CometFilter +(69) CometFilter Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 1999)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) -(71) CometProject +(70) CometProject Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] -(72) ColumnarToRow [codegen id : 1] +(71) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -(73) BroadcastExchange +(72) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt index 1022e1a90..1efc2e5d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q72/simplified.txt @@ -1,86 +1,92 @@ -WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] - CometColumnarExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] - CometProject [w_warehouse_name,i_item_desc,d_week_seq] - CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometColumnarExchange [cs_item_sk,cs_order_number] #2 - RowToColumnar - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - ColumnarToRow - InputAdapter - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 - CometFilter [d_date_sk,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] +TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] + WholeStageCodegen (7) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] + InputAdapter + Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + WholeStageCodegen (6) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] + Project [w_warehouse_name,i_item_desc,d_week_seq] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter + WholeStageCodegen (4) + Sort [cs_item_sk,cs_order_number] + InputAdapter + Exchange [cs_item_sk,cs_order_number] #2 + WholeStageCodegen (3) + Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_promo_sk,p_promo_sk] + Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] + ColumnarToRow + InputAdapter + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #7 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #8 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometFilter [d_date_sk,d_week_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #11 - WholeStageCodegen (1) + BroadcastExchange #12 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] - CometSort [cr_item_sk,cr_order_number] - CometColumnarExchange [cr_item_sk,cr_order_number] #13 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometFilter [p_promo_sk] + CometScan parquet spark_catalog.default.promotion [p_promo_sk] + InputAdapter + WholeStageCodegen (5) + ColumnarToRow + InputAdapter + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #13 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt index 6ea426978..8b4e36c96 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/explain.txt @@ -6,7 +6,7 @@ +- CometBroadcastHashJoin (28) :- CometFilter (24) : +- CometHashAggregate (23) - : +- CometColumnarExchange (22) + : +- CometExchange (22) : +- CometHashAggregate (21) : +- CometProject (20) : +- CometBroadcastHashJoin (19) @@ -134,9 +134,9 @@ Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt index 5fb8a197a..441736532 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q73/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometColumnarExchange [ss_ticket_number,ss_customer_sk] #2 + CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt index 2fad5f7a5..d5eb481f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/explain.txt @@ -1,83 +1,75 @@ == Physical Plan == -TakeOrderedAndProject (79) -+- * Project (78) - +- * BroadcastHashJoin Inner BuildRight (77) - :- * Project (59) - : +- * BroadcastHashJoin Inner BuildRight (58) - : :- * BroadcastHashJoin Inner BuildRight (39) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- * ColumnarToRow (17) - : : : +- CometColumnarExchange (16) - : : : +- RowToColumnar (15) - : : : +- * HashAggregate (14) - : : : +- * ColumnarToRow (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- BroadcastExchange (38) - : : +- * HashAggregate (37) - : : +- * ColumnarToRow (36) - : : +- CometColumnarExchange (35) - : : +- RowToColumnar (34) - : : +- * HashAggregate (33) - : : +- * ColumnarToRow (32) - : : +- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (26) - : : : +- CometBroadcastHashJoin (25) - : : : :- CometFilter (21) - : : : : +- CometScan parquet spark_catalog.default.customer (20) - : : : +- CometBroadcastExchange (24) - : : : +- CometFilter (23) - : : : +- CometScan parquet spark_catalog.default.store_sales (22) - : : +- CometBroadcastExchange (29) - : : +- CometFilter (28) - : : +- CometScan parquet spark_catalog.default.date_dim (27) - : +- BroadcastExchange (57) - : +- * Filter (56) - : +- * HashAggregate (55) - : +- * ColumnarToRow (54) - : +- CometColumnarExchange (53) - : +- RowToColumnar (52) - : +- * HashAggregate (51) - : +- * ColumnarToRow (50) - : +- CometProject (49) - : +- CometBroadcastHashJoin (48) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometFilter (41) - : : : +- CometScan parquet spark_catalog.default.customer (40) - : : +- CometBroadcastExchange (44) - : : +- CometFilter (43) - : : +- CometScan parquet spark_catalog.default.web_sales (42) - : +- ReusedExchange (47) - +- BroadcastExchange (76) - +- * HashAggregate (75) - +- * ColumnarToRow (74) - +- CometColumnarExchange (73) - +- RowToColumnar (72) - +- * HashAggregate (71) - +- * ColumnarToRow (70) - +- CometProject (69) - +- CometBroadcastHashJoin (68) - :- CometProject (66) - : +- CometBroadcastHashJoin (65) - : :- CometFilter (61) - : : +- CometScan parquet spark_catalog.default.customer (60) - : +- CometBroadcastExchange (64) - : +- CometFilter (63) - : +- CometScan parquet spark_catalog.default.web_sales (62) - +- ReusedExchange (67) +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (53) + : +- * BroadcastHashJoin Inner BuildRight (52) + : :- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (17) + : : : +- * HashAggregate (16) + : : : +- Exchange (15) + : : : +- * HashAggregate (14) + : : : +- * ColumnarToRow (13) + : : : +- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- Exchange (32) + : : +- * HashAggregate (31) + : : +- * ColumnarToRow (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometFilter (19) + : : : : +- CometScan parquet spark_catalog.default.customer (18) + : : : +- CometBroadcastExchange (22) + : : : +- CometFilter (21) + : : : +- CometScan parquet spark_catalog.default.store_sales (20) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (51) + : +- * Filter (50) + : +- * HashAggregate (49) + : +- Exchange (48) + : +- * HashAggregate (47) + : +- * ColumnarToRow (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (42) + : : +- CometBroadcastHashJoin (41) + : : :- CometFilter (37) + : : : +- CometScan parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (40) + : : +- CometFilter (39) + : : +- CometScan parquet spark_catalog.default.web_sales (38) + : +- ReusedExchange (43) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- Exchange (66) + +- * HashAggregate (65) + +- * ColumnarToRow (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometFilter (55) + : : +- CometScan parquet spark_catalog.default.customer (54) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometScan parquet spark_catalog.default.web_sales (56) + +- ReusedExchange (61) (1) Scan parquet spark_catalog.default.customer @@ -150,39 +142,33 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] Aggregate Attributes [1]: [sum#11] Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#12] -(15) RowToColumnar +(15) Exchange Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#12] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(16) CometColumnarExchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#12] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(17) ColumnarToRow [codegen id : 8] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#12] - -(18) HashAggregate [codegen id : 8] +(16) HashAggregate [codegen id : 8] Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#12] Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10] Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#13] Results [2]: [c_customer_id#2 AS customer_id#14, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#13,17,2) AS year_total#15] -(19) Filter [codegen id : 8] +(17) Filter [codegen id : 8] Input [2]: [customer_id#14, year_total#15] Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) -(20) Scan parquet spark_catalog.default.customer +(18) Scan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(21) CometFilter +(19) CometFilter Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_customer_id#17)) -(22) Scan parquet spark_catalog.default.store_sales +(20) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] Batched: true Location: InMemoryFileIndex [] @@ -190,96 +176,90 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#22), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(23) CometFilter +(21) CometFilter Input [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] Condition : isnotnull(ss_customer_sk#20) -(24) CometBroadcastExchange +(22) CometBroadcastExchange Input [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(25) CometBroadcastHashJoin +(23) CometBroadcastHashJoin Left output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(26) CometProject +(24) CometProject Input [7]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] Arguments: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, ss_sold_date_sk#22] -(27) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [2]: [d_date_sk#24, d_year#25] Condition : (((isnotnull(d_year#25) AND (d_year#25 = 2002)) AND d_year#25 IN (2001,2002)) AND isnotnull(d_date_sk#24)) -(29) CometBroadcastExchange +(27) CometBroadcastExchange Input [2]: [d_date_sk#24, d_year#25] Arguments: [d_date_sk#24, d_year#25] -(30) CometBroadcastHashJoin +(28) CometBroadcastHashJoin Left output [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, ss_sold_date_sk#22] Right output [2]: [d_date_sk#24, d_year#25] Arguments: [ss_sold_date_sk#22], [d_date_sk#24], Inner, BuildRight -(31) CometProject +(29) CometProject Input [7]: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#24, d_year#25] Arguments: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, d_year#25], [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, d_year#25] -(32) ColumnarToRow [codegen id : 2] +(30) ColumnarToRow [codegen id : 2] Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, d_year#25] -(33) HashAggregate [codegen id : 2] +(31) HashAggregate [codegen id : 2] Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, d_year#25] Keys [4]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] Aggregate Attributes [1]: [sum#26] Results [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, sum#27] -(34) RowToColumnar -Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, sum#27] - -(35) CometColumnarExchange -Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, sum#27] -Arguments: hashpartitioning(c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(36) ColumnarToRow [codegen id : 3] +(32) Exchange Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, sum#27] +Arguments: hashpartitioning(c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(37) HashAggregate [codegen id : 3] +(33) HashAggregate [codegen id : 3] Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, sum#27] Keys [4]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25] Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#21))#13] Results [4]: [c_customer_id#17 AS customer_id#28, c_first_name#18 AS customer_first_name#29, c_last_name#19 AS customer_last_name#30, MakeDecimal(sum(UnscaledValue(ss_net_paid#21))#13,17,2) AS year_total#31] -(38) BroadcastExchange +(34) BroadcastExchange Input [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(39) BroadcastHashJoin [codegen id : 8] +(35) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#14] Right keys [1]: [customer_id#28] Join type: Inner Join condition: None -(40) Scan parquet spark_catalog.default.customer +(36) Scan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(41) CometFilter +(37) CometFilter Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_customer_id#33)) -(42) Scan parquet spark_catalog.default.web_sales +(38) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] Batched: true Location: InMemoryFileIndex [] @@ -287,92 +267,86 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#38), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(43) CometFilter +(39) CometFilter Input [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] Condition : isnotnull(ws_bill_customer_sk#36) -(44) CometBroadcastExchange +(40) CometBroadcastExchange Input [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] Arguments: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] -(45) CometBroadcastHashJoin +(41) CometBroadcastHashJoin Left output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Right output [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] Arguments: [c_customer_sk#32], [ws_bill_customer_sk#36], Inner, BuildRight -(46) CometProject +(42) CometProject Input [7]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35, ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, ws_sold_date_sk#38], [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, ws_sold_date_sk#38] -(47) ReusedExchange [Reuses operator id: 10] +(43) ReusedExchange [Reuses operator id: 10] Output [2]: [d_date_sk#40, d_year#41] -(48) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, ws_sold_date_sk#38] Right output [2]: [d_date_sk#40, d_year#41] Arguments: [ws_sold_date_sk#38], [d_date_sk#40], Inner, BuildRight -(49) CometProject +(45) CometProject Input [7]: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, ws_sold_date_sk#38, d_date_sk#40, d_year#41] Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, d_year#41], [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, d_year#41] -(50) ColumnarToRow [codegen id : 4] +(46) ColumnarToRow [codegen id : 4] Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, d_year#41] -(51) HashAggregate [codegen id : 4] +(47) HashAggregate [codegen id : 4] Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, d_year#41] Keys [4]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#37))] Aggregate Attributes [1]: [sum#42] Results [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, sum#43] -(52) RowToColumnar -Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, sum#43] - -(53) CometColumnarExchange -Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, sum#43] -Arguments: hashpartitioning(c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(54) ColumnarToRow [codegen id : 5] +(48) Exchange Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, sum#43] +Arguments: hashpartitioning(c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(55) HashAggregate [codegen id : 5] +(49) HashAggregate [codegen id : 5] Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, sum#43] Keys [4]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41] Functions [1]: [sum(UnscaledValue(ws_net_paid#37))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#37))#44] Results [2]: [c_customer_id#33 AS customer_id#45, MakeDecimal(sum(UnscaledValue(ws_net_paid#37))#44,17,2) AS year_total#46] -(56) Filter [codegen id : 5] +(50) Filter [codegen id : 5] Input [2]: [customer_id#45, year_total#46] Condition : (isnotnull(year_total#46) AND (year_total#46 > 0.00)) -(57) BroadcastExchange +(51) BroadcastExchange Input [2]: [customer_id#45, year_total#46] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(58) BroadcastHashJoin [codegen id : 8] +(52) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#14] Right keys [1]: [customer_id#45] Join type: Inner Join condition: None -(59) Project [codegen id : 8] +(53) Project [codegen id : 8] Output [7]: [customer_id#14, year_total#15, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#46] Input [8]: [customer_id#14, year_total#15, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#45, year_total#46] -(60) Scan parquet spark_catalog.default.customer +(54) Scan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(61) CometFilter +(55) CometFilter Input [4]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50] Condition : (isnotnull(c_customer_sk#47) AND isnotnull(c_customer_id#48)) -(62) Scan parquet spark_catalog.default.web_sales +(56) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] @@ -380,134 +354,128 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#53), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(63) CometFilter +(57) CometFilter Input [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] Condition : isnotnull(ws_bill_customer_sk#51) -(64) CometBroadcastExchange +(58) CometBroadcastExchange Input [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] Arguments: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] -(65) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [4]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50] Right output [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] Arguments: [c_customer_sk#47], [ws_bill_customer_sk#51], Inner, BuildRight -(66) CometProject +(60) CometProject Input [7]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50, ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] Arguments: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, ws_sold_date_sk#53], [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, ws_sold_date_sk#53] -(67) ReusedExchange [Reuses operator id: 29] +(61) ReusedExchange [Reuses operator id: 27] Output [2]: [d_date_sk#55, d_year#56] -(68) CometBroadcastHashJoin +(62) CometBroadcastHashJoin Left output [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, ws_sold_date_sk#53] Right output [2]: [d_date_sk#55, d_year#56] Arguments: [ws_sold_date_sk#53], [d_date_sk#55], Inner, BuildRight -(69) CometProject +(63) CometProject Input [7]: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, ws_sold_date_sk#53, d_date_sk#55, d_year#56] Arguments: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, d_year#56], [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, d_year#56] -(70) ColumnarToRow [codegen id : 6] +(64) ColumnarToRow [codegen id : 6] Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, d_year#56] -(71) HashAggregate [codegen id : 6] +(65) HashAggregate [codegen id : 6] Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, d_year#56] Keys [4]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#52))] Aggregate Attributes [1]: [sum#57] Results [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, sum#58] -(72) RowToColumnar -Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, sum#58] - -(73) CometColumnarExchange -Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, sum#58] -Arguments: hashpartitioning(c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(74) ColumnarToRow [codegen id : 7] +(66) Exchange Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, sum#58] +Arguments: hashpartitioning(c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(75) HashAggregate [codegen id : 7] +(67) HashAggregate [codegen id : 7] Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, sum#58] Keys [4]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56] Functions [1]: [sum(UnscaledValue(ws_net_paid#52))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#52))#44] Results [2]: [c_customer_id#48 AS customer_id#59, MakeDecimal(sum(UnscaledValue(ws_net_paid#52))#44,17,2) AS year_total#60] -(76) BroadcastExchange +(68) BroadcastExchange Input [2]: [customer_id#59, year_total#60] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(77) BroadcastHashJoin [codegen id : 8] +(69) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#14] Right keys [1]: [customer_id#59] Join type: Inner Join condition: (CASE WHEN (year_total#46 > 0.00) THEN (year_total#60 / year_total#46) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#31 / year_total#15) END) -(78) Project [codegen id : 8] +(70) Project [codegen id : 8] Output [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] Input [9]: [customer_id#14, year_total#15, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#46, customer_id#59, year_total#60] -(79) TakeOrderedAndProject +(71) TakeOrderedAndProject Input [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] Arguments: 100, [customer_id#28 ASC NULLS FIRST, customer_id#28 ASC NULLS FIRST, customer_id#28 ASC NULLS FIRST], [customer_id#28, customer_first_name#29, customer_last_name#30] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (83) -+- * ColumnarToRow (82) - +- CometFilter (81) - +- CometScan parquet spark_catalog.default.date_dim (80) +BroadcastExchange (75) ++- * ColumnarToRow (74) + +- CometFilter (73) + +- CometScan parquet spark_catalog.default.date_dim (72) -(80) Scan parquet spark_catalog.default.date_dim +(72) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(81) CometFilter +(73) CometFilter Input [2]: [d_date_sk#9, d_year#10] Condition : (((isnotnull(d_year#10) AND (d_year#10 = 2001)) AND d_year#10 IN (2001,2002)) AND isnotnull(d_date_sk#9)) -(82) ColumnarToRow [codegen id : 1] +(74) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#9, d_year#10] -(83) BroadcastExchange +(75) BroadcastExchange Input [2]: [d_date_sk#9, d_year#10] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#23 -BroadcastExchange (87) -+- * ColumnarToRow (86) - +- CometFilter (85) - +- CometScan parquet spark_catalog.default.date_dim (84) +Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#23 +BroadcastExchange (79) ++- * ColumnarToRow (78) + +- CometFilter (77) + +- CometScan parquet spark_catalog.default.date_dim (76) -(84) Scan parquet spark_catalog.default.date_dim +(76) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(85) CometFilter +(77) CometFilter Input [2]: [d_date_sk#24, d_year#25] Condition : (((isnotnull(d_year#25) AND (d_year#25 = 2002)) AND d_year#25 IN (2001,2002)) AND isnotnull(d_date_sk#24)) -(86) ColumnarToRow [codegen id : 1] +(78) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#24, d_year#25] -(87) BroadcastExchange +(79) BroadcastExchange Input [2]: [d_date_sk#24, d_year#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] -Subquery:3 Hosting operator id = 42 Hosting Expression = ws_sold_date_sk#38 IN dynamicpruning#8 +Subquery:3 Hosting operator id = 38 Hosting Expression = ws_sold_date_sk#38 IN dynamicpruning#8 -Subquery:4 Hosting operator id = 62 Hosting Expression = ws_sold_date_sk#53 IN dynamicpruning#23 +Subquery:4 Hosting operator id = 56 Hosting Expression = ws_sold_date_sk#53 IN dynamicpruning#23 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt index 0cc944868..a7aa864ce 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q74/simplified.txt @@ -7,11 +7,38 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] BroadcastHashJoin [customer_id,customer_id] Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - RowToColumnar - WholeStageCodegen (1) + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + WholeStageCodegen (2) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] ColumnarToRow InputAdapter @@ -21,94 +48,59 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #8 WholeStageCodegen (1) ColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #10 WholeStageCodegen (5) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 - CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + WholeStageCodegen (4) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_year] #4 InputAdapter BroadcastExchange #13 WholeStageCodegen (7) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + WholeStageCodegen (6) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt index 08fb52740..31a832dcc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/explain.txt @@ -1,137 +1,129 @@ == Physical Plan == -TakeOrderedAndProject (133) -+- * Project (132) - +- * SortMergeJoin Inner (131) - :- * ColumnarToRow (71) - : +- CometSort (70) - : +- CometColumnarExchange (69) - : +- RowToColumnar (68) - : +- * Filter (67) - : +- * HashAggregate (66) - : +- * ColumnarToRow (65) - : +- CometColumnarExchange (64) - : +- RowToColumnar (63) - : +- * HashAggregate (62) - : +- * ColumnarToRow (61) - : +- CometHashAggregate (60) - : +- CometColumnarExchange (59) - : +- CometHashAggregate (58) - : +- CometUnion (57) - : :- CometProject (22) - : : +- CometSortMergeJoin (21) - : : :- CometSort (15) - : : : +- CometColumnarExchange (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.item (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.date_dim (9) - : : +- CometSort (20) - : : +- CometColumnarExchange (19) - : : +- CometProject (18) - : : +- CometFilter (17) - : : +- CometScan parquet spark_catalog.default.catalog_returns (16) - : :- CometProject (39) - : : +- CometSortMergeJoin (38) - : : :- CometSort (32) - : : : +- CometColumnarExchange (31) - : : : +- CometProject (30) - : : : +- CometBroadcastHashJoin (29) - : : : :- CometProject (27) - : : : : +- CometBroadcastHashJoin (26) - : : : : :- CometFilter (24) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (23) - : : : : +- ReusedExchange (25) - : : : +- ReusedExchange (28) - : : +- CometSort (37) - : : +- CometColumnarExchange (36) - : : +- CometProject (35) - : : +- CometFilter (34) - : : +- CometScan parquet spark_catalog.default.store_returns (33) - : +- CometProject (56) - : +- CometSortMergeJoin (55) - : :- CometSort (49) - : : +- CometColumnarExchange (48) - : : +- CometProject (47) - : : +- CometBroadcastHashJoin (46) - : : :- CometProject (44) - : : : +- CometBroadcastHashJoin (43) - : : : :- CometFilter (41) - : : : : +- CometScan parquet spark_catalog.default.web_sales (40) - : : : +- ReusedExchange (42) - : : +- ReusedExchange (45) - : +- CometSort (54) - : +- CometColumnarExchange (53) - : +- CometProject (52) - : +- CometFilter (51) - : +- CometScan parquet spark_catalog.default.web_returns (50) - +- * ColumnarToRow (130) - +- CometSort (129) - +- CometColumnarExchange (128) - +- RowToColumnar (127) - +- * Filter (126) - +- * HashAggregate (125) - +- * ColumnarToRow (124) - +- CometColumnarExchange (123) - +- RowToColumnar (122) - +- * HashAggregate (121) - +- * ColumnarToRow (120) - +- CometHashAggregate (119) - +- CometColumnarExchange (118) - +- CometHashAggregate (117) - +- CometUnion (116) - :- CometProject (87) - : +- CometSortMergeJoin (86) - : :- CometSort (83) - : : +- CometColumnarExchange (82) - : : +- CometProject (81) - : : +- CometBroadcastHashJoin (80) - : : :- CometProject (76) - : : : +- CometBroadcastHashJoin (75) - : : : :- CometFilter (73) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (72) - : : : +- ReusedExchange (74) - : : +- CometBroadcastExchange (79) - : : +- CometFilter (78) - : : +- CometScan parquet spark_catalog.default.date_dim (77) - : +- CometSort (85) - : +- ReusedExchange (84) - :- CometProject (101) - : +- CometSortMergeJoin (100) - : :- CometSort (97) - : : +- CometColumnarExchange (96) - : : +- CometProject (95) - : : +- CometBroadcastHashJoin (94) - : : :- CometProject (92) - : : : +- CometBroadcastHashJoin (91) - : : : :- CometFilter (89) - : : : : +- CometScan parquet spark_catalog.default.store_sales (88) - : : : +- ReusedExchange (90) - : : +- ReusedExchange (93) - : +- CometSort (99) - : +- ReusedExchange (98) - +- CometProject (115) - +- CometSortMergeJoin (114) - :- CometSort (111) - : +- CometColumnarExchange (110) - : +- CometProject (109) - : +- CometBroadcastHashJoin (108) - : :- CometProject (106) - : : +- CometBroadcastHashJoin (105) - : : :- CometFilter (103) - : : : +- CometScan parquet spark_catalog.default.web_sales (102) - : : +- ReusedExchange (104) - : +- ReusedExchange (107) - +- CometSort (113) - +- ReusedExchange (112) +TakeOrderedAndProject (125) ++- * Project (124) + +- * SortMergeJoin Inner (123) + :- * Sort (67) + : +- Exchange (66) + : +- * Filter (65) + : +- * HashAggregate (64) + : +- Exchange (63) + : +- * HashAggregate (62) + : +- * ColumnarToRow (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) + : :- CometProject (22) + : : +- CometSortMergeJoin (21) + : : :- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan parquet spark_catalog.default.date_dim (9) + : : +- CometSort (20) + : : +- CometExchange (19) + : : +- CometProject (18) + : : +- CometFilter (17) + : : +- CometScan parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometScan parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometScan parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometScan parquet spark_catalog.default.web_returns (50) + +- * Sort (122) + +- Exchange (121) + +- * Filter (120) + +- * HashAggregate (119) + +- Exchange (118) + +- * HashAggregate (117) + +- * ColumnarToRow (116) + +- CometHashAggregate (115) + +- CometExchange (114) + +- CometHashAggregate (113) + +- CometUnion (112) + :- CometProject (83) + : +- CometSortMergeJoin (82) + : :- CometSort (79) + : : +- CometExchange (78) + : : +- CometProject (77) + : : +- CometBroadcastHashJoin (76) + : : :- CometProject (72) + : : : +- CometBroadcastHashJoin (71) + : : : :- CometFilter (69) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (68) + : : : +- ReusedExchange (70) + : : +- CometBroadcastExchange (75) + : : +- CometFilter (74) + : : +- CometScan parquet spark_catalog.default.date_dim (73) + : +- CometSort (81) + : +- ReusedExchange (80) + :- CometProject (97) + : +- CometSortMergeJoin (96) + : :- CometSort (93) + : : +- CometExchange (92) + : : +- CometProject (91) + : : +- CometBroadcastHashJoin (90) + : : :- CometProject (88) + : : : +- CometBroadcastHashJoin (87) + : : : :- CometFilter (85) + : : : : +- CometScan parquet spark_catalog.default.store_sales (84) + : : : +- ReusedExchange (86) + : : +- ReusedExchange (89) + : +- CometSort (95) + : +- ReusedExchange (94) + +- CometProject (111) + +- CometSortMergeJoin (110) + :- CometSort (107) + : +- CometExchange (106) + : +- CometProject (105) + : +- CometBroadcastHashJoin (104) + : :- CometProject (102) + : : +- CometBroadcastHashJoin (101) + : : :- CometFilter (99) + : : : +- CometScan parquet spark_catalog.default.web_sales (98) + : : +- ReusedExchange (100) + : +- ReusedExchange (103) + +- CometSort (109) + +- ReusedExchange (108) (1) Scan parquet spark_catalog.default.catalog_sales @@ -198,9 +190,9 @@ Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -(14) CometColumnarExchange +(14) CometExchange Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] @@ -221,9 +213,9 @@ Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -(19) CometColumnarExchange +(19) CometExchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -274,9 +266,9 @@ Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -(31) CometColumnarExchange +(31) CometExchange Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] @@ -297,9 +289,9 @@ Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -(36) CometColumnarExchange +(36) CometExchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -350,9 +342,9 @@ Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -(48) CometColumnarExchange +(48) CometExchange Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] @@ -373,9 +365,9 @@ Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -(53) CometColumnarExchange +(53) CometExchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -400,9 +392,9 @@ Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_ Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] Functions: [] -(59) CometColumnarExchange +(59) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] @@ -419,42 +411,30 @@ Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#2 Aggregate Attributes [2]: [sum#62, sum#63] Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] -(63) RowToColumnar +(63) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(64) CometColumnarExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(65) ColumnarToRow [codegen id : 2] -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] - -(66) HashAggregate [codegen id : 2] +(64) HashAggregate [codegen id : 2] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Aggregate Attributes [2]: [sum(sales_cnt#20)#66, sum(UnscaledValue(sales_amt#21))#67] Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum(sales_cnt#20)#66 AS sales_cnt#68, MakeDecimal(sum(UnscaledValue(sales_amt#21))#67,18,2) AS sales_amt#69] -(67) Filter [codegen id : 2] +(65) Filter [codegen id : 2] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] Condition : isnotnull(sales_cnt#68) -(68) RowToColumnar -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] - -(69) CometColumnarExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(70) CometSort +(66) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69], [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST] +Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(71) ColumnarToRow [codegen id : 3] +(67) Sort [codegen id : 3] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] +Arguments: [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST], false, 0 -(72) Scan parquet spark_catalog.default.catalog_sales +(68) Scan parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74] Batched: true Location: InMemoryFileIndex [] @@ -462,71 +442,71 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#74), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(73) CometFilter +(69) CometFilter Input [5]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74] Condition : isnotnull(cs_item_sk#70) -(74) ReusedExchange [Reuses operator id: 6] +(70) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] -(75) CometBroadcastHashJoin +(71) CometBroadcastHashJoin Left output [5]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74] Right output [5]: [i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Arguments: [cs_item_sk#70], [i_item_sk#76], Inner, BuildRight -(76) CometProject +(72) CometProject Input [10]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Arguments: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80], [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] -(77) Scan parquet spark_catalog.default.date_dim +(73) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#81, d_year#82] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(78) CometFilter +(74) CometFilter Input [2]: [d_date_sk#81, d_year#82] Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) -(79) CometBroadcastExchange +(75) CometBroadcastExchange Input [2]: [d_date_sk#81, d_year#82] Arguments: [d_date_sk#81, d_year#82] -(80) CometBroadcastHashJoin +(76) CometBroadcastHashJoin Left output [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Right output [2]: [d_date_sk#81, d_year#82] Arguments: [cs_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight -(81) CometProject +(77) CometProject Input [11]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_date_sk#81, d_year#82] Arguments: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82], [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] -(82) CometColumnarExchange +(78) CometExchange Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] -Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(83) CometSort +(79) CometSort Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] Arguments: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82], [cs_order_number#71 ASC NULLS FIRST, cs_item_sk#70 ASC NULLS FIRST] -(84) ReusedExchange [Reuses operator id: 19] +(80) ReusedExchange [Reuses operator id: 19] Output [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] -(85) CometSort +(81) CometSort Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] Arguments: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86], [cr_order_number#84 ASC NULLS FIRST, cr_item_sk#83 ASC NULLS FIRST] -(86) CometSortMergeJoin +(82) CometSortMergeJoin Left output [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] Right output [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] Arguments: [cs_order_number#71, cs_item_sk#70], [cr_order_number#84, cr_item_sk#83], LeftOuter -(87) CometProject +(83) CometProject Input [13]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82, cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] Arguments: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88], [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, (cs_quantity#72 - coalesce(cr_return_quantity#85, 0)) AS sales_cnt#87, (cs_ext_sales_price#73 - coalesce(cr_return_amount#86, 0.00)) AS sales_amt#88] -(88) Scan parquet spark_catalog.default.store_sales +(84) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93] Batched: true Location: InMemoryFileIndex [] @@ -534,59 +514,59 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#93), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(89) CometFilter +(85) CometFilter Input [5]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93] Condition : isnotnull(ss_item_sk#89) -(90) ReusedExchange [Reuses operator id: 6] +(86) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] -(91) CometBroadcastHashJoin +(87) CometBroadcastHashJoin Left output [5]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93] Right output [5]: [i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] Arguments: [ss_item_sk#89], [i_item_sk#95], Inner, BuildRight -(92) CometProject +(88) CometProject Input [10]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93, i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] Arguments: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99], [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] -(93) ReusedExchange [Reuses operator id: 79] +(89) ReusedExchange [Reuses operator id: 75] Output [2]: [d_date_sk#100, d_year#101] -(94) CometBroadcastHashJoin +(90) CometBroadcastHashJoin Left output [9]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] Right output [2]: [d_date_sk#100, d_year#101] Arguments: [ss_sold_date_sk#93], [d_date_sk#100], Inner, BuildRight -(95) CometProject +(91) CometProject Input [11]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_date_sk#100, d_year#101] Arguments: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101], [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101] -(96) CometColumnarExchange +(92) CometExchange Input [9]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101] -Arguments: hashpartitioning(ss_ticket_number#90, ss_item_sk#89, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(ss_ticket_number#90, ss_item_sk#89, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(97) CometSort +(93) CometSort Input [9]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101] Arguments: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101], [ss_ticket_number#90 ASC NULLS FIRST, ss_item_sk#89 ASC NULLS FIRST] -(98) ReusedExchange [Reuses operator id: 36] +(94) ReusedExchange [Reuses operator id: 36] Output [4]: [sr_item_sk#102, sr_ticket_number#103, sr_return_quantity#104, sr_return_amt#105] -(99) CometSort +(95) CometSort Input [4]: [sr_item_sk#102, sr_ticket_number#103, sr_return_quantity#104, sr_return_amt#105] Arguments: [sr_item_sk#102, sr_ticket_number#103, sr_return_quantity#104, sr_return_amt#105], [sr_ticket_number#103 ASC NULLS FIRST, sr_item_sk#102 ASC NULLS FIRST] -(100) CometSortMergeJoin +(96) CometSortMergeJoin Left output [9]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101] Right output [4]: [sr_item_sk#102, sr_ticket_number#103, sr_return_quantity#104, sr_return_amt#105] Arguments: [ss_ticket_number#90, ss_item_sk#89], [sr_ticket_number#103, sr_item_sk#102], LeftOuter -(101) CometProject +(97) CometProject Input [13]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101, sr_item_sk#102, sr_ticket_number#103, sr_return_quantity#104, sr_return_amt#105] Arguments: [d_year#101, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, sales_cnt#106, sales_amt#107], [d_year#101, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, (ss_quantity#91 - coalesce(sr_return_quantity#104, 0)) AS sales_cnt#106, (ss_ext_sales_price#92 - coalesce(sr_return_amt#105, 0.00)) AS sales_amt#107] -(102) Scan parquet spark_catalog.default.web_sales +(98) Scan parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112] Batched: true Location: InMemoryFileIndex [] @@ -594,160 +574,148 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#112), dynamicpruningexpression(ws_s PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(103) CometFilter +(99) CometFilter Input [5]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112] Condition : isnotnull(ws_item_sk#108) -(104) ReusedExchange [Reuses operator id: 6] +(100) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#114, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] -(105) CometBroadcastHashJoin +(101) CometBroadcastHashJoin Left output [5]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112] Right output [5]: [i_item_sk#114, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] Arguments: [ws_item_sk#108], [i_item_sk#114], Inner, BuildRight -(106) CometProject +(102) CometProject Input [10]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112, i_item_sk#114, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] Arguments: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118], [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] -(107) ReusedExchange [Reuses operator id: 79] +(103) ReusedExchange [Reuses operator id: 75] Output [2]: [d_date_sk#119, d_year#120] -(108) CometBroadcastHashJoin +(104) CometBroadcastHashJoin Left output [9]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] Right output [2]: [d_date_sk#119, d_year#120] Arguments: [ws_sold_date_sk#112], [d_date_sk#119], Inner, BuildRight -(109) CometProject +(105) CometProject Input [11]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_date_sk#119, d_year#120] Arguments: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120], [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120] -(110) CometColumnarExchange +(106) CometExchange Input [9]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120] -Arguments: hashpartitioning(ws_order_number#109, ws_item_sk#108, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: hashpartitioning(ws_order_number#109, ws_item_sk#108, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(111) CometSort +(107) CometSort Input [9]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120] Arguments: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120], [ws_order_number#109 ASC NULLS FIRST, ws_item_sk#108 ASC NULLS FIRST] -(112) ReusedExchange [Reuses operator id: 53] +(108) ReusedExchange [Reuses operator id: 53] Output [4]: [wr_item_sk#121, wr_order_number#122, wr_return_quantity#123, wr_return_amt#124] -(113) CometSort +(109) CometSort Input [4]: [wr_item_sk#121, wr_order_number#122, wr_return_quantity#123, wr_return_amt#124] Arguments: [wr_item_sk#121, wr_order_number#122, wr_return_quantity#123, wr_return_amt#124], [wr_order_number#122 ASC NULLS FIRST, wr_item_sk#121 ASC NULLS FIRST] -(114) CometSortMergeJoin +(110) CometSortMergeJoin Left output [9]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120] Right output [4]: [wr_item_sk#121, wr_order_number#122, wr_return_quantity#123, wr_return_amt#124] Arguments: [ws_order_number#109, ws_item_sk#108], [wr_order_number#122, wr_item_sk#121], LeftOuter -(115) CometProject +(111) CometProject Input [13]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120, wr_item_sk#121, wr_order_number#122, wr_return_quantity#123, wr_return_amt#124] Arguments: [d_year#120, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, sales_cnt#125, sales_amt#126], [d_year#120, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, (ws_quantity#110 - coalesce(wr_return_quantity#123, 0)) AS sales_cnt#125, (ws_ext_sales_price#111 - coalesce(wr_return_amt#124, 0.00)) AS sales_amt#126] -(116) CometUnion +(112) CometUnion Child 0 Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Child 1 Input [7]: [d_year#101, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, sales_cnt#106, sales_amt#107] Child 2 Input [7]: [d_year#120, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, sales_cnt#125, sales_amt#126] -(117) CometHashAggregate +(113) CometHashAggregate Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Functions: [] -(118) CometColumnarExchange +(114) CometExchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(119) CometHashAggregate +(115) CometHashAggregate Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Functions: [] -(120) ColumnarToRow [codegen id : 4] +(116) ColumnarToRow [codegen id : 4] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] -(121) HashAggregate [codegen id : 4] +(117) HashAggregate [codegen id : 4] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [partial_sum(sales_cnt#87), partial_sum(UnscaledValue(sales_amt#88))] Aggregate Attributes [2]: [sum#127, sum#128] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#129, sum#130] -(122) RowToColumnar +(118) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#129, sum#130] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=14] -(123) CometColumnarExchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#129, sum#130] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] - -(124) ColumnarToRow [codegen id : 5] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#129, sum#130] - -(125) HashAggregate [codegen id : 5] +(119) HashAggregate [codegen id : 5] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#129, sum#130] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [sum(sales_cnt#87), sum(UnscaledValue(sales_amt#88))] Aggregate Attributes [2]: [sum(sales_cnt#87)#66, sum(UnscaledValue(sales_amt#88))#67] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum(sales_cnt#87)#66 AS sales_cnt#131, MakeDecimal(sum(UnscaledValue(sales_amt#88))#67,18,2) AS sales_amt#132] -(126) Filter [codegen id : 5] +(120) Filter [codegen id : 5] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] Condition : isnotnull(sales_cnt#131) -(127) RowToColumnar -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] - -(128) CometColumnarExchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] -Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] - -(129) CometSort +(121) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] -Arguments: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132], [i_brand_id#77 ASC NULLS FIRST, i_class_id#78 ASC NULLS FIRST, i_category_id#79 ASC NULLS FIRST, i_manufact_id#80 ASC NULLS FIRST] +Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=15] -(130) ColumnarToRow [codegen id : 6] +(122) Sort [codegen id : 6] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] +Arguments: [i_brand_id#77 ASC NULLS FIRST, i_class_id#78 ASC NULLS FIRST, i_category_id#79 ASC NULLS FIRST, i_manufact_id#80 ASC NULLS FIRST], false, 0 -(131) SortMergeJoin [codegen id : 7] +(123) SortMergeJoin [codegen id : 7] Left keys [4]: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Right keys [4]: [i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Join type: Inner Join condition: ((cast(sales_cnt#68 as decimal(17,2)) / cast(sales_cnt#131 as decimal(17,2))) < 0.90000000000000000000) -(132) Project [codegen id : 7] +(124) Project [codegen id : 7] Output [10]: [d_year#82 AS prev_year#133, d_year#14 AS year#134, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#131 AS prev_yr_cnt#135, sales_cnt#68 AS curr_yr_cnt#136, (sales_cnt#68 - sales_cnt#131) AS sales_cnt_diff#137, (sales_amt#69 - sales_amt#132) AS sales_amt_diff#138] Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69, d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] -(133) TakeOrderedAndProject +(125) TakeOrderedAndProject Input [10]: [prev_year#133, year#134, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#135, curr_yr_cnt#136, sales_cnt_diff#137, sales_amt_diff#138] Arguments: 100, [sales_cnt_diff#137 ASC NULLS FIRST], [prev_year#133, year#134, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#135, curr_yr_cnt#136, sales_cnt_diff#137, sales_amt_diff#138] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (137) -+- * ColumnarToRow (136) - +- CometFilter (135) - +- CometScan parquet spark_catalog.default.date_dim (134) +BroadcastExchange (129) ++- * ColumnarToRow (128) + +- CometFilter (127) + +- CometScan parquet spark_catalog.default.date_dim (126) -(134) Scan parquet spark_catalog.default.date_dim +(126) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(135) CometFilter +(127) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) -(136) ColumnarToRow [codegen id : 1] +(128) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#13, d_year#14] -(137) BroadcastExchange +(129) BroadcastExchange Input [2]: [d_date_sk#13, d_year#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] @@ -755,33 +723,33 @@ Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#26 IN d Subquery:3 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 -Subquery:4 Hosting operator id = 72 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#75 -BroadcastExchange (141) -+- * ColumnarToRow (140) - +- CometFilter (139) - +- CometScan parquet spark_catalog.default.date_dim (138) +Subquery:4 Hosting operator id = 68 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#75 +BroadcastExchange (133) ++- * ColumnarToRow (132) + +- CometFilter (131) + +- CometScan parquet spark_catalog.default.date_dim (130) -(138) Scan parquet spark_catalog.default.date_dim +(130) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#81, d_year#82] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(139) CometFilter +(131) CometFilter Input [2]: [d_date_sk#81, d_year#82] Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) -(140) ColumnarToRow [codegen id : 1] +(132) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#81, d_year#82] -(141) BroadcastExchange +(133) BroadcastExchange Input [2]: [d_date_sk#81, d_year#82] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] -Subquery:5 Hosting operator id = 88 Hosting Expression = ss_sold_date_sk#93 IN dynamicpruning#75 +Subquery:5 Hosting operator id = 84 Hosting Expression = ss_sold_date_sk#93 IN dynamicpruning#75 -Subquery:6 Hosting operator id = 102 Hosting Expression = ws_sold_date_sk#112 IN dynamicpruning#75 +Subquery:6 Hosting operator id = 98 Hosting Expression = ws_sold_date_sk#112 IN dynamicpruning#75 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt index 3d82352b7..e52b2f7d8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q75/simplified.txt @@ -4,163 +4,155 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt] InputAdapter WholeStageCodegen (3) - ColumnarToRow + Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] InputAdapter - CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - RowToColumnar - WholeStageCodegen (2) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #4 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #7 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometColumnarExchange [cr_order_number,cr_item_sk] #8 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #9 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #10 - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #11 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometColumnarExchange [wr_order_number,wr_item_sk] #12 - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + WholeStageCodegen (2) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + InputAdapter + Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + WholeStageCodegen (1) + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #4 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometBroadcastExchange [d_date_sk,d_year] #7 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometExchange [cr_order_number,cr_item_sk] #8 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #9 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + ReusedExchange [d_date_sk,d_year] #7 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometExchange [sr_ticket_number,sr_item_sk] #10 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #11 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + ReusedExchange [d_date_sk,d_year] #7 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometExchange [wr_order_number,wr_item_sk] #12 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] InputAdapter WholeStageCodegen (6) - ColumnarToRow + Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] InputAdapter - CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 - RowToColumnar - WholeStageCodegen (5) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #16 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #17 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - CometBroadcastExchange [d_date_sk,d_year] #18 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #8 - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #19 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #10 - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #20 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #12 + Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + WholeStageCodegen (5) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + InputAdapter + Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + WholeStageCodegen (4) + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #16 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #17 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + CometBroadcastExchange [d_date_sk,d_year] #18 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #8 + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #19 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + ReusedExchange [d_date_sk,d_year] #18 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #10 + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #20 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + ReusedExchange [d_date_sk,d_year] #18 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/explain.txt index 8aa8b1c8c..088855406 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/explain.txt @@ -1,40 +1,38 @@ == Physical Plan == -TakeOrderedAndProject (36) -+- * HashAggregate (35) - +- * ColumnarToRow (34) - +- CometColumnarExchange (33) - +- RowToColumnar (32) - +- * HashAggregate (31) - +- * ColumnarToRow (30) - +- CometUnion (29) - :- CometProject (12) - : +- CometBroadcastHashJoin (11) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.item (3) - : +- CometBroadcastExchange (10) - : +- CometFilter (9) - : +- CometScan parquet spark_catalog.default.date_dim (8) - :- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.web_sales (13) - : : +- ReusedExchange (15) - : +- ReusedExchange (18) - +- CometProject (28) - +- CometBroadcastHashJoin (27) - :- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometFilter (22) - : : +- CometScan parquet spark_catalog.default.catalog_sales (21) - : +- ReusedExchange (23) - +- ReusedExchange (26) +TakeOrderedAndProject (34) ++- * HashAggregate (33) + +- Exchange (32) + +- * HashAggregate (31) + +- * ColumnarToRow (30) + +- CometUnion (29) + :- CometProject (12) + : +- CometBroadcastHashJoin (11) + : :- CometProject (7) + : : +- CometBroadcastHashJoin (6) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.item (3) + : +- CometBroadcastExchange (10) + : +- CometFilter (9) + : +- CometScan parquet spark_catalog.default.date_dim (8) + :- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.web_sales (13) + : : +- ReusedExchange (15) + : +- ReusedExchange (18) + +- CometProject (28) + +- CometBroadcastHashJoin (27) + :- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometFilter (22) + : : +- CometScan parquet spark_catalog.default.catalog_sales (21) + : +- ReusedExchange (23) + +- ReusedExchange (26) (1) Scan parquet spark_catalog.default.store_sales @@ -184,24 +182,18 @@ Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12)) Aggregate Attributes [2]: [count#37, sum#38] Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -(32) RowToColumnar +(32) Exchange Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] +Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(33) CometColumnarExchange -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(34) ColumnarToRow [codegen id : 2] -Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] - -(35) HashAggregate [codegen id : 2] +(33) HashAggregate [codegen id : 2] Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#39, sum#40] Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] Functions [2]: [count(1), sum(UnscaledValue(ext_sales_price#12))] Aggregate Attributes [2]: [count(1)#41, sum(UnscaledValue(ext_sales_price#12))#42] Results [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count(1)#41 AS sales_cnt#43, MakeDecimal(sum(UnscaledValue(ext_sales_price#12))#42,17,2) AS sales_amt#44] -(36) TakeOrderedAndProject +(34) TakeOrderedAndProject Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] Arguments: 100, [channel#10 ASC NULLS FIRST, col_name#11 ASC NULLS FIRST, d_year#8 ASC NULLS FIRST, d_qoy#9 ASC NULLS FIRST, i_category#6 ASC NULLS FIRST], [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, sales_cnt#43, sales_amt#44] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/simplified.txt index 66700151a..cfcf6a99a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q76/simplified.txt @@ -1,40 +1,38 @@ TakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] WholeStageCodegen (2) HashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum] [count(1),sum(UnscaledValue(ext_sales_price)),sales_cnt,sales_amt,count,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,col_name,d_year,d_qoy,i_category] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] [count,sum,count,sum] - ColumnarToRow - InputAdapter - CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] - CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] - CometBroadcastExchange [i_item_sk,i_category] #2 - CometFilter [i_item_sk,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_category] - CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] - CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] - ReusedExchange [i_item_sk,i_category] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #3 - CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] - CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] - CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] - CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - ReusedExchange [i_item_sk,i_category] #2 - ReusedExchange [d_date_sk,d_year,d_qoy] #3 + InputAdapter + Exchange [channel,col_name,d_year,d_qoy,i_category] #1 + WholeStageCodegen (1) + HashAggregate [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] [count,sum,count,sum] + ColumnarToRow + InputAdapter + CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_category] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_sold_date_sk] + CometBroadcastExchange [i_item_sk,i_category] #2 + CometFilter [i_item_sk,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_category] + CometBroadcastExchange [d_date_sk,d_year,d_qoy] #3 + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometProject [ws_ship_customer_sk,ws_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_category] + CometFilter [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ship_customer_sk,ws_ext_sales_price,ws_sold_date_sk] + ReusedExchange [i_item_sk,i_category] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 + CometProject [cs_ship_addr_sk,cs_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category,d_date_sk,d_year,d_qoy] + CometProject [cs_ship_addr_sk,cs_ext_sales_price,cs_sold_date_sk,i_category] + CometBroadcastHashJoin [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_category] + CometFilter [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_addr_sk,cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + ReusedExchange [i_item_sk,i_category] #2 + ReusedExchange [d_date_sk,d_year,d_qoy] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt index 7acc17e5f..c177fb453 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/explain.txt @@ -1,104 +1,90 @@ == Physical Plan == -TakeOrderedAndProject (100) -+- * HashAggregate (99) - +- * ColumnarToRow (98) - +- CometColumnarExchange (97) - +- RowToColumnar (96) - +- * HashAggregate (95) - +- * Expand (94) - +- Union (93) - :- * Project (36) - : +- * BroadcastHashJoin LeftOuter BuildRight (35) - : :- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometColumnarExchange (17) - : : +- RowToColumnar (16) - : : +- * HashAggregate (15) - : : +- * ColumnarToRow (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) - : +- BroadcastExchange (34) - : +- * HashAggregate (33) - : +- * ColumnarToRow (32) - : +- CometColumnarExchange (31) - : +- RowToColumnar (30) - : +- * HashAggregate (29) - : +- * ColumnarToRow (28) - : +- CometProject (27) - : +- CometBroadcastHashJoin (26) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometFilter (21) - : : : +- CometScan parquet spark_catalog.default.store_returns (20) - : : +- ReusedExchange (22) - : +- ReusedExchange (25) - :- * Project (59) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (58) - : :- BroadcastExchange (47) - : : +- * HashAggregate (46) - : : +- * ColumnarToRow (45) - : : +- CometColumnarExchange (44) - : : +- RowToColumnar (43) - : : +- * HashAggregate (42) - : : +- * ColumnarToRow (41) - : : +- CometProject (40) - : : +- CometBroadcastHashJoin (39) - : : :- CometScan parquet spark_catalog.default.catalog_sales (37) - : : +- ReusedExchange (38) - : +- * HashAggregate (57) - : +- * ColumnarToRow (56) - : +- CometColumnarExchange (55) - : +- RowToColumnar (54) - : +- * HashAggregate (53) - : +- * ColumnarToRow (52) - : +- CometProject (51) - : +- CometBroadcastHashJoin (50) - : :- CometScan parquet spark_catalog.default.catalog_returns (48) - : +- ReusedExchange (49) - +- * Project (92) - +- * BroadcastHashJoin LeftOuter BuildRight (91) - :- * HashAggregate (75) - : +- * ColumnarToRow (74) - : +- CometColumnarExchange (73) - : +- RowToColumnar (72) - : +- * HashAggregate (71) - : +- * ColumnarToRow (70) - : +- CometProject (69) - : +- CometBroadcastHashJoin (68) - : :- CometProject (64) - : : +- CometBroadcastHashJoin (63) - : : :- CometFilter (61) - : : : +- CometScan parquet spark_catalog.default.web_sales (60) - : : +- ReusedExchange (62) - : +- CometBroadcastExchange (67) - : +- CometFilter (66) - : +- CometScan parquet spark_catalog.default.web_page (65) - +- BroadcastExchange (90) - +- * HashAggregate (89) - +- * ColumnarToRow (88) - +- CometColumnarExchange (87) - +- RowToColumnar (86) - +- * HashAggregate (85) - +- * ColumnarToRow (84) - +- CometProject (83) - +- CometBroadcastHashJoin (82) - :- CometProject (80) - : +- CometBroadcastHashJoin (79) - : :- CometFilter (77) - : : +- CometScan parquet spark_catalog.default.web_returns (76) - : +- ReusedExchange (78) - +- ReusedExchange (81) +TakeOrderedAndProject (86) ++- * HashAggregate (85) + +- Exchange (84) + +- * HashAggregate (83) + +- * Expand (82) + +- Union (81) + :- * Project (32) + : +- * BroadcastHashJoin LeftOuter BuildRight (31) + : :- * HashAggregate (17) + : : +- Exchange (16) + : : +- * HashAggregate (15) + : : +- * ColumnarToRow (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.store (9) + : +- BroadcastExchange (30) + : +- * HashAggregate (29) + : +- Exchange (28) + : +- * HashAggregate (27) + : +- * ColumnarToRow (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometScan parquet spark_catalog.default.store_returns (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + :- * Project (51) + : +- * BroadcastNestedLoopJoin Inner BuildLeft (50) + : :- BroadcastExchange (41) + : : +- * HashAggregate (40) + : : +- Exchange (39) + : : +- * HashAggregate (38) + : : +- * ColumnarToRow (37) + : : +- CometProject (36) + : : +- CometBroadcastHashJoin (35) + : : :- CometScan parquet spark_catalog.default.catalog_sales (33) + : : +- ReusedExchange (34) + : +- * HashAggregate (49) + : +- Exchange (48) + : +- * HashAggregate (47) + : +- * ColumnarToRow (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometScan parquet spark_catalog.default.catalog_returns (42) + : +- ReusedExchange (43) + +- * Project (80) + +- * BroadcastHashJoin LeftOuter BuildRight (79) + :- * HashAggregate (65) + : +- Exchange (64) + : +- * HashAggregate (63) + : +- * ColumnarToRow (62) + : +- CometProject (61) + : +- CometBroadcastHashJoin (60) + : :- CometProject (56) + : : +- CometBroadcastHashJoin (55) + : : :- CometFilter (53) + : : : +- CometScan parquet spark_catalog.default.web_sales (52) + : : +- ReusedExchange (54) + : +- CometBroadcastExchange (59) + : +- CometFilter (58) + : +- CometScan parquet spark_catalog.default.web_page (57) + +- BroadcastExchange (78) + +- * HashAggregate (77) + +- Exchange (76) + +- * HashAggregate (75) + +- * ColumnarToRow (74) + +- CometProject (73) + +- CometBroadcastHashJoin (72) + :- CometProject (70) + : +- CometBroadcastHashJoin (69) + : :- CometFilter (67) + : : +- CometScan parquet spark_catalog.default.web_returns (66) + : +- ReusedExchange (68) + +- ReusedExchange (71) (1) Scan parquet spark_catalog.default.store_sales @@ -175,24 +161,18 @@ Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(Un Aggregate Attributes [2]: [sum#9, sum#10] Results [3]: [s_store_sk#8, sum#11, sum#12] -(16) RowToColumnar +(16) Exchange Input [3]: [s_store_sk#8, sum#11, sum#12] +Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [3]: [s_store_sk#8, sum#11, sum#12] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 4] -Input [3]: [s_store_sk#8, sum#11, sum#12] - -(19) HashAggregate [codegen id : 4] +(17) HashAggregate [codegen id : 4] Input [3]: [s_store_sk#8, sum#11, sum#12] Keys [1]: [s_store_sk#8] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#13, sum(UnscaledValue(ss_net_profit#3))#14] Results [3]: [s_store_sk#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#13,17,2) AS sales#15, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#14,17,2) AS profit#16] -(20) Scan parquet spark_catalog.default.store_returns +(18) Scan parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19, sr_returned_date_sk#20] Batched: true Location: InMemoryFileIndex [] @@ -200,180 +180,162 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#20), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(21) CometFilter +(19) CometFilter Input [4]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19, sr_returned_date_sk#20] Condition : isnotnull(sr_store_sk#17) -(22) ReusedExchange [Reuses operator id: 6] +(20) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#22] -(23) CometBroadcastHashJoin +(21) CometBroadcastHashJoin Left output [4]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19, sr_returned_date_sk#20] Right output [1]: [d_date_sk#22] Arguments: [sr_returned_date_sk#20], [d_date_sk#22], Inner, BuildRight -(24) CometProject +(22) CometProject Input [5]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19, sr_returned_date_sk#20, d_date_sk#22] Arguments: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19], [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19] -(25) ReusedExchange [Reuses operator id: 11] +(23) ReusedExchange [Reuses operator id: 11] Output [1]: [s_store_sk#23] -(26) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [3]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19] Right output [1]: [s_store_sk#23] Arguments: [sr_store_sk#17], [s_store_sk#23], Inner, BuildRight -(27) CometProject +(25) CometProject Input [4]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19, s_store_sk#23] Arguments: [sr_return_amt#18, sr_net_loss#19, s_store_sk#23], [sr_return_amt#18, sr_net_loss#19, s_store_sk#23] -(28) ColumnarToRow [codegen id : 2] +(26) ColumnarToRow [codegen id : 2] Input [3]: [sr_return_amt#18, sr_net_loss#19, s_store_sk#23] -(29) HashAggregate [codegen id : 2] +(27) HashAggregate [codegen id : 2] Input [3]: [sr_return_amt#18, sr_net_loss#19, s_store_sk#23] Keys [1]: [s_store_sk#23] Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#18)), partial_sum(UnscaledValue(sr_net_loss#19))] Aggregate Attributes [2]: [sum#24, sum#25] Results [3]: [s_store_sk#23, sum#26, sum#27] -(30) RowToColumnar -Input [3]: [s_store_sk#23, sum#26, sum#27] - -(31) CometColumnarExchange -Input [3]: [s_store_sk#23, sum#26, sum#27] -Arguments: hashpartitioning(s_store_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(32) ColumnarToRow [codegen id : 3] +(28) Exchange Input [3]: [s_store_sk#23, sum#26, sum#27] +Arguments: hashpartitioning(s_store_sk#23, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(33) HashAggregate [codegen id : 3] +(29) HashAggregate [codegen id : 3] Input [3]: [s_store_sk#23, sum#26, sum#27] Keys [1]: [s_store_sk#23] Functions [2]: [sum(UnscaledValue(sr_return_amt#18)), sum(UnscaledValue(sr_net_loss#19))] Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#18))#28, sum(UnscaledValue(sr_net_loss#19))#29] Results [3]: [s_store_sk#23, MakeDecimal(sum(UnscaledValue(sr_return_amt#18))#28,17,2) AS returns#30, MakeDecimal(sum(UnscaledValue(sr_net_loss#19))#29,17,2) AS profit_loss#31] -(34) BroadcastExchange +(30) BroadcastExchange Input [3]: [s_store_sk#23, returns#30, profit_loss#31] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(35) BroadcastHashJoin [codegen id : 4] +(31) BroadcastHashJoin [codegen id : 4] Left keys [1]: [s_store_sk#8] Right keys [1]: [s_store_sk#23] Join type: LeftOuter Join condition: None -(36) Project [codegen id : 4] +(32) Project [codegen id : 4] Output [5]: [sales#15, coalesce(returns#30, 0.00) AS returns#32, (profit#16 - coalesce(profit_loss#31, 0.00)) AS profit#33, store channel AS channel#34, s_store_sk#8 AS id#35] Input [6]: [s_store_sk#8, sales#15, profit#16, s_store_sk#23, returns#30, profit_loss#31] -(37) Scan parquet spark_catalog.default.catalog_sales +(33) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38, cs_sold_date_sk#39] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#39), dynamicpruningexpression(cs_sold_date_sk#39 IN dynamicpruning#40)] ReadSchema: struct -(38) ReusedExchange [Reuses operator id: 6] +(34) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#41] -(39) CometBroadcastHashJoin +(35) CometBroadcastHashJoin Left output [4]: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38, cs_sold_date_sk#39] Right output [1]: [d_date_sk#41] Arguments: [cs_sold_date_sk#39], [d_date_sk#41], Inner, BuildRight -(40) CometProject +(36) CometProject Input [5]: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38, cs_sold_date_sk#39, d_date_sk#41] Arguments: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38], [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38] -(41) ColumnarToRow [codegen id : 5] +(37) ColumnarToRow [codegen id : 5] Input [3]: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38] -(42) HashAggregate [codegen id : 5] +(38) HashAggregate [codegen id : 5] Input [3]: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38] Keys [1]: [cs_call_center_sk#36] Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#37)), partial_sum(UnscaledValue(cs_net_profit#38))] Aggregate Attributes [2]: [sum#42, sum#43] Results [3]: [cs_call_center_sk#36, sum#44, sum#45] -(43) RowToColumnar -Input [3]: [cs_call_center_sk#36, sum#44, sum#45] - -(44) CometColumnarExchange +(39) Exchange Input [3]: [cs_call_center_sk#36, sum#44, sum#45] -Arguments: hashpartitioning(cs_call_center_sk#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_call_center_sk#36, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(45) ColumnarToRow [codegen id : 6] -Input [3]: [cs_call_center_sk#36, sum#44, sum#45] - -(46) HashAggregate [codegen id : 6] +(40) HashAggregate [codegen id : 6] Input [3]: [cs_call_center_sk#36, sum#44, sum#45] Keys [1]: [cs_call_center_sk#36] Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#37)), sum(UnscaledValue(cs_net_profit#38))] Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#37))#46, sum(UnscaledValue(cs_net_profit#38))#47] Results [3]: [cs_call_center_sk#36, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#37))#46,17,2) AS sales#48, MakeDecimal(sum(UnscaledValue(cs_net_profit#38))#47,17,2) AS profit#49] -(47) BroadcastExchange +(41) BroadcastExchange Input [3]: [cs_call_center_sk#36, sales#48, profit#49] Arguments: IdentityBroadcastMode, [plan_id=5] -(48) Scan parquet spark_catalog.default.catalog_returns +(42) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cr_returned_date_sk#52), dynamicpruningexpression(cr_returned_date_sk#52 IN dynamicpruning#53)] ReadSchema: struct -(49) ReusedExchange [Reuses operator id: 6] +(43) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#54] -(50) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [3]: [cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Right output [1]: [d_date_sk#54] Arguments: [cr_returned_date_sk#52], [d_date_sk#54], Inner, BuildRight -(51) CometProject +(45) CometProject Input [4]: [cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52, d_date_sk#54] Arguments: [cr_return_amount#50, cr_net_loss#51], [cr_return_amount#50, cr_net_loss#51] -(52) ColumnarToRow [codegen id : 7] +(46) ColumnarToRow [codegen id : 7] Input [2]: [cr_return_amount#50, cr_net_loss#51] -(53) HashAggregate [codegen id : 7] +(47) HashAggregate [codegen id : 7] Input [2]: [cr_return_amount#50, cr_net_loss#51] Keys: [] Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#50)), partial_sum(UnscaledValue(cr_net_loss#51))] Aggregate Attributes [2]: [sum#55, sum#56] Results [2]: [sum#57, sum#58] -(54) RowToColumnar -Input [2]: [sum#57, sum#58] - -(55) CometColumnarExchange +(48) Exchange Input [2]: [sum#57, sum#58] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(56) ColumnarToRow -Input [2]: [sum#57, sum#58] - -(57) HashAggregate +(49) HashAggregate Input [2]: [sum#57, sum#58] Keys: [] Functions [2]: [sum(UnscaledValue(cr_return_amount#50)), sum(UnscaledValue(cr_net_loss#51))] Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#50))#59, sum(UnscaledValue(cr_net_loss#51))#60] Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#50))#59,17,2) AS returns#61, MakeDecimal(sum(UnscaledValue(cr_net_loss#51))#60,17,2) AS profit_loss#62] -(58) BroadcastNestedLoopJoin [codegen id : 8] +(50) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(59) Project [codegen id : 8] +(51) Project [codegen id : 8] Output [5]: [sales#48, returns#61, (profit#49 - profit_loss#62) AS profit#63, catalog channel AS channel#64, cs_call_center_sk#36 AS id#65] Input [5]: [cs_call_center_sk#36, sales#48, profit#49, returns#61, profit_loss#62] -(60) Scan parquet spark_catalog.default.web_sales +(52) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68, ws_sold_date_sk#69] Batched: true Location: InMemoryFileIndex [] @@ -381,74 +343,68 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#69), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct -(61) CometFilter +(53) CometFilter Input [4]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68, ws_sold_date_sk#69] Condition : isnotnull(ws_web_page_sk#66) -(62) ReusedExchange [Reuses operator id: 6] +(54) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#71] -(63) CometBroadcastHashJoin +(55) CometBroadcastHashJoin Left output [4]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68, ws_sold_date_sk#69] Right output [1]: [d_date_sk#71] Arguments: [ws_sold_date_sk#69], [d_date_sk#71], Inner, BuildRight -(64) CometProject +(56) CometProject Input [5]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68, ws_sold_date_sk#69, d_date_sk#71] Arguments: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68], [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68] -(65) Scan parquet spark_catalog.default.web_page +(57) Scan parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#72] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(66) CometFilter +(58) CometFilter Input [1]: [wp_web_page_sk#72] Condition : isnotnull(wp_web_page_sk#72) -(67) CometBroadcastExchange +(59) CometBroadcastExchange Input [1]: [wp_web_page_sk#72] Arguments: [wp_web_page_sk#72] -(68) CometBroadcastHashJoin +(60) CometBroadcastHashJoin Left output [3]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68] Right output [1]: [wp_web_page_sk#72] Arguments: [ws_web_page_sk#66], [wp_web_page_sk#72], Inner, BuildRight -(69) CometProject +(61) CometProject Input [4]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68, wp_web_page_sk#72] Arguments: [ws_ext_sales_price#67, ws_net_profit#68, wp_web_page_sk#72], [ws_ext_sales_price#67, ws_net_profit#68, wp_web_page_sk#72] -(70) ColumnarToRow [codegen id : 9] +(62) ColumnarToRow [codegen id : 9] Input [3]: [ws_ext_sales_price#67, ws_net_profit#68, wp_web_page_sk#72] -(71) HashAggregate [codegen id : 9] +(63) HashAggregate [codegen id : 9] Input [3]: [ws_ext_sales_price#67, ws_net_profit#68, wp_web_page_sk#72] Keys [1]: [wp_web_page_sk#72] Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#67)), partial_sum(UnscaledValue(ws_net_profit#68))] Aggregate Attributes [2]: [sum#73, sum#74] Results [3]: [wp_web_page_sk#72, sum#75, sum#76] -(72) RowToColumnar -Input [3]: [wp_web_page_sk#72, sum#75, sum#76] - -(73) CometColumnarExchange +(64) Exchange Input [3]: [wp_web_page_sk#72, sum#75, sum#76] -Arguments: hashpartitioning(wp_web_page_sk#72, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(wp_web_page_sk#72, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(74) ColumnarToRow [codegen id : 12] -Input [3]: [wp_web_page_sk#72, sum#75, sum#76] - -(75) HashAggregate [codegen id : 12] +(65) HashAggregate [codegen id : 12] Input [3]: [wp_web_page_sk#72, sum#75, sum#76] Keys [1]: [wp_web_page_sk#72] Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#67)), sum(UnscaledValue(ws_net_profit#68))] Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#67))#77, sum(UnscaledValue(ws_net_profit#68))#78] Results [3]: [wp_web_page_sk#72, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#67))#77,17,2) AS sales#79, MakeDecimal(sum(UnscaledValue(ws_net_profit#68))#78,17,2) AS profit#80] -(76) Scan parquet spark_catalog.default.web_returns +(66) Scan parquet spark_catalog.default.web_returns Output [4]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83, wr_returned_date_sk#84] Batched: true Location: InMemoryFileIndex [] @@ -456,149 +412,137 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#84), dynamicpruningexpression(w PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct -(77) CometFilter +(67) CometFilter Input [4]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83, wr_returned_date_sk#84] Condition : isnotnull(wr_web_page_sk#81) -(78) ReusedExchange [Reuses operator id: 6] +(68) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#86] -(79) CometBroadcastHashJoin +(69) CometBroadcastHashJoin Left output [4]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83, wr_returned_date_sk#84] Right output [1]: [d_date_sk#86] Arguments: [wr_returned_date_sk#84], [d_date_sk#86], Inner, BuildRight -(80) CometProject +(70) CometProject Input [5]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83, wr_returned_date_sk#84, d_date_sk#86] Arguments: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83], [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83] -(81) ReusedExchange [Reuses operator id: 67] +(71) ReusedExchange [Reuses operator id: 59] Output [1]: [wp_web_page_sk#87] -(82) CometBroadcastHashJoin +(72) CometBroadcastHashJoin Left output [3]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83] Right output [1]: [wp_web_page_sk#87] Arguments: [wr_web_page_sk#81], [wp_web_page_sk#87], Inner, BuildRight -(83) CometProject +(73) CometProject Input [4]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83, wp_web_page_sk#87] Arguments: [wr_return_amt#82, wr_net_loss#83, wp_web_page_sk#87], [wr_return_amt#82, wr_net_loss#83, wp_web_page_sk#87] -(84) ColumnarToRow [codegen id : 10] +(74) ColumnarToRow [codegen id : 10] Input [3]: [wr_return_amt#82, wr_net_loss#83, wp_web_page_sk#87] -(85) HashAggregate [codegen id : 10] +(75) HashAggregate [codegen id : 10] Input [3]: [wr_return_amt#82, wr_net_loss#83, wp_web_page_sk#87] Keys [1]: [wp_web_page_sk#87] Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#82)), partial_sum(UnscaledValue(wr_net_loss#83))] Aggregate Attributes [2]: [sum#88, sum#89] Results [3]: [wp_web_page_sk#87, sum#90, sum#91] -(86) RowToColumnar -Input [3]: [wp_web_page_sk#87, sum#90, sum#91] - -(87) CometColumnarExchange -Input [3]: [wp_web_page_sk#87, sum#90, sum#91] -Arguments: hashpartitioning(wp_web_page_sk#87, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(88) ColumnarToRow [codegen id : 11] +(76) Exchange Input [3]: [wp_web_page_sk#87, sum#90, sum#91] +Arguments: hashpartitioning(wp_web_page_sk#87, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(89) HashAggregate [codegen id : 11] +(77) HashAggregate [codegen id : 11] Input [3]: [wp_web_page_sk#87, sum#90, sum#91] Keys [1]: [wp_web_page_sk#87] Functions [2]: [sum(UnscaledValue(wr_return_amt#82)), sum(UnscaledValue(wr_net_loss#83))] Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#82))#92, sum(UnscaledValue(wr_net_loss#83))#93] Results [3]: [wp_web_page_sk#87, MakeDecimal(sum(UnscaledValue(wr_return_amt#82))#92,17,2) AS returns#94, MakeDecimal(sum(UnscaledValue(wr_net_loss#83))#93,17,2) AS profit_loss#95] -(90) BroadcastExchange +(78) BroadcastExchange Input [3]: [wp_web_page_sk#87, returns#94, profit_loss#95] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] -(91) BroadcastHashJoin [codegen id : 12] +(79) BroadcastHashJoin [codegen id : 12] Left keys [1]: [wp_web_page_sk#72] Right keys [1]: [wp_web_page_sk#87] Join type: LeftOuter Join condition: None -(92) Project [codegen id : 12] +(80) Project [codegen id : 12] Output [5]: [sales#79, coalesce(returns#94, 0.00) AS returns#96, (profit#80 - coalesce(profit_loss#95, 0.00)) AS profit#97, web channel AS channel#98, wp_web_page_sk#72 AS id#99] Input [6]: [wp_web_page_sk#72, sales#79, profit#80, wp_web_page_sk#87, returns#94, profit_loss#95] -(93) Union +(81) Union -(94) Expand [codegen id : 13] +(82) Expand [codegen id : 13] Input [5]: [sales#15, returns#32, profit#33, channel#34, id#35] Arguments: [[sales#15, returns#32, profit#33, channel#34, id#35, 0], [sales#15, returns#32, profit#33, channel#34, null, 1], [sales#15, returns#32, profit#33, null, null, 3]], [sales#15, returns#32, profit#33, channel#100, id#101, spark_grouping_id#102] -(95) HashAggregate [codegen id : 13] +(83) HashAggregate [codegen id : 13] Input [6]: [sales#15, returns#32, profit#33, channel#100, id#101, spark_grouping_id#102] Keys [3]: [channel#100, id#101, spark_grouping_id#102] Functions [3]: [partial_sum(sales#15), partial_sum(returns#32), partial_sum(profit#33)] Aggregate Attributes [6]: [sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] Results [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -(96) RowToColumnar -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] - -(97) CometColumnarExchange -Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] -Arguments: hashpartitioning(channel#100, id#101, spark_grouping_id#102, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] - -(98) ColumnarToRow [codegen id : 14] +(84) Exchange Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] +Arguments: hashpartitioning(channel#100, id#101, spark_grouping_id#102, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(99) HashAggregate [codegen id : 14] +(85) HashAggregate [codegen id : 14] Input [9]: [channel#100, id#101, spark_grouping_id#102, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114] Keys [3]: [channel#100, id#101, spark_grouping_id#102] Functions [3]: [sum(sales#15), sum(returns#32), sum(profit#33)] Aggregate Attributes [3]: [sum(sales#15)#115, sum(returns#32)#116, sum(profit#33)#117] Results [5]: [channel#100, id#101, sum(sales#15)#115 AS sales#118, sum(returns#32)#116 AS returns#119, sum(profit#33)#117 AS profit#120] -(100) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [5]: [channel#100, id#101, sales#118, returns#119, profit#120] Arguments: 100, [channel#100 ASC NULLS FIRST, id#101 ASC NULLS FIRST], [channel#100, id#101, sales#118, returns#119, profit#120] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (105) -+- * ColumnarToRow (104) - +- CometProject (103) - +- CometFilter (102) - +- CometScan parquet spark_catalog.default.date_dim (101) +BroadcastExchange (91) ++- * ColumnarToRow (90) + +- CometProject (89) + +- CometFilter (88) + +- CometScan parquet spark_catalog.default.date_dim (87) -(101) Scan parquet spark_catalog.default.date_dim +(87) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_date#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] ReadSchema: struct -(102) CometFilter +(88) CometFilter Input [2]: [d_date_sk#6, d_date#7] Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6)) -(103) CometProject +(89) CometProject Input [2]: [d_date_sk#6, d_date#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(104) ColumnarToRow [codegen id : 1] +(90) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(105) BroadcastExchange +(91) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] -Subquery:2 Hosting operator id = 20 Hosting Expression = sr_returned_date_sk#20 IN dynamicpruning#5 +Subquery:2 Hosting operator id = 18 Hosting Expression = sr_returned_date_sk#20 IN dynamicpruning#5 -Subquery:3 Hosting operator id = 37 Hosting Expression = cs_sold_date_sk#39 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#39 IN dynamicpruning#5 -Subquery:4 Hosting operator id = 48 Hosting Expression = cr_returned_date_sk#52 IN dynamicpruning#5 +Subquery:4 Hosting operator id = 42 Hosting Expression = cr_returned_date_sk#52 IN dynamicpruning#5 -Subquery:5 Hosting operator id = 60 Hosting Expression = ws_sold_date_sk#69 IN dynamicpruning#5 +Subquery:5 Hosting operator id = 52 Hosting Expression = ws_sold_date_sk#69 IN dynamicpruning#5 -Subquery:6 Hosting operator id = 76 Hosting Expression = wr_returned_date_sk#84 IN dynamicpruning#5 +Subquery:6 Hosting operator id = 66 Hosting Expression = wr_returned_date_sk#84 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt index b5b538743..91786374e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q77/simplified.txt @@ -1,144 +1,130 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (14) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (13) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (4) - Project [sales,returns,profit,profit_loss,s_store_sk] - BroadcastHashJoin [s_store_sk,s_store_sk] - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] - ColumnarToRow + InputAdapter + Exchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (13) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (4) + Project [sales,returns,profit,profit_loss,s_store_sk] + BroadcastHashJoin [s_store_sk,s_store_sk] + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] + InputAdapter + Exchange [s_store_sk] #2 + WholeStageCodegen (1) + HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometScan parquet spark_catalog.default.store [s_store_sk] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] InputAdapter - CometColumnarExchange [s_store_sk] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_sk] #7 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_return_amt,sr_net_loss,s_store_sk] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] - CometProject [sr_store_sk,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [s_store_sk] #5 - WholeStageCodegen (8) - Project [sales,returns,profit,profit_loss,cs_call_center_sk] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (6) - HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [cs_call_center_sk] #9 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] - ColumnarToRow + Exchange [s_store_sk] #7 + WholeStageCodegen (2) + HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 + WholeStageCodegen (8) + Project [sales,returns,profit,profit_loss,cs_call_center_sk] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (6) + HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] InputAdapter - CometColumnarExchange #10 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - WholeStageCodegen (12) - Project [sales,returns,profit,profit_loss,wp_web_page_sk] - BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] - ColumnarToRow + Exchange [cs_call_center_sk] #9 + WholeStageCodegen (5) + HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] + InputAdapter + Exchange #10 + WholeStageCodegen (7) + HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (12) + Project [sales,returns,profit,profit_loss,wp_web_page_sk] + BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] + InputAdapter + Exchange [wp_web_page_sk] #11 + WholeStageCodegen (9) + HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [wp_web_page_sk] #12 + CometFilter [wp_web_page_sk] + CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] + InputAdapter + BroadcastExchange #13 + WholeStageCodegen (11) + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] InputAdapter - CometColumnarExchange [wp_web_page_sk] #11 - RowToColumnar - WholeStageCodegen (9) - HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] - CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [wp_web_page_sk] #12 - CometFilter [wp_web_page_sk] - CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] - InputAdapter - BroadcastExchange #13 - WholeStageCodegen (11) - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [wp_web_page_sk] #14 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] - CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] - CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [wp_web_page_sk] #12 + Exchange [wp_web_page_sk] #14 + WholeStageCodegen (10) + HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt index f2886073f..415dfb371 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/explain.txt @@ -1,79 +1,73 @@ == Physical Plan == -TakeOrderedAndProject (75) -+- * Project (74) - +- * SortMergeJoin Inner (73) - :- * Project (49) - : +- * SortMergeJoin Inner (48) - : :- * Sort (24) - : : +- * HashAggregate (23) - : : +- * ColumnarToRow (22) - : : +- CometColumnarExchange (21) - : : +- RowToColumnar (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometFilter (11) - : : : +- CometSortMergeJoin (10) - : : : :- CometSort (4) - : : : : +- CometColumnarExchange (3) - : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometSort (9) - : : : +- CometColumnarExchange (8) - : : : +- CometProject (7) - : : : +- CometFilter (6) - : : : +- CometScan parquet spark_catalog.default.store_returns (5) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.date_dim (13) - : +- * Sort (47) - : +- * Filter (46) - : +- * HashAggregate (45) - : +- * ColumnarToRow (44) - : +- CometColumnarExchange (43) - : +- RowToColumnar (42) - : +- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (36) - : : +- CometFilter (35) - : : +- CometSortMergeJoin (34) - : : :- CometSort (28) - : : : +- CometColumnarExchange (27) - : : : +- CometFilter (26) - : : : +- CometScan parquet spark_catalog.default.web_sales (25) - : : +- CometSort (33) - : : +- CometColumnarExchange (32) - : : +- CometProject (31) - : : +- CometFilter (30) - : : +- CometScan parquet spark_catalog.default.web_returns (29) - : +- ReusedExchange (37) - +- * Sort (72) - +- * Filter (71) - +- * HashAggregate (70) - +- * ColumnarToRow (69) - +- CometColumnarExchange (68) - +- RowToColumnar (67) - +- * HashAggregate (66) - +- * ColumnarToRow (65) - +- CometProject (64) - +- CometBroadcastHashJoin (63) - :- CometProject (61) - : +- CometFilter (60) - : +- CometSortMergeJoin (59) - : :- CometSort (53) - : : +- CometColumnarExchange (52) - : : +- CometFilter (51) - : : +- CometScan parquet spark_catalog.default.catalog_sales (50) - : +- CometSort (58) - : +- CometColumnarExchange (57) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometScan parquet spark_catalog.default.catalog_returns (54) - +- ReusedExchange (62) +TakeOrderedAndProject (69) ++- * Project (68) + +- * SortMergeJoin Inner (67) + :- * Project (45) + : +- * SortMergeJoin Inner (44) + : :- * Sort (22) + : : +- * HashAggregate (21) + : : +- Exchange (20) + : : +- * HashAggregate (19) + : : +- * ColumnarToRow (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometFilter (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometScan parquet spark_catalog.default.store_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.date_dim (13) + : +- * Sort (43) + : +- * Filter (42) + : +- * HashAggregate (41) + : +- Exchange (40) + : +- * HashAggregate (39) + : +- * ColumnarToRow (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (34) + : : +- CometFilter (33) + : : +- CometSortMergeJoin (32) + : : :- CometSort (26) + : : : +- CometExchange (25) + : : : +- CometFilter (24) + : : : +- CometScan parquet spark_catalog.default.web_sales (23) + : : +- CometSort (31) + : : +- CometExchange (30) + : : +- CometProject (29) + : : +- CometFilter (28) + : : +- CometScan parquet spark_catalog.default.web_returns (27) + : +- ReusedExchange (35) + +- * Sort (66) + +- * Filter (65) + +- * HashAggregate (64) + +- Exchange (63) + +- * HashAggregate (62) + +- * ColumnarToRow (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometFilter (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometFilter (47) + : : +- CometScan parquet spark_catalog.default.catalog_sales (46) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometScan parquet spark_catalog.default.catalog_returns (50) + +- ReusedExchange (58) (1) Scan parquet spark_catalog.default.store_sales @@ -88,9 +82,9 @@ ReadSchema: struct -(26) CometFilter +(24) CometFilter Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] Condition : (isnotnull(ws_item_sk#27) AND isnotnull(ws_bill_customer_sk#28)) -(27) CometColumnarExchange +(25) CometExchange Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] -Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(28) CometSort +(26) CometSort Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] Arguments: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33], [ws_order_number#29 ASC NULLS FIRST, ws_item_sk#27 ASC NULLS FIRST] -(29) Scan parquet spark_catalog.default.web_returns +(27) Scan parquet spark_catalog.default.web_returns Output [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(30) CometFilter +(28) CometFilter Input [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] Condition : (isnotnull(wr_order_number#36) AND isnotnull(wr_item_sk#35)) -(31) CometProject +(29) CometProject Input [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] Arguments: [wr_item_sk#35, wr_order_number#36], [wr_item_sk#35, wr_order_number#36] -(32) CometColumnarExchange +(30) CometExchange Input [2]: [wr_item_sk#35, wr_order_number#36] -Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(33) CometSort +(31) CometSort Input [2]: [wr_item_sk#35, wr_order_number#36] Arguments: [wr_item_sk#35, wr_order_number#36], [wr_order_number#36 ASC NULLS FIRST, wr_item_sk#35 ASC NULLS FIRST] -(34) CometSortMergeJoin +(32) CometSortMergeJoin Left output [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] Right output [2]: [wr_item_sk#35, wr_order_number#36] Arguments: [ws_order_number#29, ws_item_sk#27], [wr_order_number#36, wr_item_sk#35], LeftOuter -(35) CometFilter +(33) CometFilter Input [9]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33, wr_item_sk#35, wr_order_number#36] Condition : isnull(wr_order_number#36) -(36) CometProject +(34) CometProject Input [9]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33, wr_item_sk#35, wr_order_number#36] Arguments: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33], [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] -(37) ReusedExchange [Reuses operator id: 15] +(35) ReusedExchange [Reuses operator id: 15] Output [2]: [d_date_sk#38, d_year#39] -(38) CometBroadcastHashJoin +(36) CometBroadcastHashJoin Left output [6]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] Right output [2]: [d_date_sk#38, d_year#39] Arguments: [ws_sold_date_sk#33], [d_date_sk#38], Inner, BuildRight -(39) CometProject +(37) CometProject Input [8]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33, d_date_sk#38, d_year#39] Arguments: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, d_year#39], [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, d_year#39] -(40) ColumnarToRow [codegen id : 3] +(38) ColumnarToRow [codegen id : 3] Input [6]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, d_year#39] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [6]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, d_year#39] Keys [3]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28] Functions [3]: [partial_sum(ws_quantity#30), partial_sum(UnscaledValue(ws_wholesale_cost#31)), partial_sum(UnscaledValue(ws_sales_price#32))] Aggregate Attributes [3]: [sum#40, sum#41, sum#42] Results [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] -(42) RowToColumnar -Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] - -(43) CometColumnarExchange -Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] -Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] +Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] Keys [3]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28] Functions [3]: [sum(ws_quantity#30), sum(UnscaledValue(ws_wholesale_cost#31)), sum(UnscaledValue(ws_sales_price#32))] Aggregate Attributes [3]: [sum(ws_quantity#30)#46, sum(UnscaledValue(ws_wholesale_cost#31))#47, sum(UnscaledValue(ws_sales_price#32))#48] Results [6]: [d_year#39 AS ws_sold_year#49, ws_item_sk#27, ws_bill_customer_sk#28 AS ws_customer_sk#50, sum(ws_quantity#30)#46 AS ws_qty#51, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#31))#47,17,2) AS ws_wc#52, MakeDecimal(sum(UnscaledValue(ws_sales_price#32))#48,17,2) AS ws_sp#53] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [6]: [ws_sold_year#49, ws_item_sk#27, ws_customer_sk#50, ws_qty#51, ws_wc#52, ws_sp#53] Condition : (coalesce(ws_qty#51, 0) > 0) -(47) Sort [codegen id : 4] +(43) Sort [codegen id : 4] Input [6]: [ws_sold_year#49, ws_item_sk#27, ws_customer_sk#50, ws_qty#51, ws_wc#52, ws_sp#53] Arguments: [ws_sold_year#49 ASC NULLS FIRST, ws_item_sk#27 ASC NULLS FIRST, ws_customer_sk#50 ASC NULLS FIRST], false, 0 -(48) SortMergeJoin [codegen id : 5] +(44) SortMergeJoin [codegen id : 5] Left keys [3]: [ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2] Right keys [3]: [ws_sold_year#49, ws_item_sk#27, ws_customer_sk#50] Join type: Inner Join condition: None -(49) Project [codegen id : 5] +(45) Project [codegen id : 5] Output [9]: [ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2, ss_qty#24, ss_wc#25, ss_sp#26, ws_qty#51, ws_wc#52, ws_sp#53] Input [12]: [ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2, ss_qty#24, ss_wc#25, ss_sp#26, ws_sold_year#49, ws_item_sk#27, ws_customer_sk#50, ws_qty#51, ws_wc#52, ws_sp#53] -(50) Scan parquet spark_catalog.default.catalog_sales +(46) Scan parquet spark_catalog.default.catalog_sales Output [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] Batched: true Location: InMemoryFileIndex [] @@ -308,144 +290,138 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#60), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(51) CometFilter +(47) CometFilter Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] Condition : (isnotnull(cs_item_sk#55) AND isnotnull(cs_bill_customer_sk#54)) -(52) CometColumnarExchange +(48) CometExchange Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] -Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(53) CometSort +(49) CometSort Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] Arguments: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60], [cs_order_number#56 ASC NULLS FIRST, cs_item_sk#55 ASC NULLS FIRST] -(54) Scan parquet spark_catalog.default.catalog_returns +(50) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(55) CometFilter +(51) CometFilter Input [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] Condition : (isnotnull(cr_order_number#63) AND isnotnull(cr_item_sk#62)) -(56) CometProject +(52) CometProject Input [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] Arguments: [cr_item_sk#62, cr_order_number#63], [cr_item_sk#62, cr_order_number#63] -(57) CometColumnarExchange +(53) CometExchange Input [2]: [cr_item_sk#62, cr_order_number#63] -Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(58) CometSort +(54) CometSort Input [2]: [cr_item_sk#62, cr_order_number#63] Arguments: [cr_item_sk#62, cr_order_number#63], [cr_order_number#63 ASC NULLS FIRST, cr_item_sk#62 ASC NULLS FIRST] -(59) CometSortMergeJoin +(55) CometSortMergeJoin Left output [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] Right output [2]: [cr_item_sk#62, cr_order_number#63] Arguments: [cs_order_number#56, cs_item_sk#55], [cr_order_number#63, cr_item_sk#62], LeftOuter -(60) CometFilter +(56) CometFilter Input [9]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60, cr_item_sk#62, cr_order_number#63] Condition : isnull(cr_order_number#63) -(61) CometProject +(57) CometProject Input [9]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60, cr_item_sk#62, cr_order_number#63] Arguments: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60], [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] -(62) ReusedExchange [Reuses operator id: 15] +(58) ReusedExchange [Reuses operator id: 15] Output [2]: [d_date_sk#65, d_year#66] -(63) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [6]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] Right output [2]: [d_date_sk#65, d_year#66] Arguments: [cs_sold_date_sk#60], [d_date_sk#65], Inner, BuildRight -(64) CometProject +(60) CometProject Input [8]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60, d_date_sk#65, d_year#66] Arguments: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, d_year#66], [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, d_year#66] -(65) ColumnarToRow [codegen id : 6] +(61) ColumnarToRow [codegen id : 6] Input [6]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, d_year#66] -(66) HashAggregate [codegen id : 6] +(62) HashAggregate [codegen id : 6] Input [6]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, d_year#66] Keys [3]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54] Functions [3]: [partial_sum(cs_quantity#57), partial_sum(UnscaledValue(cs_wholesale_cost#58)), partial_sum(UnscaledValue(cs_sales_price#59))] Aggregate Attributes [3]: [sum#67, sum#68, sum#69] Results [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] -(67) RowToColumnar -Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] - -(68) CometColumnarExchange -Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(69) ColumnarToRow [codegen id : 7] +(63) Exchange Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] +Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(70) HashAggregate [codegen id : 7] +(64) HashAggregate [codegen id : 7] Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] Keys [3]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54] Functions [3]: [sum(cs_quantity#57), sum(UnscaledValue(cs_wholesale_cost#58)), sum(UnscaledValue(cs_sales_price#59))] Aggregate Attributes [3]: [sum(cs_quantity#57)#73, sum(UnscaledValue(cs_wholesale_cost#58))#74, sum(UnscaledValue(cs_sales_price#59))#75] Results [6]: [d_year#66 AS cs_sold_year#76, cs_item_sk#55, cs_bill_customer_sk#54 AS cs_customer_sk#77, sum(cs_quantity#57)#73 AS cs_qty#78, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#58))#74,17,2) AS cs_wc#79, MakeDecimal(sum(UnscaledValue(cs_sales_price#59))#75,17,2) AS cs_sp#80] -(71) Filter [codegen id : 7] +(65) Filter [codegen id : 7] Input [6]: [cs_sold_year#76, cs_item_sk#55, cs_customer_sk#77, cs_qty#78, cs_wc#79, cs_sp#80] Condition : (coalesce(cs_qty#78, 0) > 0) -(72) Sort [codegen id : 7] +(66) Sort [codegen id : 7] Input [6]: [cs_sold_year#76, cs_item_sk#55, cs_customer_sk#77, cs_qty#78, cs_wc#79, cs_sp#80] Arguments: [cs_sold_year#76 ASC NULLS FIRST, cs_item_sk#55 ASC NULLS FIRST, cs_customer_sk#77 ASC NULLS FIRST], false, 0 -(73) SortMergeJoin [codegen id : 8] +(67) SortMergeJoin [codegen id : 8] Left keys [3]: [ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2] Right keys [3]: [cs_sold_year#76, cs_item_sk#55, cs_customer_sk#77] Join type: Inner Join condition: None -(74) Project [codegen id : 8] +(68) Project [codegen id : 8] Output [12]: [round((cast(ss_qty#24 as double) / cast(coalesce((ws_qty#51 + cs_qty#78), 1) as double)), 2) AS ratio#81, ss_qty#24 AS store_qty#82, ss_wc#25 AS store_wholesale_cost#83, ss_sp#26 AS store_sales_price#84, (coalesce(ws_qty#51, 0) + coalesce(cs_qty#78, 0)) AS other_chan_qty#85, (coalesce(ws_wc#52, 0.00) + coalesce(cs_wc#79, 0.00)) AS other_chan_wholesale_cost#86, (coalesce(ws_sp#53, 0.00) + coalesce(cs_sp#80, 0.00)) AS other_chan_sales_price#87, ss_qty#24, ss_wc#25, ss_sp#26, ws_qty#51, cs_qty#78] Input [15]: [ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2, ss_qty#24, ss_wc#25, ss_sp#26, ws_qty#51, ws_wc#52, ws_sp#53, cs_sold_year#76, cs_item_sk#55, cs_customer_sk#77, cs_qty#78, cs_wc#79, cs_sp#80] -(75) TakeOrderedAndProject +(69) TakeOrderedAndProject Input [12]: [ratio#81, store_qty#82, store_wholesale_cost#83, store_sales_price#84, other_chan_qty#85, other_chan_wholesale_cost#86, other_chan_sales_price#87, ss_qty#24, ss_wc#25, ss_sp#26, ws_qty#51, cs_qty#78] Arguments: 100, [ratio#81 ASC NULLS FIRST, ss_qty#24 DESC NULLS LAST, ss_wc#25 DESC NULLS LAST, ss_sp#26 DESC NULLS LAST, other_chan_qty#85 ASC NULLS FIRST, other_chan_wholesale_cost#86 ASC NULLS FIRST, other_chan_sales_price#87 ASC NULLS FIRST, round((cast(ss_qty#24 as double) / cast(coalesce((ws_qty#51 + cs_qty#78), 1) as double)), 2) ASC NULLS FIRST], [ratio#81, store_qty#82, store_wholesale_cost#83, store_sales_price#84, other_chan_qty#85, other_chan_wholesale_cost#86, other_chan_sales_price#87] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (79) -+- * ColumnarToRow (78) - +- CometFilter (77) - +- CometScan parquet spark_catalog.default.date_dim (76) +BroadcastExchange (73) ++- * ColumnarToRow (72) + +- CometFilter (71) + +- CometScan parquet spark_catalog.default.date_dim (70) -(76) Scan parquet spark_catalog.default.date_dim +(70) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(77) CometFilter +(71) CometFilter Input [2]: [d_date_sk#12, d_year#13] Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) -(78) ColumnarToRow [codegen id : 1] +(72) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#12, d_year#13] -(79) BroadcastExchange +(73) BroadcastExchange Input [2]: [d_date_sk#12, d_year#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] -Subquery:2 Hosting operator id = 25 Hosting Expression = ws_sold_date_sk#33 IN dynamicpruning#8 +Subquery:2 Hosting operator id = 23 Hosting Expression = ws_sold_date_sk#33 IN dynamicpruning#8 -Subquery:3 Hosting operator id = 50 Hosting Expression = cs_sold_date_sk#60 IN dynamicpruning#8 +Subquery:3 Hosting operator id = 46 Hosting Expression = cs_sold_date_sk#60 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt index 1475eef5a..48eeb00d6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q78/simplified.txt @@ -10,93 +10,87 @@ TakeOrderedAndProject [ratio,ss_qty,ss_wc,ss_sp,other_chan_qty,other_chan_wholes WholeStageCodegen (2) Sort [ss_sold_year,ss_item_sk,ss_customer_sk] HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,ss_item_sk,ss_customer_sk] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #2 - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #5 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + Exchange [d_year,ss_item_sk,ss_customer_sk] #1 + WholeStageCodegen (1) + HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometExchange [ss_ticket_number,ss_item_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #5 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter WholeStageCodegen (4) Sort [ws_sold_year,ws_item_sk,ws_customer_sk] Filter [ws_qty] HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,ws_item_sk,ws_bill_customer_sk] #6 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] - CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] - CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometColumnarExchange [ws_order_number,ws_item_sk] #7 - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [wr_item_sk,wr_order_number] - CometColumnarExchange [wr_order_number,wr_item_sk] #8 - CometProject [wr_item_sk,wr_order_number] - CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + InputAdapter + Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #6 + WholeStageCodegen (3) + HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #7 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #8 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #5 InputAdapter WholeStageCodegen (7) Sort [cs_sold_year,cs_item_sk,cs_customer_sk] Filter [cs_qty] HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,cs_item_sk,cs_bill_customer_sk] #9 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometColumnarExchange [cs_order_number,cs_item_sk] #10 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [cr_item_sk,cr_order_number] - CometColumnarExchange [cr_order_number,cr_item_sk] #11 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + InputAdapter + Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #9 + WholeStageCodegen (6) + HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #10 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_order_number,cr_item_sk] #11 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt index 946c96906..810eaa6e2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/explain.txt @@ -1,37 +1,35 @@ == Physical Plan == -TakeOrderedAndProject (33) -+- * Project (32) - +- * BroadcastHashJoin Inner BuildRight (31) - :- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometColumnarExchange (24) - : +- RowToColumnar (23) - : +- * HashAggregate (22) - : +- * ColumnarToRow (21) - : +- CometProject (20) - : +- CometBroadcastHashJoin (19) - : :- CometProject (14) - : : +- CometBroadcastHashJoin (13) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (12) - : : +- CometProject (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) - : +- CometBroadcastExchange (18) - : +- CometProject (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.household_demographics (15) - +- BroadcastExchange (30) - +- * ColumnarToRow (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.customer (27) +TakeOrderedAndProject (31) ++- * Project (30) + +- * BroadcastHashJoin Inner BuildRight (29) + :- * HashAggregate (24) + : +- Exchange (23) + : +- * HashAggregate (22) + : +- * ColumnarToRow (21) + : +- CometProject (20) + : +- CometBroadcastHashJoin (19) + : :- CometProject (14) + : : +- CometBroadcastHashJoin (13) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (12) + : : +- CometProject (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (18) + : +- CometProject (17) + : +- CometFilter (16) + : +- CometScan parquet spark_catalog.default.household_demographics (15) + +- BroadcastExchange (28) + +- * ColumnarToRow (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.customer (25) (1) Scan parquet spark_catalog.default.store_sales @@ -140,84 +138,78 @@ Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(Unscale Aggregate Attributes [2]: [sum#19, sum#20] Results [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#21, sum#22] -(23) RowToColumnar +(23) Exchange Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#21, sum#22] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(24) CometColumnarExchange -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#21, sum#22] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(25) ColumnarToRow [codegen id : 3] -Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#21, sum#22] - -(26) HashAggregate [codegen id : 3] +(24) HashAggregate [codegen id : 3] Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15, sum#21, sum#22] Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, s_city#15] Functions [2]: [sum(UnscaledValue(ss_coupon_amt#6)), sum(UnscaledValue(ss_net_profit#7))] Aggregate Attributes [2]: [sum(UnscaledValue(ss_coupon_amt#6))#23, sum(UnscaledValue(ss_net_profit#7))#24] Results [5]: [ss_ticket_number#5, ss_customer_sk#1, s_city#15, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#6))#23,17,2) AS amt#25, MakeDecimal(sum(UnscaledValue(ss_net_profit#7))#24,17,2) AS profit#26] -(27) Scan parquet spark_catalog.default.customer +(25) Scan parquet spark_catalog.default.customer Output [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Condition : isnotnull(c_customer_sk#27) -(29) ColumnarToRow [codegen id : 2] +(27) ColumnarToRow [codegen id : 2] Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] -(30) BroadcastExchange +(28) BroadcastExchange Input [3]: [c_customer_sk#27, c_first_name#28, c_last_name#29] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] -(31) BroadcastHashJoin [codegen id : 3] +(29) BroadcastHashJoin [codegen id : 3] Left keys [1]: [ss_customer_sk#1] Right keys [1]: [c_customer_sk#27] Join type: Inner Join condition: None -(32) Project [codegen id : 3] +(30) Project [codegen id : 3] Output [7]: [c_last_name#29, c_first_name#28, substr(s_city#15, 1, 30) AS substr(s_city, 1, 30)#30, ss_ticket_number#5, amt#25, profit#26, s_city#15] Input [8]: [ss_ticket_number#5, ss_customer_sk#1, s_city#15, amt#25, profit#26, c_customer_sk#27, c_first_name#28, c_last_name#29] -(33) TakeOrderedAndProject +(31) TakeOrderedAndProject Input [7]: [c_last_name#29, c_first_name#28, substr(s_city, 1, 30)#30, ss_ticket_number#5, amt#25, profit#26, s_city#15] Arguments: 100, [c_last_name#29 ASC NULLS FIRST, c_first_name#28 ASC NULLS FIRST, substr(s_city#15, 1, 30) ASC NULLS FIRST, profit#26 ASC NULLS FIRST], [c_last_name#29, c_first_name#28, substr(s_city, 1, 30)#30, ss_ticket_number#5, amt#25, profit#26] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (38) -+- * ColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan parquet spark_catalog.default.date_dim (34) +BroadcastExchange (36) ++- * ColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan parquet spark_catalog.default.date_dim (32) -(34) Scan parquet spark_catalog.default.date_dim +(32) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_year#11, d_dow#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_dow), EqualTo(d_dow,1), In(d_year, [1999,2000,2001]), IsNotNull(d_date_sk)] ReadSchema: struct -(35) CometFilter +(33) CometFilter Input [3]: [d_date_sk#10, d_year#11, d_dow#12] Condition : (((isnotnull(d_dow#12) AND (d_dow#12 = 1)) AND d_year#11 IN (1999,2000,2001)) AND isnotnull(d_date_sk#10)) -(36) CometProject +(34) CometProject Input [3]: [d_date_sk#10, d_year#11, d_dow#12] Arguments: [d_date_sk#10], [d_date_sk#10] -(37) ColumnarToRow [codegen id : 1] +(35) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#10] -(38) BroadcastExchange +(36) BroadcastExchange Input [1]: [d_date_sk#10] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt index 30a67a19b..f2883e450 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q79/simplified.txt @@ -3,42 +3,40 @@ TakeOrderedAndProject [c_last_name,c_first_name,s_city,profit,substr(s_city, 1, Project [c_last_name,c_first_name,s_city,ss_ticket_number,amt,profit] BroadcastHashJoin [ss_customer_sk,c_customer_sk] HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,sum,sum] [sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit)),amt,profit,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk,s_city] - CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] - CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_dow] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] - CometBroadcastExchange [s_store_sk,s_city] #4 - CometProject [s_store_sk,s_city] - CometFilter [s_store_sk,s_number_employees,s_city] - CometScan parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] - CometBroadcastExchange [hd_demo_sk] #5 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] + InputAdapter + Exchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city] #1 + WholeStageCodegen (1) + HashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,s_city,ss_coupon_amt,ss_net_profit] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city,hd_demo_sk] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_city] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,s_store_sk,s_city] + CometProject [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit] + CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_dow] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_dow] + CometBroadcastExchange [s_store_sk,s_city] #4 + CometProject [s_store_sk,s_city] + CometFilter [s_store_sk,s_number_employees,s_city] + CometScan parquet spark_catalog.default.store [s_store_sk,s_number_employees,s_city] + CometBroadcastExchange [hd_demo_sk] #5 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_dep_count,hd_vehicle_count] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_dep_count,hd_vehicle_count] InputAdapter BroadcastExchange #6 WholeStageCodegen (2) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt index ac20b8b5b..968b25212 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/explain.txt @@ -1,48 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * HashAggregate (43) - +- * ColumnarToRow (42) - +- CometColumnarExchange (41) - +- RowToColumnar (40) - +- * HashAggregate (39) - +- * ColumnarToRow (38) - +- CometProject (37) - +- CometBroadcastHashJoin (36) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.store (9) - +- CometBroadcastExchange (35) - +- CometHashAggregate (34) - +- CometColumnarExchange (33) - +- CometHashAggregate (32) - +- CometBroadcastHashJoin (31) - :- CometProject (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.customer_address (14) - +- CometBroadcastExchange (30) - +- CometProject (29) - +- CometFilter (28) - +- CometHashAggregate (27) - +- CometColumnarExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometFilter (18) - : +- CometScan parquet spark_catalog.default.customer_address (17) - +- CometBroadcastExchange (22) - +- CometProject (21) - +- CometFilter (20) - +- CometScan parquet spark_catalog.default.customer (19) +TakeOrderedAndProject (42) ++- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * ColumnarToRow (38) + +- CometProject (37) + +- CometBroadcastHashJoin (36) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.store (9) + +- CometBroadcastExchange (35) + +- CometHashAggregate (34) + +- CometExchange (33) + +- CometHashAggregate (32) + +- CometBroadcastHashJoin (31) + :- CometProject (16) + : +- CometFilter (15) + : +- CometScan parquet spark_catalog.default.customer_address (14) + +- CometBroadcastExchange (30) + +- CometProject (29) + +- CometFilter (28) + +- CometHashAggregate (27) + +- CometExchange (26) + +- CometHashAggregate (25) + +- CometProject (24) + +- CometBroadcastHashJoin (23) + :- CometFilter (18) + : +- CometScan parquet spark_catalog.default.customer_address (17) + +- CometBroadcastExchange (22) + +- CometProject (21) + +- CometFilter (20) + +- CometScan parquet spark_catalog.default.customer (19) (1) Scan parquet spark_catalog.default.store_sales @@ -167,9 +165,9 @@ Input [1]: [ca_zip#14] Keys [1]: [ca_zip#14] Functions [1]: [partial_count(1)] -(26) CometColumnarExchange +(26) CometExchange Input [2]: [ca_zip#14, count#17] -Arguments: hashpartitioning(ca_zip#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ca_zip#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [2]: [ca_zip#14, count#17] @@ -198,9 +196,9 @@ Input [1]: [ca_zip#12] Keys [1]: [ca_zip#12] Functions: [] -(33) CometColumnarExchange +(33) CometExchange Input [1]: [ca_zip#12] -Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ca_zip#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (34) CometHashAggregate Input [1]: [ca_zip#12] @@ -230,56 +228,50 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum#20] Results [2]: [s_store_name#9, sum#21] -(40) RowToColumnar +(40) Exchange Input [2]: [s_store_name#9, sum#21] +Arguments: hashpartitioning(s_store_name#9, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(41) CometColumnarExchange -Input [2]: [s_store_name#9, sum#21] -Arguments: hashpartitioning(s_store_name#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(42) ColumnarToRow [codegen id : 2] -Input [2]: [s_store_name#9, sum#21] - -(43) HashAggregate [codegen id : 2] +(41) HashAggregate [codegen id : 2] Input [2]: [s_store_name#9, sum#21] Keys [1]: [s_store_name#9] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] Results [2]: [s_store_name#9, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS sum(ss_net_profit)#23] -(44) TakeOrderedAndProject +(42) TakeOrderedAndProject Input [2]: [s_store_name#9, sum(ss_net_profit)#23] Arguments: 100, [s_store_name#9 ASC NULLS FIRST], [s_store_name#9, sum(ss_net_profit)#23] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (49) -+- * ColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan parquet spark_catalog.default.date_dim (45) +BroadcastExchange (47) ++- * ColumnarToRow (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan parquet spark_catalog.default.date_dim (43) -(45) Scan parquet spark_catalog.default.date_dim +(43) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_year#6, d_qoy#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(46) CometFilter +(44) CometFilter Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] Condition : ((((isnotnull(d_qoy#7) AND isnotnull(d_year#6)) AND (d_qoy#7 = 2)) AND (d_year#6 = 1998)) AND isnotnull(d_date_sk#5)) -(47) CometProject +(45) CometProject Input [3]: [d_date_sk#5, d_year#6, d_qoy#7] Arguments: [d_date_sk#5], [d_date_sk#5] -(48) ColumnarToRow [codegen id : 1] +(46) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(49) BroadcastExchange +(47) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt index 2b89ebb92..150dc4c17 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q8/simplified.txt @@ -1,56 +1,54 @@ TakeOrderedAndProject [s_store_name,sum(ss_net_profit)] WholeStageCodegen (2) HashAggregate [s_store_name,sum] [sum(UnscaledValue(ss_net_profit)),sum(ss_net_profit),sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_name] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_store_name,ss_net_profit] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_net_profit,s_store_name] - CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] - CometProject [ss_net_profit,s_store_name,s_zip] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,s_store_sk,s_store_name,s_zip] - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #4 - CometFilter [s_store_sk,s_store_name,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] - CometBroadcastExchange [ca_zip] #5 + InputAdapter + Exchange [s_store_name] #1 + WholeStageCodegen (1) + HashAggregate [s_store_name,ss_net_profit] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_net_profit,s_store_name] + CometBroadcastHashJoin [ss_net_profit,s_store_name,s_zip,ca_zip] + CometProject [ss_net_profit,s_store_name,s_zip] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,s_store_sk,s_store_name,s_zip] + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_name,s_zip] #4 + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + CometBroadcastExchange [ca_zip] #5 + CometHashAggregate [ca_zip] + CometExchange [ca_zip] #6 CometHashAggregate [ca_zip] - CometColumnarExchange [ca_zip] #6 - CometHashAggregate [ca_zip] - CometBroadcastHashJoin [ca_zip,ca_zip] - CometProject [ca_zip] [ca_zip] - CometFilter [ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_zip] - CometBroadcastExchange [ca_zip] #7 - CometProject [ca_zip] - CometFilter [ca_zip,cnt] - CometHashAggregate [ca_zip,cnt,ca_zip,count,count(1)] - CometColumnarExchange [ca_zip] #8 - CometHashAggregate [ca_zip,count] - CometProject [ca_zip] - CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] - CometFilter [ca_address_sk,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] - CometBroadcastExchange [c_current_addr_sk] #9 - CometProject [c_current_addr_sk] - CometFilter [c_current_addr_sk,c_preferred_cust_flag] - CometScan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] + CometBroadcastHashJoin [ca_zip,ca_zip] + CometProject [ca_zip] [ca_zip] + CometFilter [ca_zip] + CometScan parquet spark_catalog.default.customer_address [ca_zip] + CometBroadcastExchange [ca_zip] #7 + CometProject [ca_zip] + CometFilter [ca_zip,cnt] + CometHashAggregate [ca_zip,cnt,ca_zip,count,count(1)] + CometExchange [ca_zip] #8 + CometHashAggregate [ca_zip,count] + CometProject [ca_zip] + CometBroadcastHashJoin [ca_address_sk,ca_zip,c_current_addr_sk] + CometFilter [ca_address_sk,ca_zip] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_zip] + CometBroadcastExchange [c_current_addr_sk] #9 + CometProject [c_current_addr_sk] + CometFilter [c_current_addr_sk,c_preferred_cust_flag] + CometScan parquet spark_catalog.default.customer [c_current_addr_sk,c_preferred_cust_flag] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt index 97db908a7..518dc7d83 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/explain.txt @@ -1,114 +1,106 @@ == Physical Plan == -TakeOrderedAndProject (110) -+- * HashAggregate (109) - +- * ColumnarToRow (108) - +- CometColumnarExchange (107) - +- RowToColumnar (106) - +- * HashAggregate (105) - +- * Expand (104) - +- Union (103) - :- * HashAggregate (40) - : +- * ColumnarToRow (39) - : +- CometColumnarExchange (38) - : +- RowToColumnar (37) - : +- * HashAggregate (36) - : +- * ColumnarToRow (35) - : +- CometProject (34) - : +- CometBroadcastHashJoin (33) - : :- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometProject (22) - : : : +- CometBroadcastHashJoin (21) - : : : :- CometProject (17) - : : : : +- CometBroadcastHashJoin (16) - : : : : :- CometProject (11) - : : : : : +- CometSortMergeJoin (10) - : : : : : :- CometSort (4) - : : : : : : +- CometColumnarExchange (3) - : : : : : : +- CometFilter (2) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : +- CometSort (9) - : : : : : +- CometColumnarExchange (8) - : : : : : +- CometProject (7) - : : : : : +- CometFilter (6) - : : : : : +- CometScan parquet spark_catalog.default.store_returns (5) - : : : : +- CometBroadcastExchange (15) - : : : : +- CometProject (14) - : : : : +- CometFilter (13) - : : : : +- CometScan parquet spark_catalog.default.date_dim (12) - : : : +- CometBroadcastExchange (20) - : : : +- CometFilter (19) - : : : +- CometScan parquet spark_catalog.default.store (18) - : : +- CometBroadcastExchange (26) - : : +- CometProject (25) - : : +- CometFilter (24) - : : +- CometScan parquet spark_catalog.default.item (23) - : +- CometBroadcastExchange (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.promotion (29) - :- * HashAggregate (71) - : +- * ColumnarToRow (70) - : +- CometColumnarExchange (69) - : +- RowToColumnar (68) - : +- * HashAggregate (67) - : +- * ColumnarToRow (66) - : +- CometProject (65) - : +- CometBroadcastHashJoin (64) - : :- CometProject (62) - : : +- CometBroadcastHashJoin (61) - : : :- CometProject (59) - : : : +- CometBroadcastHashJoin (58) - : : : :- CometProject (54) - : : : : +- CometBroadcastHashJoin (53) - : : : : :- CometProject (51) - : : : : : +- CometSortMergeJoin (50) - : : : : : :- CometSort (44) - : : : : : : +- CometColumnarExchange (43) - : : : : : : +- CometFilter (42) - : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (41) - : : : : : +- CometSort (49) - : : : : : +- CometColumnarExchange (48) - : : : : : +- CometProject (47) - : : : : : +- CometFilter (46) - : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (45) - : : : : +- ReusedExchange (52) - : : : +- CometBroadcastExchange (57) - : : : +- CometFilter (56) - : : : +- CometScan parquet spark_catalog.default.catalog_page (55) - : : +- ReusedExchange (60) - : +- ReusedExchange (63) - +- * HashAggregate (102) - +- * ColumnarToRow (101) - +- CometColumnarExchange (100) - +- RowToColumnar (99) - +- * HashAggregate (98) - +- * ColumnarToRow (97) - +- CometProject (96) - +- CometBroadcastHashJoin (95) - :- CometProject (93) - : +- CometBroadcastHashJoin (92) - : :- CometProject (90) - : : +- CometBroadcastHashJoin (89) - : : :- CometProject (85) - : : : +- CometBroadcastHashJoin (84) - : : : :- CometProject (82) - : : : : +- CometSortMergeJoin (81) - : : : : :- CometSort (75) - : : : : : +- CometColumnarExchange (74) - : : : : : +- CometFilter (73) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (72) - : : : : +- CometSort (80) - : : : : +- CometColumnarExchange (79) - : : : : +- CometProject (78) - : : : : +- CometFilter (77) - : : : : +- CometScan parquet spark_catalog.default.web_returns (76) - : : : +- ReusedExchange (83) - : : +- CometBroadcastExchange (88) - : : +- CometFilter (87) - : : +- CometScan parquet spark_catalog.default.web_site (86) - : +- ReusedExchange (91) - +- ReusedExchange (94) +TakeOrderedAndProject (102) ++- * HashAggregate (101) + +- Exchange (100) + +- * HashAggregate (99) + +- * Expand (98) + +- Union (97) + :- * HashAggregate (38) + : +- Exchange (37) + : +- * HashAggregate (36) + : +- * ColumnarToRow (35) + : +- CometProject (34) + : +- CometBroadcastHashJoin (33) + : :- CometProject (28) + : : +- CometBroadcastHashJoin (27) + : : :- CometProject (22) + : : : +- CometBroadcastHashJoin (21) + : : : :- CometProject (17) + : : : : +- CometBroadcastHashJoin (16) + : : : : :- CometProject (11) + : : : : : +- CometSortMergeJoin (10) + : : : : : :- CometSort (4) + : : : : : : +- CometExchange (3) + : : : : : : +- CometFilter (2) + : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : +- CometSort (9) + : : : : : +- CometExchange (8) + : : : : : +- CometProject (7) + : : : : : +- CometFilter (6) + : : : : : +- CometScan parquet spark_catalog.default.store_returns (5) + : : : : +- CometBroadcastExchange (15) + : : : : +- CometProject (14) + : : : : +- CometFilter (13) + : : : : +- CometScan parquet spark_catalog.default.date_dim (12) + : : : +- CometBroadcastExchange (20) + : : : +- CometFilter (19) + : : : +- CometScan parquet spark_catalog.default.store (18) + : : +- CometBroadcastExchange (26) + : : +- CometProject (25) + : : +- CometFilter (24) + : : +- CometScan parquet spark_catalog.default.item (23) + : +- CometBroadcastExchange (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan parquet spark_catalog.default.promotion (29) + :- * HashAggregate (67) + : +- Exchange (66) + : +- * HashAggregate (65) + : +- * ColumnarToRow (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (57) + : : : +- CometBroadcastHashJoin (56) + : : : :- CometProject (52) + : : : : +- CometBroadcastHashJoin (51) + : : : : :- CometProject (49) + : : : : : +- CometSortMergeJoin (48) + : : : : : :- CometSort (42) + : : : : : : +- CometExchange (41) + : : : : : : +- CometFilter (40) + : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (39) + : : : : : +- CometSort (47) + : : : : : +- CometExchange (46) + : : : : : +- CometProject (45) + : : : : : +- CometFilter (44) + : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (43) + : : : : +- ReusedExchange (50) + : : : +- CometBroadcastExchange (55) + : : : +- CometFilter (54) + : : : +- CometScan parquet spark_catalog.default.catalog_page (53) + : : +- ReusedExchange (58) + : +- ReusedExchange (61) + +- * HashAggregate (96) + +- Exchange (95) + +- * HashAggregate (94) + +- * ColumnarToRow (93) + +- CometProject (92) + +- CometBroadcastHashJoin (91) + :- CometProject (89) + : +- CometBroadcastHashJoin (88) + : :- CometProject (86) + : : +- CometBroadcastHashJoin (85) + : : :- CometProject (81) + : : : +- CometBroadcastHashJoin (80) + : : : :- CometProject (78) + : : : : +- CometSortMergeJoin (77) + : : : : :- CometSort (71) + : : : : : +- CometExchange (70) + : : : : : +- CometFilter (69) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (68) + : : : : +- CometSort (76) + : : : : +- CometExchange (75) + : : : : +- CometProject (74) + : : : : +- CometFilter (73) + : : : : +- CometScan parquet spark_catalog.default.web_returns (72) + : : : +- ReusedExchange (79) + : : +- CometBroadcastExchange (84) + : : +- CometFilter (83) + : : +- CometScan parquet spark_catalog.default.web_site (82) + : +- ReusedExchange (87) + +- ReusedExchange (90) (1) Scan parquet spark_catalog.default.store_sales @@ -123,9 +115,9 @@ ReadSchema: struct -(42) CometFilter +(40) CometFilter Input [7]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Condition : ((isnotnull(cs_catalog_page_sk#40) AND isnotnull(cs_item_sk#41)) AND isnotnull(cs_promo_sk#42)) -(43) CometColumnarExchange +(41) CometExchange Input [7]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] -Arguments: hashpartitioning(cs_item_sk#41, cs_order_number#43, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_item_sk#41, cs_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(44) CometSort +(42) CometSort Input [7]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Arguments: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46], [cs_item_sk#41 ASC NULLS FIRST, cs_order_number#43 ASC NULLS FIRST] -(45) Scan parquet spark_catalog.default.catalog_returns +(43) Scan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(46) CometFilter +(44) CometFilter Input [5]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Condition : (isnotnull(cr_item_sk#48) AND isnotnull(cr_order_number#49)) -(47) CometProject +(45) CometProject Input [5]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Arguments: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51], [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51] -(48) CometColumnarExchange +(46) CometExchange Input [4]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51] -Arguments: hashpartitioning(cr_item_sk#48, cr_order_number#49, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(cr_item_sk#48, cr_order_number#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(49) CometSort +(47) CometSort Input [4]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51] Arguments: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51], [cr_item_sk#48 ASC NULLS FIRST, cr_order_number#49 ASC NULLS FIRST] -(50) CometSortMergeJoin +(48) CometSortMergeJoin Left output [7]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Right output [4]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51] Arguments: [cs_item_sk#41, cs_order_number#43], [cr_item_sk#48, cr_order_number#49], LeftOuter -(51) CometProject +(49) CometProject Input [11]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46, cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51] Arguments: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46, cr_return_amount#50, cr_net_loss#51], [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46, cr_return_amount#50, cr_net_loss#51] -(52) ReusedExchange [Reuses operator id: 15] +(50) ReusedExchange [Reuses operator id: 15] Output [1]: [d_date_sk#53] -(53) CometBroadcastHashJoin +(51) CometBroadcastHashJoin Left output [8]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46, cr_return_amount#50, cr_net_loss#51] Right output [1]: [d_date_sk#53] Arguments: [cs_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight -(54) CometProject +(52) CometProject Input [9]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46, cr_return_amount#50, cr_net_loss#51, d_date_sk#53] Arguments: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51], [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51] -(55) Scan parquet spark_catalog.default.catalog_page +(53) Scan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#54, cp_catalog_page_id#55] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(56) CometFilter +(54) CometFilter Input [2]: [cp_catalog_page_sk#54, cp_catalog_page_id#55] Condition : isnotnull(cp_catalog_page_sk#54) -(57) CometBroadcastExchange +(55) CometBroadcastExchange Input [2]: [cp_catalog_page_sk#54, cp_catalog_page_id#55] Arguments: [cp_catalog_page_sk#54, cp_catalog_page_id#55] -(58) CometBroadcastHashJoin +(56) CometBroadcastHashJoin Left output [7]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51] Right output [2]: [cp_catalog_page_sk#54, cp_catalog_page_id#55] Arguments: [cs_catalog_page_sk#40], [cp_catalog_page_sk#54], Inner, BuildRight -(59) CometProject +(57) CometProject Input [9]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_sk#54, cp_catalog_page_id#55] Arguments: [cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55], [cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] -(60) ReusedExchange [Reuses operator id: 26] +(58) ReusedExchange [Reuses operator id: 26] Output [1]: [i_item_sk#56] -(61) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [7]: [cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] Right output [1]: [i_item_sk#56] Arguments: [cs_item_sk#41], [i_item_sk#56], Inner, BuildRight -(62) CometProject +(60) CometProject Input [8]: [cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55, i_item_sk#56] Arguments: [cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55], [cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] -(63) ReusedExchange [Reuses operator id: 32] +(61) ReusedExchange [Reuses operator id: 32] Output [1]: [p_promo_sk#57] -(64) CometBroadcastHashJoin +(62) CometBroadcastHashJoin Left output [6]: [cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] Right output [1]: [p_promo_sk#57] Arguments: [cs_promo_sk#42], [p_promo_sk#57], Inner, BuildRight -(65) CometProject +(63) CometProject Input [7]: [cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55, p_promo_sk#57] Arguments: [cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55], [cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] -(66) ColumnarToRow [codegen id : 3] +(64) ColumnarToRow [codegen id : 3] Input [5]: [cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] -(67) HashAggregate [codegen id : 3] +(65) HashAggregate [codegen id : 3] Input [5]: [cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] Keys [1]: [cp_catalog_page_id#55] Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#44)), partial_sum(coalesce(cast(cr_return_amount#50 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#45 - coalesce(cast(cr_net_loss#51 as decimal(12,2)), 0.00)))] Aggregate Attributes [5]: [sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] Results [6]: [cp_catalog_page_id#55, sum#63, sum#64, isEmpty#65, sum#66, isEmpty#67] -(68) RowToColumnar -Input [6]: [cp_catalog_page_id#55, sum#63, sum#64, isEmpty#65, sum#66, isEmpty#67] - -(69) CometColumnarExchange -Input [6]: [cp_catalog_page_id#55, sum#63, sum#64, isEmpty#65, sum#66, isEmpty#67] -Arguments: hashpartitioning(cp_catalog_page_id#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(70) ColumnarToRow [codegen id : 4] +(66) Exchange Input [6]: [cp_catalog_page_id#55, sum#63, sum#64, isEmpty#65, sum#66, isEmpty#67] +Arguments: hashpartitioning(cp_catalog_page_id#55, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(71) HashAggregate [codegen id : 4] +(67) HashAggregate [codegen id : 4] Input [6]: [cp_catalog_page_id#55, sum#63, sum#64, isEmpty#65, sum#66, isEmpty#67] Keys [1]: [cp_catalog_page_id#55] Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#44)), sum(coalesce(cast(cr_return_amount#50 as decimal(12,2)), 0.00)), sum((cs_net_profit#45 - coalesce(cast(cr_net_loss#51 as decimal(12,2)), 0.00)))] Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#44))#68, sum(coalesce(cast(cr_return_amount#50 as decimal(12,2)), 0.00))#69, sum((cs_net_profit#45 - coalesce(cast(cr_net_loss#51 as decimal(12,2)), 0.00)))#70] Results [5]: [MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#44))#68,17,2) AS sales#71, sum(coalesce(cast(cr_return_amount#50 as decimal(12,2)), 0.00))#69 AS returns#72, sum((cs_net_profit#45 - coalesce(cast(cr_net_loss#51 as decimal(12,2)), 0.00)))#70 AS profit#73, catalog channel AS channel#74, concat(catalog_page, cp_catalog_page_id#55) AS id#75] -(72) Scan parquet spark_catalog.default.web_sales +(68) Scan parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82] Batched: true Location: InMemoryFileIndex [] @@ -445,205 +425,193 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#82), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct -(73) CometFilter +(69) CometFilter Input [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82] Condition : ((isnotnull(ws_web_site_sk#77) AND isnotnull(ws_item_sk#76)) AND isnotnull(ws_promo_sk#78)) -(74) CometColumnarExchange +(70) CometExchange Input [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82] -Arguments: hashpartitioning(ws_item_sk#76, ws_order_number#79, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(ws_item_sk#76, ws_order_number#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(75) CometSort +(71) CometSort Input [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82] Arguments: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82], [ws_item_sk#76 ASC NULLS FIRST, ws_order_number#79 ASC NULLS FIRST] -(76) Scan parquet spark_catalog.default.web_returns +(72) Scan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct -(77) CometFilter +(73) CometFilter Input [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] Condition : (isnotnull(wr_item_sk#84) AND isnotnull(wr_order_number#85)) -(78) CometProject +(74) CometProject Input [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] Arguments: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87], [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87] -(79) CometColumnarExchange +(75) CometExchange Input [4]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87] -Arguments: hashpartitioning(wr_item_sk#84, wr_order_number#85, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(wr_item_sk#84, wr_order_number#85, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(80) CometSort +(76) CometSort Input [4]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87] Arguments: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87], [wr_item_sk#84 ASC NULLS FIRST, wr_order_number#85 ASC NULLS FIRST] -(81) CometSortMergeJoin +(77) CometSortMergeJoin Left output [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82] Right output [4]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87] Arguments: [ws_item_sk#76, ws_order_number#79], [wr_item_sk#84, wr_order_number#85], LeftOuter -(82) CometProject +(78) CometProject Input [11]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82, wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87] Arguments: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82, wr_return_amt#86, wr_net_loss#87], [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82, wr_return_amt#86, wr_net_loss#87] -(83) ReusedExchange [Reuses operator id: 15] +(79) ReusedExchange [Reuses operator id: 15] Output [1]: [d_date_sk#89] -(84) CometBroadcastHashJoin +(80) CometBroadcastHashJoin Left output [8]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82, wr_return_amt#86, wr_net_loss#87] Right output [1]: [d_date_sk#89] Arguments: [ws_sold_date_sk#82], [d_date_sk#89], Inner, BuildRight -(85) CometProject +(81) CometProject Input [9]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82, wr_return_amt#86, wr_net_loss#87, d_date_sk#89] Arguments: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87], [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87] -(86) Scan parquet spark_catalog.default.web_site +(82) Scan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#90, web_site_id#91] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(87) CometFilter +(83) CometFilter Input [2]: [web_site_sk#90, web_site_id#91] Condition : isnotnull(web_site_sk#90) -(88) CometBroadcastExchange +(84) CometBroadcastExchange Input [2]: [web_site_sk#90, web_site_id#91] Arguments: [web_site_sk#90, web_site_id#91] -(89) CometBroadcastHashJoin +(85) CometBroadcastHashJoin Left output [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87] Right output [2]: [web_site_sk#90, web_site_id#91] Arguments: [ws_web_site_sk#77], [web_site_sk#90], Inner, BuildRight -(90) CometProject +(86) CometProject Input [9]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_sk#90, web_site_id#91] Arguments: [ws_item_sk#76, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91], [ws_item_sk#76, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] -(91) ReusedExchange [Reuses operator id: 26] +(87) ReusedExchange [Reuses operator id: 26] Output [1]: [i_item_sk#92] -(92) CometBroadcastHashJoin +(88) CometBroadcastHashJoin Left output [7]: [ws_item_sk#76, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] Right output [1]: [i_item_sk#92] Arguments: [ws_item_sk#76], [i_item_sk#92], Inner, BuildRight -(93) CometProject +(89) CometProject Input [8]: [ws_item_sk#76, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91, i_item_sk#92] Arguments: [ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91], [ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] -(94) ReusedExchange [Reuses operator id: 32] +(90) ReusedExchange [Reuses operator id: 32] Output [1]: [p_promo_sk#93] -(95) CometBroadcastHashJoin +(91) CometBroadcastHashJoin Left output [6]: [ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] Right output [1]: [p_promo_sk#93] Arguments: [ws_promo_sk#78], [p_promo_sk#93], Inner, BuildRight -(96) CometProject +(92) CometProject Input [7]: [ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91, p_promo_sk#93] Arguments: [ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91], [ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] -(97) ColumnarToRow [codegen id : 5] +(93) ColumnarToRow [codegen id : 5] Input [5]: [ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] -(98) HashAggregate [codegen id : 5] +(94) HashAggregate [codegen id : 5] Input [5]: [ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] Keys [1]: [web_site_id#91] Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#80)), partial_sum(coalesce(cast(wr_return_amt#86 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#87 as decimal(12,2)), 0.00)))] Aggregate Attributes [5]: [sum#94, sum#95, isEmpty#96, sum#97, isEmpty#98] Results [6]: [web_site_id#91, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103] -(99) RowToColumnar -Input [6]: [web_site_id#91, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103] - -(100) CometColumnarExchange -Input [6]: [web_site_id#91, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103] -Arguments: hashpartitioning(web_site_id#91, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(101) ColumnarToRow [codegen id : 6] +(95) Exchange Input [6]: [web_site_id#91, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103] +Arguments: hashpartitioning(web_site_id#91, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(102) HashAggregate [codegen id : 6] +(96) HashAggregate [codegen id : 6] Input [6]: [web_site_id#91, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103] Keys [1]: [web_site_id#91] Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#80)), sum(coalesce(cast(wr_return_amt#86 as decimal(12,2)), 0.00)), sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#87 as decimal(12,2)), 0.00)))] Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#80))#104, sum(coalesce(cast(wr_return_amt#86 as decimal(12,2)), 0.00))#105, sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#87 as decimal(12,2)), 0.00)))#106] Results [5]: [MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#80))#104,17,2) AS sales#107, sum(coalesce(cast(wr_return_amt#86 as decimal(12,2)), 0.00))#105 AS returns#108, sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#87 as decimal(12,2)), 0.00)))#106 AS profit#109, web channel AS channel#110, concat(web_site, web_site_id#91) AS id#111] -(103) Union +(97) Union -(104) Expand [codegen id : 7] +(98) Expand [codegen id : 7] Input [5]: [sales#35, returns#36, profit#37, channel#38, id#39] Arguments: [[sales#35, returns#36, profit#37, channel#38, id#39, 0], [sales#35, returns#36, profit#37, channel#38, null, 1], [sales#35, returns#36, profit#37, null, null, 3]], [sales#35, returns#36, profit#37, channel#112, id#113, spark_grouping_id#114] -(105) HashAggregate [codegen id : 7] +(99) HashAggregate [codegen id : 7] Input [6]: [sales#35, returns#36, profit#37, channel#112, id#113, spark_grouping_id#114] Keys [3]: [channel#112, id#113, spark_grouping_id#114] Functions [3]: [partial_sum(sales#35), partial_sum(returns#36), partial_sum(profit#37)] Aggregate Attributes [6]: [sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120] Results [9]: [channel#112, id#113, spark_grouping_id#114, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -(106) RowToColumnar -Input [9]: [channel#112, id#113, spark_grouping_id#114, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] - -(107) CometColumnarExchange -Input [9]: [channel#112, id#113, spark_grouping_id#114, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] -Arguments: hashpartitioning(channel#112, id#113, spark_grouping_id#114, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] - -(108) ColumnarToRow [codegen id : 8] +(100) Exchange Input [9]: [channel#112, id#113, spark_grouping_id#114, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] +Arguments: hashpartitioning(channel#112, id#113, spark_grouping_id#114, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(109) HashAggregate [codegen id : 8] +(101) HashAggregate [codegen id : 8] Input [9]: [channel#112, id#113, spark_grouping_id#114, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] Keys [3]: [channel#112, id#113, spark_grouping_id#114] Functions [3]: [sum(sales#35), sum(returns#36), sum(profit#37)] Aggregate Attributes [3]: [sum(sales#35)#127, sum(returns#36)#128, sum(profit#37)#129] Results [5]: [channel#112, id#113, sum(sales#35)#127 AS sales#130, sum(returns#36)#128 AS returns#131, sum(profit#37)#129 AS profit#132] -(110) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [5]: [channel#112, id#113, sales#130, returns#131, profit#132] Arguments: 100, [channel#112 ASC NULLS FIRST, id#113 ASC NULLS FIRST], [channel#112, id#113, sales#130, returns#131, profit#132] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (115) -+- * ColumnarToRow (114) - +- CometProject (113) - +- CometFilter (112) - +- CometScan parquet spark_catalog.default.date_dim (111) +BroadcastExchange (107) ++- * ColumnarToRow (106) + +- CometProject (105) + +- CometFilter (104) + +- CometScan parquet spark_catalog.default.date_dim (103) -(111) Scan parquet spark_catalog.default.date_dim +(103) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-22), IsNotNull(d_date_sk)] ReadSchema: struct -(112) CometFilter +(104) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2000-08-23)) AND (d_date#15 <= 2000-09-22)) AND isnotnull(d_date_sk#14)) -(113) CometProject +(105) CometProject Input [2]: [d_date_sk#14, d_date#15] Arguments: [d_date_sk#14], [d_date_sk#14] -(114) ColumnarToRow [codegen id : 1] +(106) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(115) BroadcastExchange +(107) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] -Subquery:2 Hosting operator id = 41 Hosting Expression = cs_sold_date_sk#46 IN dynamicpruning#8 +Subquery:2 Hosting operator id = 39 Hosting Expression = cs_sold_date_sk#46 IN dynamicpruning#8 -Subquery:3 Hosting operator id = 72 Hosting Expression = ws_sold_date_sk#82 IN dynamicpruning#8 +Subquery:3 Hosting operator id = 68 Hosting Expression = ws_sold_date_sk#82 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt index bb5fbbbcc..717c94e21 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q80/simplified.txt @@ -1,136 +1,128 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (8) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk,ss_ticket_number] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometColumnarExchange [sr_item_sk,sr_ticket_number] #5 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #7 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #8 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #9 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (4) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [cp_catalog_page_id] #10 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometColumnarExchange [cs_item_sk,cs_order_number] #11 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometColumnarExchange [cr_item_sk,cr_order_number] #12 - CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (6) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [web_site_id] #14 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] - CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] - CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometColumnarExchange [ws_item_sk,ws_order_number] #15 - CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometColumnarExchange [wr_item_sk,wr_order_number] #16 - CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [web_site_sk,web_site_id] #17 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 + InputAdapter + Exchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (7) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (2) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [s_store_id] #2 + WholeStageCodegen (1) + HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #5 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #8 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #9 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (4) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [cp_catalog_page_id] #10 + WholeStageCodegen (3) + HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #11 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (6) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [web_site_id] #14 + WholeStageCodegen (5) + HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #15 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #16 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt index 9c8482d6e..b266f4a24 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/explain.txt @@ -1,60 +1,54 @@ == Physical Plan == -TakeOrderedAndProject (56) -+- * Project (55) - +- * BroadcastHashJoin Inner BuildRight (54) - :- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (43) - : : +- * BroadcastHashJoin Inner BuildRight (42) - : : :- * Filter (20) - : : : +- * HashAggregate (19) - : : : +- * ColumnarToRow (18) - : : : +- CometColumnarExchange (17) - : : : +- RowToColumnar (16) - : : : +- * HashAggregate (15) - : : : +- * ColumnarToRow (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.customer_address (9) - : : +- BroadcastExchange (41) - : : +- * Filter (40) - : : +- * HashAggregate (39) - : : +- * ColumnarToRow (38) - : : +- CometColumnarExchange (37) - : : +- RowToColumnar (36) - : : +- * HashAggregate (35) - : : +- * HashAggregate (34) - : : +- * ColumnarToRow (33) - : : +- CometColumnarExchange (32) - : : +- RowToColumnar (31) - : : +- * HashAggregate (30) - : : +- * ColumnarToRow (29) - : : +- CometProject (28) - : : +- CometBroadcastHashJoin (27) - : : :- CometProject (25) - : : : +- CometBroadcastHashJoin (24) - : : : :- CometFilter (22) - : : : : +- CometScan parquet spark_catalog.default.catalog_returns (21) - : : : +- ReusedExchange (23) - : : +- ReusedExchange (26) - : +- BroadcastExchange (47) - : +- * ColumnarToRow (46) - : +- CometFilter (45) - : +- CometScan parquet spark_catalog.default.customer (44) - +- BroadcastExchange (53) - +- * ColumnarToRow (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.customer_address (50) +TakeOrderedAndProject (50) ++- * Project (49) + +- * BroadcastHashJoin Inner BuildRight (48) + :- * Project (43) + : +- * BroadcastHashJoin Inner BuildRight (42) + : :- * Project (37) + : : +- * BroadcastHashJoin Inner BuildRight (36) + : : :- * Filter (18) + : : : +- * HashAggregate (17) + : : : +- Exchange (16) + : : : +- * HashAggregate (15) + : : : +- * ColumnarToRow (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan parquet spark_catalog.default.customer_address (9) + : : +- BroadcastExchange (35) + : : +- * Filter (34) + : : +- * HashAggregate (33) + : : +- Exchange (32) + : : +- * HashAggregate (31) + : : +- * HashAggregate (30) + : : +- Exchange (29) + : : +- * HashAggregate (28) + : : +- * ColumnarToRow (27) + : : +- CometProject (26) + : : +- CometBroadcastHashJoin (25) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometFilter (20) + : : : : +- CometScan parquet spark_catalog.default.catalog_returns (19) + : : : +- ReusedExchange (21) + : : +- ReusedExchange (24) + : +- BroadcastExchange (41) + : +- * ColumnarToRow (40) + : +- CometFilter (39) + : +- CometScan parquet spark_catalog.default.customer (38) + +- BroadcastExchange (47) + +- * ColumnarToRow (46) + +- CometFilter (45) + +- CometScan parquet spark_catalog.default.customer_address (44) (1) Scan parquet spark_catalog.default.catalog_returns @@ -131,28 +125,22 @@ Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] Aggregate Attributes [1]: [sum#10] Results [3]: [cr_returning_customer_sk#1, ca_state#9, sum#11] -(16) RowToColumnar +(16) Exchange Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#11] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#11] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 7] -Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#11] - -(19) HashAggregate [codegen id : 7] +(17) HashAggregate [codegen id : 7] Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#11] Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))] Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#3))#12] Results [3]: [cr_returning_customer_sk#1 AS ctr_customer_sk#13, ca_state#9 AS ctr_state#14, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#3))#12,17,2) AS ctr_total_return#15] -(20) Filter [codegen id : 7] +(18) Filter [codegen id : 7] Input [3]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15] Condition : isnotnull(ctr_total_return#15) -(21) Scan parquet spark_catalog.default.catalog_returns +(19) Scan parquet spark_catalog.default.catalog_returns Output [4]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18, cr_returned_date_sk#19] Batched: true Location: InMemoryFileIndex [] @@ -160,195 +148,183 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#19), dynamicpruningexpression(c PushedFilters: [IsNotNull(cr_returning_addr_sk)] ReadSchema: struct -(22) CometFilter +(20) CometFilter Input [4]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18, cr_returned_date_sk#19] Condition : isnotnull(cr_returning_addr_sk#17) -(23) ReusedExchange [Reuses operator id: 6] +(21) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#21] -(24) CometBroadcastHashJoin +(22) CometBroadcastHashJoin Left output [4]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18, cr_returned_date_sk#19] Right output [1]: [d_date_sk#21] Arguments: [cr_returned_date_sk#19], [d_date_sk#21], Inner, BuildRight -(25) CometProject +(23) CometProject Input [5]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18, cr_returned_date_sk#19, d_date_sk#21] Arguments: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18], [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18] -(26) ReusedExchange [Reuses operator id: 11] +(24) ReusedExchange [Reuses operator id: 11] Output [2]: [ca_address_sk#22, ca_state#23] -(27) CometBroadcastHashJoin +(25) CometBroadcastHashJoin Left output [3]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18] Right output [2]: [ca_address_sk#22, ca_state#23] Arguments: [cr_returning_addr_sk#17], [ca_address_sk#22], Inner, BuildRight -(28) CometProject +(26) CometProject Input [5]: [cr_returning_customer_sk#16, cr_returning_addr_sk#17, cr_return_amt_inc_tax#18, ca_address_sk#22, ca_state#23] Arguments: [cr_returning_customer_sk#16, cr_return_amt_inc_tax#18, ca_state#23], [cr_returning_customer_sk#16, cr_return_amt_inc_tax#18, ca_state#23] -(29) ColumnarToRow [codegen id : 2] +(27) ColumnarToRow [codegen id : 2] Input [3]: [cr_returning_customer_sk#16, cr_return_amt_inc_tax#18, ca_state#23] -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [3]: [cr_returning_customer_sk#16, cr_return_amt_inc_tax#18, ca_state#23] Keys [2]: [cr_returning_customer_sk#16, ca_state#23] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#18))] Aggregate Attributes [1]: [sum#24] Results [3]: [cr_returning_customer_sk#16, ca_state#23, sum#25] -(31) RowToColumnar +(29) Exchange Input [3]: [cr_returning_customer_sk#16, ca_state#23, sum#25] +Arguments: hashpartitioning(cr_returning_customer_sk#16, ca_state#23, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(32) CometColumnarExchange -Input [3]: [cr_returning_customer_sk#16, ca_state#23, sum#25] -Arguments: hashpartitioning(cr_returning_customer_sk#16, ca_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(33) ColumnarToRow [codegen id : 3] -Input [3]: [cr_returning_customer_sk#16, ca_state#23, sum#25] - -(34) HashAggregate [codegen id : 3] +(30) HashAggregate [codegen id : 3] Input [3]: [cr_returning_customer_sk#16, ca_state#23, sum#25] Keys [2]: [cr_returning_customer_sk#16, ca_state#23] Functions [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#18))] Aggregate Attributes [1]: [sum(UnscaledValue(cr_return_amt_inc_tax#18))#12] Results [2]: [ca_state#23 AS ctr_state#26, MakeDecimal(sum(UnscaledValue(cr_return_amt_inc_tax#18))#12,17,2) AS ctr_total_return#27] -(35) HashAggregate [codegen id : 3] +(31) HashAggregate [codegen id : 3] Input [2]: [ctr_state#26, ctr_total_return#27] Keys [1]: [ctr_state#26] Functions [1]: [partial_avg(ctr_total_return#27)] Aggregate Attributes [2]: [sum#28, count#29] Results [3]: [ctr_state#26, sum#30, count#31] -(36) RowToColumnar -Input [3]: [ctr_state#26, sum#30, count#31] - -(37) CometColumnarExchange +(32) Exchange Input [3]: [ctr_state#26, sum#30, count#31] -Arguments: hashpartitioning(ctr_state#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ctr_state#26, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(38) ColumnarToRow [codegen id : 4] -Input [3]: [ctr_state#26, sum#30, count#31] - -(39) HashAggregate [codegen id : 4] +(33) HashAggregate [codegen id : 4] Input [3]: [ctr_state#26, sum#30, count#31] Keys [1]: [ctr_state#26] Functions [1]: [avg(ctr_total_return#27)] Aggregate Attributes [1]: [avg(ctr_total_return#27)#32] Results [2]: [(avg(ctr_total_return#27)#32 * 1.2) AS (avg(ctr_total_return) * 1.2)#33, ctr_state#26] -(40) Filter [codegen id : 4] +(34) Filter [codegen id : 4] Input [2]: [(avg(ctr_total_return) * 1.2)#33, ctr_state#26] Condition : isnotnull((avg(ctr_total_return) * 1.2)#33) -(41) BroadcastExchange +(35) BroadcastExchange Input [2]: [(avg(ctr_total_return) * 1.2)#33, ctr_state#26] Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=4] -(42) BroadcastHashJoin [codegen id : 7] +(36) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ctr_state#14] Right keys [1]: [ctr_state#26] Join type: Inner Join condition: (cast(ctr_total_return#15 as decimal(24,7)) > (avg(ctr_total_return) * 1.2)#33) -(43) Project [codegen id : 7] +(37) Project [codegen id : 7] Output [2]: [ctr_customer_sk#13, ctr_total_return#15] Input [5]: [ctr_customer_sk#13, ctr_state#14, ctr_total_return#15, (avg(ctr_total_return) * 1.2)#33, ctr_state#26] -(44) Scan parquet spark_catalog.default.customer +(38) Scan parquet spark_catalog.default.customer Output [6]: [c_customer_sk#34, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(45) CometFilter +(39) CometFilter Input [6]: [c_customer_sk#34, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39] Condition : (isnotnull(c_customer_sk#34) AND isnotnull(c_current_addr_sk#36)) -(46) ColumnarToRow [codegen id : 5] +(40) ColumnarToRow [codegen id : 5] Input [6]: [c_customer_sk#34, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39] -(47) BroadcastExchange +(41) BroadcastExchange Input [6]: [c_customer_sk#34, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(48) BroadcastHashJoin [codegen id : 7] +(42) BroadcastHashJoin [codegen id : 7] Left keys [1]: [ctr_customer_sk#13] Right keys [1]: [c_customer_sk#34] Join type: Inner Join condition: None -(49) Project [codegen id : 7] +(43) Project [codegen id : 7] Output [6]: [ctr_total_return#15, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39] Input [8]: [ctr_customer_sk#13, ctr_total_return#15, c_customer_sk#34, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39] -(50) Scan parquet spark_catalog.default.customer_address +(44) Scan parquet spark_catalog.default.customer_address Output [12]: [ca_address_sk#40, ca_street_number#41, ca_street_name#42, ca_street_type#43, ca_suite_number#44, ca_city#45, ca_county#46, ca_state#47, ca_zip#48, ca_country#49, ca_gmt_offset#50, ca_location_type#51] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_state), EqualTo(ca_state,GA), IsNotNull(ca_address_sk)] ReadSchema: struct -(51) CometFilter +(45) CometFilter Input [12]: [ca_address_sk#40, ca_street_number#41, ca_street_name#42, ca_street_type#43, ca_suite_number#44, ca_city#45, ca_county#46, ca_state#47, ca_zip#48, ca_country#49, ca_gmt_offset#50, ca_location_type#51] Condition : ((isnotnull(ca_state#47) AND (ca_state#47 = GA)) AND isnotnull(ca_address_sk#40)) -(52) ColumnarToRow [codegen id : 6] +(46) ColumnarToRow [codegen id : 6] Input [12]: [ca_address_sk#40, ca_street_number#41, ca_street_name#42, ca_street_type#43, ca_suite_number#44, ca_city#45, ca_county#46, ca_state#47, ca_zip#48, ca_country#49, ca_gmt_offset#50, ca_location_type#51] -(53) BroadcastExchange +(47) BroadcastExchange Input [12]: [ca_address_sk#40, ca_street_number#41, ca_street_name#42, ca_street_type#43, ca_suite_number#44, ca_city#45, ca_county#46, ca_state#47, ca_zip#48, ca_country#49, ca_gmt_offset#50, ca_location_type#51] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] -(54) BroadcastHashJoin [codegen id : 7] +(48) BroadcastHashJoin [codegen id : 7] Left keys [1]: [c_current_addr_sk#36] Right keys [1]: [ca_address_sk#40] Join type: Inner Join condition: None -(55) Project [codegen id : 7] +(49) Project [codegen id : 7] Output [16]: [c_customer_id#35, c_salutation#37, c_first_name#38, c_last_name#39, ca_street_number#41, ca_street_name#42, ca_street_type#43, ca_suite_number#44, ca_city#45, ca_county#46, ca_state#47, ca_zip#48, ca_country#49, ca_gmt_offset#50, ca_location_type#51, ctr_total_return#15] Input [18]: [ctr_total_return#15, c_customer_id#35, c_current_addr_sk#36, c_salutation#37, c_first_name#38, c_last_name#39, ca_address_sk#40, ca_street_number#41, ca_street_name#42, ca_street_type#43, ca_suite_number#44, ca_city#45, ca_county#46, ca_state#47, ca_zip#48, ca_country#49, ca_gmt_offset#50, ca_location_type#51] -(56) TakeOrderedAndProject +(50) TakeOrderedAndProject Input [16]: [c_customer_id#35, c_salutation#37, c_first_name#38, c_last_name#39, ca_street_number#41, ca_street_name#42, ca_street_type#43, ca_suite_number#44, ca_city#45, ca_county#46, ca_state#47, ca_zip#48, ca_country#49, ca_gmt_offset#50, ca_location_type#51, ctr_total_return#15] Arguments: 100, [c_customer_id#35 ASC NULLS FIRST, c_salutation#37 ASC NULLS FIRST, c_first_name#38 ASC NULLS FIRST, c_last_name#39 ASC NULLS FIRST, ca_street_number#41 ASC NULLS FIRST, ca_street_name#42 ASC NULLS FIRST, ca_street_type#43 ASC NULLS FIRST, ca_suite_number#44 ASC NULLS FIRST, ca_city#45 ASC NULLS FIRST, ca_county#46 ASC NULLS FIRST, ca_state#47 ASC NULLS FIRST, ca_zip#48 ASC NULLS FIRST, ca_country#49 ASC NULLS FIRST, ca_gmt_offset#50 ASC NULLS FIRST, ca_location_type#51 ASC NULLS FIRST, ctr_total_return#15 ASC NULLS FIRST], [c_customer_id#35, c_salutation#37, c_first_name#38, c_last_name#39, ca_street_number#41, ca_street_name#42, ca_street_type#43, ca_suite_number#44, ca_city#45, ca_county#46, ca_state#47, ca_zip#48, ca_country#49, ca_gmt_offset#50, ca_location_type#51, ctr_total_return#15] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cr_returned_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (61) -+- * ColumnarToRow (60) - +- CometProject (59) - +- CometFilter (58) - +- CometScan parquet spark_catalog.default.date_dim (57) +BroadcastExchange (55) ++- * ColumnarToRow (54) + +- CometProject (53) + +- CometFilter (52) + +- CometScan parquet spark_catalog.default.date_dim (51) -(57) Scan parquet spark_catalog.default.date_dim +(51) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_year#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(58) CometFilter +(52) CometFilter Input [2]: [d_date_sk#6, d_year#7] Condition : ((isnotnull(d_year#7) AND (d_year#7 = 2000)) AND isnotnull(d_date_sk#6)) -(59) CometProject +(53) CometProject Input [2]: [d_date_sk#6, d_year#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(60) ColumnarToRow [codegen id : 1] +(54) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(61) BroadcastExchange +(55) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] -Subquery:2 Hosting operator id = 21 Hosting Expression = cr_returned_date_sk#19 IN dynamicpruning#5 +Subquery:2 Hosting operator id = 19 Hosting Expression = cr_returned_date_sk#19 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt index da54f3a03..991c00de9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q81/simplified.txt @@ -8,64 +8,58 @@ TakeOrderedAndProject [c_customer_id,c_salutation,c_first_name,c_last_name,ca_st BroadcastHashJoin [ctr_state,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2)] Filter [ctr_total_return] HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_customer_sk,ctr_state,ctr_total_return,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [cr_returning_customer_sk,ca_state] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] - CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] - CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [ca_address_sk,ca_state] #4 - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + Exchange [cr_returning_customer_sk,ca_state] #1 + WholeStageCodegen (1) + HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] + CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] + CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [ca_address_sk,ca_state] #4 + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] InputAdapter BroadcastExchange #5 WholeStageCodegen (4) Filter [(avg(ctr_total_return) * 1.2)] HashAggregate [ctr_state,sum,count] [avg(ctr_total_return),(avg(ctr_total_return) * 1.2),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ctr_state] #6 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] - HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [cr_returning_customer_sk,ca_state] #7 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] - CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] - CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] - CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #3 - ReusedExchange [ca_address_sk,ca_state] #4 + InputAdapter + Exchange [ctr_state] #6 + WholeStageCodegen (3) + HashAggregate [ctr_state,ctr_total_return] [sum,count,sum,count] + HashAggregate [cr_returning_customer_sk,ca_state,sum] [sum(UnscaledValue(cr_return_amt_inc_tax)),ctr_state,ctr_total_return,sum] + InputAdapter + Exchange [cr_returning_customer_sk,ca_state] #7 + WholeStageCodegen (2) + HashAggregate [cr_returning_customer_sk,ca_state,cr_return_amt_inc_tax] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] + CometProject [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax] + CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk,d_date_sk] + CometFilter [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #3 + ReusedExchange [ca_address_sk,ca_state] #4 InputAdapter BroadcastExchange #8 WholeStageCodegen (5) diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt index 2519fbc97..774b6b112 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometBroadcastHashJoin (20) @@ -132,9 +132,9 @@ Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Functions: [] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt index 869073de1..42c3f40e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q82/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometColumnarExchange [i_item_id,i_item_desc,i_current_price] #1 + CometExchange [i_item_id,i_item_desc,i_current_price] #1 CometHashAggregate [i_item_id,i_item_desc,i_current_price] CometProject [i_item_id,i_item_desc,i_current_price] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,ss_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt index 49ee36742..d9d319813 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/explain.txt @@ -1,60 +1,54 @@ == Physical Plan == -TakeOrderedAndProject (56) -+- * Project (55) - +- * BroadcastHashJoin Inner BuildRight (54) - :- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * HashAggregate (21) - : : +- * ColumnarToRow (20) - : : +- CometColumnarExchange (19) - : : +- RowToColumnar (18) - : : +- * HashAggregate (17) - : : +- * ColumnarToRow (16) - : : +- CometProject (15) - : : +- CometBroadcastHashJoin (14) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_returns (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.item (3) - : : +- CometBroadcastExchange (13) - : : +- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- ReusedExchange (10) - : +- BroadcastExchange (36) - : +- * HashAggregate (35) - : +- * ColumnarToRow (34) - : +- CometColumnarExchange (33) - : +- RowToColumnar (32) - : +- * HashAggregate (31) - : +- * ColumnarToRow (30) - : +- CometProject (29) - : +- CometBroadcastHashJoin (28) - : :- CometProject (26) - : : +- CometBroadcastHashJoin (25) - : : :- CometFilter (23) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (22) - : : +- ReusedExchange (24) - : +- ReusedExchange (27) - +- BroadcastExchange (53) - +- * HashAggregate (52) - +- * ColumnarToRow (51) - +- CometColumnarExchange (50) - +- RowToColumnar (49) - +- * HashAggregate (48) - +- * ColumnarToRow (47) - +- CometProject (46) - +- CometBroadcastHashJoin (45) - :- CometProject (43) - : +- CometBroadcastHashJoin (42) - : :- CometFilter (40) - : : +- CometScan parquet spark_catalog.default.web_returns (39) - : +- ReusedExchange (41) - +- ReusedExchange (44) +TakeOrderedAndProject (50) ++- * Project (49) + +- * BroadcastHashJoin Inner BuildRight (48) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * HashAggregate (19) + : : +- Exchange (18) + : : +- * HashAggregate (17) + : : +- * ColumnarToRow (16) + : : +- CometProject (15) + : : +- CometBroadcastHashJoin (14) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_returns (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.item (3) + : : +- CometBroadcastExchange (13) + : : +- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- ReusedExchange (10) + : +- BroadcastExchange (32) + : +- * HashAggregate (31) + : +- Exchange (30) + : +- * HashAggregate (29) + : +- * ColumnarToRow (28) + : +- CometProject (27) + : +- CometBroadcastHashJoin (26) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometFilter (21) + : : : +- CometScan parquet spark_catalog.default.catalog_returns (20) + : : +- ReusedExchange (22) + : +- ReusedExchange (25) + +- BroadcastExchange (47) + +- * HashAggregate (46) + +- Exchange (45) + +- * HashAggregate (44) + +- * ColumnarToRow (43) + +- CometProject (42) + +- CometBroadcastHashJoin (41) + :- CometProject (39) + : +- CometBroadcastHashJoin (38) + : :- CometFilter (36) + : : +- CometScan parquet spark_catalog.default.web_returns (35) + : +- ReusedExchange (37) + +- ReusedExchange (40) (1) Scan parquet spark_catalog.default.store_returns @@ -104,7 +98,7 @@ ReadSchema: struct Input [2]: [d_date_sk#7, d_date#8] Condition : isnotnull(d_date_sk#7) -(10) ReusedExchange [Reuses operator id: 66] +(10) ReusedExchange [Reuses operator id: 60] Output [1]: [d_date#9] (11) CometBroadcastHashJoin @@ -139,24 +133,18 @@ Functions [1]: [partial_sum(sr_return_quantity#2)] Aggregate Attributes [1]: [sum#10] Results [2]: [i_item_id#6, sum#11] -(18) RowToColumnar +(18) Exchange Input [2]: [i_item_id#6, sum#11] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(19) CometColumnarExchange -Input [2]: [i_item_id#6, sum#11] -Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(20) ColumnarToRow [codegen id : 6] -Input [2]: [i_item_id#6, sum#11] - -(21) HashAggregate [codegen id : 6] +(19) HashAggregate [codegen id : 6] Input [2]: [i_item_id#6, sum#11] Keys [1]: [i_item_id#6] Functions [1]: [sum(sr_return_quantity#2)] Aggregate Attributes [1]: [sum(sr_return_quantity#2)#12] Results [2]: [i_item_id#6 AS item_id#13, sum(sr_return_quantity#2)#12 AS sr_item_qty#14] -(22) Scan parquet spark_catalog.default.catalog_returns +(20) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#15, cr_return_quantity#16, cr_returned_date_sk#17] Batched: true Location: InMemoryFileIndex [] @@ -164,76 +152,70 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#17), dynamicpruningexpression(c PushedFilters: [IsNotNull(cr_item_sk)] ReadSchema: struct -(23) CometFilter +(21) CometFilter Input [3]: [cr_item_sk#15, cr_return_quantity#16, cr_returned_date_sk#17] Condition : isnotnull(cr_item_sk#15) -(24) ReusedExchange [Reuses operator id: 5] +(22) ReusedExchange [Reuses operator id: 5] Output [2]: [i_item_sk#19, i_item_id#20] -(25) CometBroadcastHashJoin +(23) CometBroadcastHashJoin Left output [3]: [cr_item_sk#15, cr_return_quantity#16, cr_returned_date_sk#17] Right output [2]: [i_item_sk#19, i_item_id#20] Arguments: [cr_item_sk#15], [i_item_sk#19], Inner, BuildRight -(26) CometProject +(24) CometProject Input [5]: [cr_item_sk#15, cr_return_quantity#16, cr_returned_date_sk#17, i_item_sk#19, i_item_id#20] Arguments: [cr_return_quantity#16, cr_returned_date_sk#17, i_item_id#20], [cr_return_quantity#16, cr_returned_date_sk#17, i_item_id#20] -(27) ReusedExchange [Reuses operator id: 13] +(25) ReusedExchange [Reuses operator id: 13] Output [1]: [d_date_sk#21] -(28) CometBroadcastHashJoin +(26) CometBroadcastHashJoin Left output [3]: [cr_return_quantity#16, cr_returned_date_sk#17, i_item_id#20] Right output [1]: [d_date_sk#21] Arguments: [cr_returned_date_sk#17], [d_date_sk#21], Inner, BuildRight -(29) CometProject +(27) CometProject Input [4]: [cr_return_quantity#16, cr_returned_date_sk#17, i_item_id#20, d_date_sk#21] Arguments: [cr_return_quantity#16, i_item_id#20], [cr_return_quantity#16, i_item_id#20] -(30) ColumnarToRow [codegen id : 2] +(28) ColumnarToRow [codegen id : 2] Input [2]: [cr_return_quantity#16, i_item_id#20] -(31) HashAggregate [codegen id : 2] +(29) HashAggregate [codegen id : 2] Input [2]: [cr_return_quantity#16, i_item_id#20] Keys [1]: [i_item_id#20] Functions [1]: [partial_sum(cr_return_quantity#16)] Aggregate Attributes [1]: [sum#22] Results [2]: [i_item_id#20, sum#23] -(32) RowToColumnar -Input [2]: [i_item_id#20, sum#23] - -(33) CometColumnarExchange -Input [2]: [i_item_id#20, sum#23] -Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(34) ColumnarToRow [codegen id : 3] +(30) Exchange Input [2]: [i_item_id#20, sum#23] +Arguments: hashpartitioning(i_item_id#20, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(35) HashAggregate [codegen id : 3] +(31) HashAggregate [codegen id : 3] Input [2]: [i_item_id#20, sum#23] Keys [1]: [i_item_id#20] Functions [1]: [sum(cr_return_quantity#16)] Aggregate Attributes [1]: [sum(cr_return_quantity#16)#24] Results [2]: [i_item_id#20 AS item_id#25, sum(cr_return_quantity#16)#24 AS cr_item_qty#26] -(36) BroadcastExchange +(32) BroadcastExchange Input [2]: [item_id#25, cr_item_qty#26] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(37) BroadcastHashJoin [codegen id : 6] +(33) BroadcastHashJoin [codegen id : 6] Left keys [1]: [item_id#13] Right keys [1]: [item_id#25] Join type: Inner Join condition: None -(38) Project [codegen id : 6] +(34) Project [codegen id : 6] Output [3]: [item_id#13, sr_item_qty#14, cr_item_qty#26] Input [4]: [item_id#13, sr_item_qty#14, item_id#25, cr_item_qty#26] -(39) Scan parquet spark_catalog.default.web_returns +(35) Scan parquet spark_catalog.default.web_returns Output [3]: [wr_item_sk#27, wr_return_quantity#28, wr_returned_date_sk#29] Batched: true Location: InMemoryFileIndex [] @@ -241,165 +223,159 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#29), dynamicpruningexpression(w PushedFilters: [IsNotNull(wr_item_sk)] ReadSchema: struct -(40) CometFilter +(36) CometFilter Input [3]: [wr_item_sk#27, wr_return_quantity#28, wr_returned_date_sk#29] Condition : isnotnull(wr_item_sk#27) -(41) ReusedExchange [Reuses operator id: 5] +(37) ReusedExchange [Reuses operator id: 5] Output [2]: [i_item_sk#31, i_item_id#32] -(42) CometBroadcastHashJoin +(38) CometBroadcastHashJoin Left output [3]: [wr_item_sk#27, wr_return_quantity#28, wr_returned_date_sk#29] Right output [2]: [i_item_sk#31, i_item_id#32] Arguments: [wr_item_sk#27], [i_item_sk#31], Inner, BuildRight -(43) CometProject +(39) CometProject Input [5]: [wr_item_sk#27, wr_return_quantity#28, wr_returned_date_sk#29, i_item_sk#31, i_item_id#32] Arguments: [wr_return_quantity#28, wr_returned_date_sk#29, i_item_id#32], [wr_return_quantity#28, wr_returned_date_sk#29, i_item_id#32] -(44) ReusedExchange [Reuses operator id: 13] +(40) ReusedExchange [Reuses operator id: 13] Output [1]: [d_date_sk#33] -(45) CometBroadcastHashJoin +(41) CometBroadcastHashJoin Left output [3]: [wr_return_quantity#28, wr_returned_date_sk#29, i_item_id#32] Right output [1]: [d_date_sk#33] Arguments: [wr_returned_date_sk#29], [d_date_sk#33], Inner, BuildRight -(46) CometProject +(42) CometProject Input [4]: [wr_return_quantity#28, wr_returned_date_sk#29, i_item_id#32, d_date_sk#33] Arguments: [wr_return_quantity#28, i_item_id#32], [wr_return_quantity#28, i_item_id#32] -(47) ColumnarToRow [codegen id : 4] +(43) ColumnarToRow [codegen id : 4] Input [2]: [wr_return_quantity#28, i_item_id#32] -(48) HashAggregate [codegen id : 4] +(44) HashAggregate [codegen id : 4] Input [2]: [wr_return_quantity#28, i_item_id#32] Keys [1]: [i_item_id#32] Functions [1]: [partial_sum(wr_return_quantity#28)] Aggregate Attributes [1]: [sum#34] Results [2]: [i_item_id#32, sum#35] -(49) RowToColumnar -Input [2]: [i_item_id#32, sum#35] - -(50) CometColumnarExchange +(45) Exchange Input [2]: [i_item_id#32, sum#35] -Arguments: hashpartitioning(i_item_id#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_item_id#32, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(51) ColumnarToRow [codegen id : 5] -Input [2]: [i_item_id#32, sum#35] - -(52) HashAggregate [codegen id : 5] +(46) HashAggregate [codegen id : 5] Input [2]: [i_item_id#32, sum#35] Keys [1]: [i_item_id#32] Functions [1]: [sum(wr_return_quantity#28)] Aggregate Attributes [1]: [sum(wr_return_quantity#28)#36] Results [2]: [i_item_id#32 AS item_id#37, sum(wr_return_quantity#28)#36 AS wr_item_qty#38] -(53) BroadcastExchange +(47) BroadcastExchange Input [2]: [item_id#37, wr_item_qty#38] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(54) BroadcastHashJoin [codegen id : 6] +(48) BroadcastHashJoin [codegen id : 6] Left keys [1]: [item_id#13] Right keys [1]: [item_id#37] Join type: Inner Join condition: None -(55) Project [codegen id : 6] +(49) Project [codegen id : 6] Output [8]: [item_id#13, sr_item_qty#14, (((cast(sr_item_qty#14 as double) / cast(((sr_item_qty#14 + cr_item_qty#26) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS sr_dev#39, cr_item_qty#26, (((cast(cr_item_qty#26 as double) / cast(((sr_item_qty#14 + cr_item_qty#26) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS cr_dev#40, wr_item_qty#38, (((cast(wr_item_qty#38 as double) / cast(((sr_item_qty#14 + cr_item_qty#26) + wr_item_qty#38) as double)) / 3.0) * 100.0) AS wr_dev#41, (cast(((sr_item_qty#14 + cr_item_qty#26) + wr_item_qty#38) as decimal(20,0)) / 3.0) AS average#42] Input [5]: [item_id#13, sr_item_qty#14, cr_item_qty#26, item_id#37, wr_item_qty#38] -(56) TakeOrderedAndProject +(50) TakeOrderedAndProject Input [8]: [item_id#13, sr_item_qty#14, sr_dev#39, cr_item_qty#26, cr_dev#40, wr_item_qty#38, wr_dev#41, average#42] Arguments: 100, [item_id#13 ASC NULLS FIRST, sr_item_qty#14 ASC NULLS FIRST], [item_id#13, sr_item_qty#14, sr_dev#39, cr_item_qty#26, cr_dev#40, wr_item_qty#38, wr_dev#41, average#42] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = sr_returned_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (70) -+- * ColumnarToRow (69) - +- CometProject (68) - +- CometBroadcastHashJoin (67) - :- CometFilter (58) - : +- CometScan parquet spark_catalog.default.date_dim (57) - +- CometBroadcastExchange (66) - +- CometProject (65) - +- CometBroadcastHashJoin (64) - :- CometScan parquet spark_catalog.default.date_dim (59) - +- CometBroadcastExchange (63) - +- CometProject (62) - +- CometFilter (61) - +- CometScan parquet spark_catalog.default.date_dim (60) - - -(57) Scan parquet spark_catalog.default.date_dim +BroadcastExchange (64) ++- * ColumnarToRow (63) + +- CometProject (62) + +- CometBroadcastHashJoin (61) + :- CometFilter (52) + : +- CometScan parquet spark_catalog.default.date_dim (51) + +- CometBroadcastExchange (60) + +- CometProject (59) + +- CometBroadcastHashJoin (58) + :- CometScan parquet spark_catalog.default.date_dim (53) + +- CometBroadcastExchange (57) + +- CometProject (56) + +- CometFilter (55) + +- CometScan parquet spark_catalog.default.date_dim (54) + + +(51) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#7, d_date#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(58) CometFilter +(52) CometFilter Input [2]: [d_date_sk#7, d_date#8] Condition : isnotnull(d_date_sk#7) -(59) Scan parquet spark_catalog.default.date_dim +(53) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date#9, d_week_seq#43] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] ReadSchema: struct -(60) Scan parquet spark_catalog.default.date_dim +(54) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date#44, d_week_seq#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [In(d_date, [2000-06-30,2000-09-27,2000-11-17])] ReadSchema: struct -(61) CometFilter +(55) CometFilter Input [2]: [d_date#44, d_week_seq#45] Condition : d_date#44 IN (2000-06-30,2000-09-27,2000-11-17) -(62) CometProject +(56) CometProject Input [2]: [d_date#44, d_week_seq#45] Arguments: [d_week_seq#45], [d_week_seq#45] -(63) CometBroadcastExchange +(57) CometBroadcastExchange Input [1]: [d_week_seq#45] Arguments: [d_week_seq#45] -(64) CometBroadcastHashJoin +(58) CometBroadcastHashJoin Left output [2]: [d_date#9, d_week_seq#43] Right output [1]: [d_week_seq#45] Arguments: [d_week_seq#43], [d_week_seq#45], LeftSemi, BuildRight -(65) CometProject +(59) CometProject Input [2]: [d_date#9, d_week_seq#43] Arguments: [d_date#9], [d_date#9] -(66) CometBroadcastExchange +(60) CometBroadcastExchange Input [1]: [d_date#9] Arguments: [d_date#9] -(67) CometBroadcastHashJoin +(61) CometBroadcastHashJoin Left output [2]: [d_date_sk#7, d_date#8] Right output [1]: [d_date#9] Arguments: [d_date#8], [d_date#9], LeftSemi, BuildRight -(68) CometProject +(62) CometProject Input [2]: [d_date_sk#7, d_date#8] Arguments: [d_date_sk#7], [d_date_sk#7] -(69) ColumnarToRow [codegen id : 1] +(63) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#7] -(70) BroadcastExchange +(64) BroadcastExchange Input [1]: [d_date_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -Subquery:2 Hosting operator id = 22 Hosting Expression = cr_returned_date_sk#17 IN dynamicpruning#4 +Subquery:2 Hosting operator id = 20 Hosting Expression = cr_returned_date_sk#17 IN dynamicpruning#4 -Subquery:3 Hosting operator id = 39 Hosting Expression = wr_returned_date_sk#29 IN dynamicpruning#4 +Subquery:3 Hosting operator id = 35 Hosting Expression = wr_returned_date_sk#29 IN dynamicpruning#4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt index 505ef9b5c..8cd3f3f45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q83.ansi/simplified.txt @@ -5,85 +5,79 @@ TakeOrderedAndProject [item_id,sr_item_qty,sr_dev,cr_item_qty,cr_dev,wr_item_qty Project [item_id,sr_item_qty,cr_item_qty] BroadcastHashJoin [item_id,item_id] HashAggregate [i_item_id,sum] [sum(sr_return_quantity),item_id,sr_item_qty,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,sr_return_quantity] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_return_quantity,i_item_id] - CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] - CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] - CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] - CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date] #3 - CometProject [d_date] - CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [d_week_seq] #4 - CometProject [d_week_seq] - CometFilter [d_date,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] - CometBroadcastExchange [i_item_sk,i_item_id] #5 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometBroadcastHashJoin [d_date_sk,d_date,d_date] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - ReusedExchange [d_date] #3 + InputAdapter + Exchange [i_item_id] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,sr_return_quantity] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [sr_return_quantity,i_item_id] + CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] + CometProject [sr_return_quantity,sr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [sr_item_sk,sr_return_quantity,sr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_return_quantity,sr_returned_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date] #3 + CometProject [d_date] + CometBroadcastHashJoin [d_date,d_week_seq,d_week_seq] + CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [d_week_seq] #4 + CometProject [d_week_seq] + CometFilter [d_date,d_week_seq] + CometScan parquet spark_catalog.default.date_dim [d_date,d_week_seq] + CometBroadcastExchange [i_item_sk,i_item_id] #5 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometBroadcastHashJoin [d_date_sk,d_date,d_date] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + ReusedExchange [d_date] #3 InputAdapter BroadcastExchange #7 WholeStageCodegen (3) HashAggregate [i_item_id,sum] [sum(cr_return_quantity),item_id,cr_item_qty,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #8 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_item_id,cr_return_quantity] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cr_return_quantity,i_item_id] - CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] - CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] - CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] - CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_item_id] #5 - ReusedExchange [d_date_sk] #6 - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (5) - HashAggregate [i_item_id,sum] [sum(wr_return_quantity),item_id,wr_item_qty,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #10 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [i_item_id,wr_return_quantity] [sum,sum] + InputAdapter + Exchange [i_item_id] #8 + WholeStageCodegen (2) + HashAggregate [i_item_id,cr_return_quantity] [sum,sum] ColumnarToRow InputAdapter - CometProject [wr_return_quantity,i_item_id] - CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] - CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] - CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] - CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometProject [cr_return_quantity,i_item_id] + CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] + CometProject [cr_return_quantity,cr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [cr_item_sk,cr_return_quantity,cr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [cr_item_sk,cr_return_quantity,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_return_quantity,cr_returned_date_sk] ReusedSubquery [d_date_sk] #1 ReusedExchange [i_item_sk,i_item_id] #5 ReusedExchange [d_date_sk] #6 + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (5) + HashAggregate [i_item_id,sum] [sum(wr_return_quantity),item_id,wr_item_qty,sum] + InputAdapter + Exchange [i_item_id] #10 + WholeStageCodegen (4) + HashAggregate [i_item_id,wr_return_quantity] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [wr_return_quantity,i_item_id] + CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] + CometProject [wr_return_quantity,wr_returned_date_sk,i_item_id] + CometBroadcastHashJoin [wr_item_sk,wr_return_quantity,wr_returned_date_sk,i_item_sk,i_item_id] + CometFilter [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_return_quantity,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [i_item_sk,i_item_id] #5 + ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt index d02f4cc80..c8becd0e9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/explain.txt @@ -1,51 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- * ColumnarToRow (45) - +- CometColumnarExchange (44) - +- RowToColumnar (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometProject (40) - +- CometBroadcastHashJoin (39) - :- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (29) - : : +- CometBroadcastHashJoin (28) - : : :- CometProject (23) - : : : +- CometBroadcastHashJoin (22) - : : : :- CometProject (18) - : : : : +- CometBroadcastHashJoin (17) - : : : : :- CometProject (13) - : : : : : +- CometBroadcastHashJoin (12) - : : : : : :- CometProject (8) - : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : :- CometBroadcastExchange (3) - : : : : : : : +- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : : : +- CometProject (6) - : : : : : : +- CometFilter (5) - : : : : : : +- CometScan parquet spark_catalog.default.web_returns (4) - : : : : : +- CometBroadcastExchange (11) - : : : : : +- CometFilter (10) - : : : : : +- CometScan parquet spark_catalog.default.web_page (9) - : : : : +- CometBroadcastExchange (16) - : : : : +- CometFilter (15) - : : : : +- CometScan parquet spark_catalog.default.customer_demographics (14) - : : : +- CometBroadcastExchange (21) - : : : +- CometFilter (20) - : : : +- CometScan parquet spark_catalog.default.customer_demographics (19) - : : +- CometBroadcastExchange (27) - : : +- CometProject (26) - : : +- CometFilter (25) - : : +- CometScan parquet spark_catalog.default.customer_address (24) - : +- CometBroadcastExchange (33) - : +- CometProject (32) - : +- CometFilter (31) - : +- CometScan parquet spark_catalog.default.date_dim (30) - +- CometBroadcastExchange (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.reason (36) +TakeOrderedAndProject (45) ++- * HashAggregate (44) + +- Exchange (43) + +- * HashAggregate (42) + +- * ColumnarToRow (41) + +- CometProject (40) + +- CometBroadcastHashJoin (39) + :- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (23) + : : : +- CometBroadcastHashJoin (22) + : : : :- CometProject (18) + : : : : +- CometBroadcastHashJoin (17) + : : : : :- CometProject (13) + : : : : : +- CometBroadcastHashJoin (12) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometBroadcastExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : : : +- CometProject (6) + : : : : : : +- CometFilter (5) + : : : : : : +- CometScan parquet spark_catalog.default.web_returns (4) + : : : : : +- CometBroadcastExchange (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan parquet spark_catalog.default.web_page (9) + : : : : +- CometBroadcastExchange (16) + : : : : +- CometFilter (15) + : : : : +- CometScan parquet spark_catalog.default.customer_demographics (14) + : : : +- CometBroadcastExchange (21) + : : : +- CometFilter (20) + : : : +- CometScan parquet spark_catalog.default.customer_demographics (19) + : : +- CometBroadcastExchange (27) + : : +- CometProject (26) + : : +- CometFilter (25) + : : +- CometScan parquet spark_catalog.default.customer_address (24) + : +- CometBroadcastExchange (33) + : +- CometProject (32) + : +- CometFilter (31) + : +- CometScan parquet spark_catalog.default.date_dim (30) + +- CometBroadcastExchange (38) + +- CometFilter (37) + +- CometScan parquet spark_catalog.default.reason (36) (1) Scan parquet spark_catalog.default.web_sales @@ -250,56 +248,50 @@ Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunde Aggregate Attributes [6]: [sum#32, count#33, sum#34, count#35, sum#36, count#37] Results [7]: [r_reason_desc#31, sum#38, count#39, sum#40, count#41, sum#42, count#43] -(43) RowToColumnar +(43) Exchange Input [7]: [r_reason_desc#31, sum#38, count#39, sum#40, count#41, sum#42, count#43] +Arguments: hashpartitioning(r_reason_desc#31, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(44) CometColumnarExchange -Input [7]: [r_reason_desc#31, sum#38, count#39, sum#40, count#41, sum#42, count#43] -Arguments: hashpartitioning(r_reason_desc#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(45) ColumnarToRow [codegen id : 2] -Input [7]: [r_reason_desc#31, sum#38, count#39, sum#40, count#41, sum#42, count#43] - -(46) HashAggregate [codegen id : 2] +(44) HashAggregate [codegen id : 2] Input [7]: [r_reason_desc#31, sum#38, count#39, sum#40, count#41, sum#42, count#43] Keys [1]: [r_reason_desc#31] Functions [3]: [avg(ws_quantity#4), avg(UnscaledValue(wr_refunded_cash#16)), avg(UnscaledValue(wr_fee#15))] Aggregate Attributes [3]: [avg(ws_quantity#4)#44, avg(UnscaledValue(wr_refunded_cash#16))#45, avg(UnscaledValue(wr_fee#15))#46] Results [4]: [substr(r_reason_desc#31, 1, 20) AS substr(r_reason_desc, 1, 20)#47, avg(ws_quantity#4)#44 AS avg(ws_quantity)#48, cast((avg(UnscaledValue(wr_refunded_cash#16))#45 / 100.0) as decimal(11,6)) AS avg(wr_refunded_cash)#49, cast((avg(UnscaledValue(wr_fee#15))#46 / 100.0) as decimal(11,6)) AS avg(wr_fee)#50] -(47) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [4]: [substr(r_reason_desc, 1, 20)#47, avg(ws_quantity)#48, avg(wr_refunded_cash)#49, avg(wr_fee)#50] Arguments: 100, [substr(r_reason_desc, 1, 20)#47 ASC NULLS FIRST, avg(ws_quantity)#48 ASC NULLS FIRST, avg(wr_refunded_cash)#49 ASC NULLS FIRST, avg(wr_fee)#50 ASC NULLS FIRST], [substr(r_reason_desc, 1, 20)#47, avg(ws_quantity)#48, avg(wr_refunded_cash)#49, avg(wr_fee)#50] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * ColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan parquet spark_catalog.default.date_dim (48) +BroadcastExchange (50) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(48) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#28, d_year#29] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(49) CometFilter +(47) CometFilter Input [2]: [d_date_sk#28, d_year#29] Condition : ((isnotnull(d_year#29) AND (d_year#29 = 2000)) AND isnotnull(d_date_sk#28)) -(50) CometProject +(48) CometProject Input [2]: [d_date_sk#28, d_year#29] Arguments: [d_date_sk#28], [d_date_sk#28] -(51) ColumnarToRow [codegen id : 1] +(49) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#28] -(52) BroadcastExchange +(50) BroadcastExchange Input [1]: [d_date_sk#28] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt index af70cc827..0d350bd59 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q85/simplified.txt @@ -1,59 +1,57 @@ TakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] WholeStageCodegen (2) HashAggregate [r_reason_desc,sum,count,sum,count,sum,count] [avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee)),substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [r_reason_desc] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [r_reason_desc,ws_quantity,wr_refunded_cash,wr_fee] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] - CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] - CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash,d_date_sk] - CometProject [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash,ca_address_sk,ca_state] - CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,wp_web_page_sk] - CometProject [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] - CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 - CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] - CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] - CometBroadcastExchange [wp_web_page_sk] #4 - CometFilter [wp_web_page_sk] - CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 + InputAdapter + Exchange [r_reason_desc] #1 + WholeStageCodegen (1) + HashAggregate [r_reason_desc,ws_quantity,wr_refunded_cash,wr_fee] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] + CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] + CometProject [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash,d_date_sk] + CometProject [ws_quantity,ws_sold_date_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash,ca_address_sk,ca_state] + CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ws_quantity,ws_net_profit,ws_sold_date_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_fee,wr_refunded_cash] + CometBroadcastHashJoin [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometBroadcastExchange [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] #2 + CometFilter [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_page_sk,ws_order_number,ws_quantity,ws_sales_price,ws_net_profit,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash] + CometFilter [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_refunded_cdemo_sk,wr_refunded_addr_sk,wr_returning_cdemo_sk,wr_reason_sk,wr_order_number,wr_fee,wr_refunded_cash,wr_returned_date_sk] + CometBroadcastExchange [wp_web_page_sk] #4 + CometFilter [wp_web_page_sk] + CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #5 CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [ca_address_sk,ca_state] #7 - CometProject [ca_address_sk,ca_state] - CometFilter [ca_address_sk,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #8 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [r_reason_sk,r_reason_desc] #9 - CometFilter [r_reason_sk,r_reason_desc] - CometScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #6 + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [ca_address_sk,ca_state] #7 + CometProject [ca_address_sk,ca_state] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #8 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [r_reason_sk,r_reason_desc] #9 + CometFilter [r_reason_sk,r_reason_desc] + CometScan parquet spark_catalog.default.reason [r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt index 4300d3efc..1c9688ca2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/explain.txt @@ -1,31 +1,27 @@ == Physical Plan == -TakeOrderedAndProject (27) -+- * Project (26) - +- Window (25) - +- * ColumnarToRow (24) - +- CometSort (23) - +- CometColumnarExchange (22) - +- RowToColumnar (21) - +- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometColumnarExchange (18) - +- RowToColumnar (17) - +- * HashAggregate (16) - +- * ColumnarToRow (15) - +- CometExpand (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.web_sales (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.date_dim (3) - +- CometBroadcastExchange (11) - +- CometFilter (10) - +- CometScan parquet spark_catalog.default.item (9) +TakeOrderedAndProject (23) ++- * Project (22) + +- Window (21) + +- * Sort (20) + +- Exchange (19) + +- * HashAggregate (18) + +- Exchange (17) + +- * HashAggregate (16) + +- * ColumnarToRow (15) + +- CometExpand (14) + +- CometProject (13) + +- CometBroadcastHashJoin (12) + :- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.date_dim (3) + +- CometBroadcastExchange (11) + +- CometFilter (10) + +- CometScan parquet spark_catalog.default.item (9) (1) Scan parquet spark_catalog.default.web_sales @@ -106,78 +102,66 @@ Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] Aggregate Attributes [1]: [sum#13] Results [4]: [i_category#10, i_class#11, spark_grouping_id#12, sum#14] -(17) RowToColumnar +(17) Exchange Input [4]: [i_category#10, i_class#11, spark_grouping_id#12, sum#14] +Arguments: hashpartitioning(i_category#10, i_class#11, spark_grouping_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(18) CometColumnarExchange -Input [4]: [i_category#10, i_class#11, spark_grouping_id#12, sum#14] -Arguments: hashpartitioning(i_category#10, i_class#11, spark_grouping_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(19) ColumnarToRow [codegen id : 2] -Input [4]: [i_category#10, i_class#11, spark_grouping_id#12, sum#14] - -(20) HashAggregate [codegen id : 2] +(18) HashAggregate [codegen id : 2] Input [4]: [i_category#10, i_class#11, spark_grouping_id#12, sum#14] Keys [3]: [i_category#10, i_class#11, spark_grouping_id#12] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#15] Results [7]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#15,17,2) AS total_sum#16, i_category#10, i_class#11, (cast((shiftright(spark_grouping_id#12, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#12, 0) & 1) as tinyint)) AS lochierarchy#17, MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#15,17,2) AS _w0#18, (cast((shiftright(spark_grouping_id#12, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#12, 0) & 1) as tinyint)) AS _w1#19, CASE WHEN (cast((shiftright(spark_grouping_id#12, 0) & 1) as tinyint) = 0) THEN i_category#10 END AS _w2#20] -(21) RowToColumnar -Input [7]: [total_sum#16, i_category#10, i_class#11, lochierarchy#17, _w0#18, _w1#19, _w2#20] - -(22) CometColumnarExchange -Input [7]: [total_sum#16, i_category#10, i_class#11, lochierarchy#17, _w0#18, _w1#19, _w2#20] -Arguments: hashpartitioning(_w1#19, _w2#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(23) CometSort +(19) Exchange Input [7]: [total_sum#16, i_category#10, i_class#11, lochierarchy#17, _w0#18, _w1#19, _w2#20] -Arguments: [total_sum#16, i_category#10, i_class#11, lochierarchy#17, _w0#18, _w1#19, _w2#20], [_w1#19 ASC NULLS FIRST, _w2#20 ASC NULLS FIRST, _w0#18 DESC NULLS LAST] +Arguments: hashpartitioning(_w1#19, _w2#20, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(24) ColumnarToRow [codegen id : 3] +(20) Sort [codegen id : 3] Input [7]: [total_sum#16, i_category#10, i_class#11, lochierarchy#17, _w0#18, _w1#19, _w2#20] +Arguments: [_w1#19 ASC NULLS FIRST, _w2#20 ASC NULLS FIRST, _w0#18 DESC NULLS LAST], false, 0 -(25) Window +(21) Window Input [7]: [total_sum#16, i_category#10, i_class#11, lochierarchy#17, _w0#18, _w1#19, _w2#20] Arguments: [rank(_w0#18) windowspecdefinition(_w1#19, _w2#20, _w0#18 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#21], [_w1#19, _w2#20], [_w0#18 DESC NULLS LAST] -(26) Project [codegen id : 4] +(22) Project [codegen id : 4] Output [5]: [total_sum#16, i_category#10, i_class#11, lochierarchy#17, rank_within_parent#21] Input [8]: [total_sum#16, i_category#10, i_class#11, lochierarchy#17, _w0#18, _w1#19, _w2#20, rank_within_parent#21] -(27) TakeOrderedAndProject +(23) TakeOrderedAndProject Input [5]: [total_sum#16, i_category#10, i_class#11, lochierarchy#17, rank_within_parent#21] Arguments: 100, [lochierarchy#17 DESC NULLS LAST, CASE WHEN (lochierarchy#17 = 0) THEN i_category#10 END ASC NULLS FIRST, rank_within_parent#21 ASC NULLS FIRST], [total_sum#16, i_category#10, i_class#11, lochierarchy#17, rank_within_parent#21] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (32) -+- * ColumnarToRow (31) - +- CometProject (30) - +- CometFilter (29) - +- CometScan parquet spark_catalog.default.date_dim (28) +BroadcastExchange (28) ++- * ColumnarToRow (27) + +- CometProject (26) + +- CometFilter (25) + +- CometScan parquet spark_catalog.default.date_dim (24) -(28) Scan parquet spark_catalog.default.date_dim +(24) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(29) CometFilter +(25) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) -(30) CometProject +(26) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(31) ColumnarToRow [codegen id : 1] +(27) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(32) BroadcastExchange +(28) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt index e8897d5fd..e4d028f2f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q86/simplified.txt @@ -4,40 +4,36 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (3) - ColumnarToRow + Sort [_w1,_w2,_w0] InputAdapter - CometSort [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2] - CometColumnarExchange [_w1,_w2] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class,spark_grouping_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_class,spark_grouping_id,ws_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] - CometProject [ws_net_paid,i_category,i_class] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] - CometProject [ws_item_sk,ws_net_paid] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #5 - CometFilter [i_item_sk,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + Exchange [_w1,_w2] #1 + WholeStageCodegen (2) + HashAggregate [i_category,i_class,spark_grouping_id,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + InputAdapter + Exchange [i_category,i_class,spark_grouping_id] #2 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,spark_grouping_id,ws_net_paid] [sum,sum] + ColumnarToRow + InputAdapter + CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] + CometProject [ws_net_paid,i_category,i_class] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] + CometProject [ws_item_sk,ws_net_paid] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_class,i_category] #5 + CometFilter [i_item_sk,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt index d7ea7098e..d023b5b31 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/explain.txt @@ -1,55 +1,53 @@ == Physical Plan == -* HashAggregate (51) -+- * ColumnarToRow (50) - +- CometColumnarExchange (49) - +- RowToColumnar (48) - +- * HashAggregate (47) - +- * Project (46) - +- * BroadcastHashJoin LeftAnti BuildRight (45) - :- * BroadcastHashJoin LeftAnti BuildRight (31) - : :- * ColumnarToRow (17) - : : +- CometHashAggregate (16) - : : +- CometColumnarExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer (9) - : +- BroadcastExchange (30) - : +- * ColumnarToRow (29) - : +- CometHashAggregate (28) - : +- CometColumnarExchange (27) - : +- CometHashAggregate (26) - : +- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometFilter (19) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) - : : +- ReusedExchange (20) - : +- ReusedExchange (23) - +- BroadcastExchange (44) - +- * ColumnarToRow (43) - +- CometHashAggregate (42) - +- CometColumnarExchange (41) - +- CometHashAggregate (40) - +- CometProject (39) - +- CometBroadcastHashJoin (38) - :- CometProject (36) - : +- CometBroadcastHashJoin (35) - : :- CometFilter (33) - : : +- CometScan parquet spark_catalog.default.web_sales (32) - : +- ReusedExchange (34) - +- ReusedExchange (37) +* HashAggregate (49) ++- Exchange (48) + +- * HashAggregate (47) + +- * Project (46) + +- * BroadcastHashJoin LeftAnti BuildRight (45) + :- * BroadcastHashJoin LeftAnti BuildRight (31) + : :- * ColumnarToRow (17) + : : +- CometHashAggregate (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.customer (9) + : +- BroadcastExchange (30) + : +- * ColumnarToRow (29) + : +- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- BroadcastExchange (44) + +- * ColumnarToRow (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometFilter (33) + : : +- CometScan parquet spark_catalog.default.web_sales (32) + : +- ReusedExchange (34) + +- ReusedExchange (37) (1) Scan parquet spark_catalog.default.store_sales @@ -121,9 +119,9 @@ Input [3]: [c_last_name#9, c_first_name#8, d_date#5] Keys [3]: [c_last_name#9, c_first_name#8, d_date#5] Functions: [] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [c_last_name#9, c_first_name#8, d_date#5] -Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [c_last_name#9, c_first_name#8, d_date#5] @@ -174,9 +172,9 @@ Input [3]: [c_last_name#17, c_first_name#16, d_date#14] Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(27) CometColumnarExchange +(27) CometExchange Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [3]: [c_last_name#17, c_first_name#16, d_date#14] @@ -237,9 +235,9 @@ Input [3]: [c_last_name#25, c_first_name#24, d_date#22] Keys [3]: [c_last_name#25, c_first_name#24, d_date#22] Functions: [] -(41) CometColumnarExchange +(41) CometExchange Input [3]: [c_last_name#25, c_first_name#24, d_date#22] -Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (42) CometHashAggregate Input [3]: [c_last_name#25, c_first_name#24, d_date#22] @@ -270,17 +268,11 @@ Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#26] Results [1]: [count#27] -(48) RowToColumnar +(48) Exchange Input [1]: [count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(49) CometColumnarExchange -Input [1]: [count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(50) ColumnarToRow [codegen id : 4] -Input [1]: [count#27] - -(51) HashAggregate [codegen id : 4] +(49) HashAggregate [codegen id : 4] Input [1]: [count#27] Keys: [] Functions [1]: [count(1)] @@ -290,32 +282,32 @@ Results [1]: [count(1)#28 AS count(1)#29] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 -BroadcastExchange (56) -+- * ColumnarToRow (55) - +- CometProject (54) - +- CometFilter (53) - +- CometScan parquet spark_catalog.default.date_dim (52) +BroadcastExchange (54) ++- * ColumnarToRow (53) + +- CometProject (52) + +- CometFilter (51) + +- CometScan parquet spark_catalog.default.date_dim (50) -(52) Scan parquet spark_catalog.default.date_dim +(50) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(53) CometFilter +(51) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) -(54) CometProject +(52) CometProject Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] -(55) ColumnarToRow [codegen id : 1] +(53) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#4, d_date#5] -(56) BroadcastExchange +(54) BroadcastExchange Input [2]: [d_date_sk#4, d_date#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt index 69f33a57a..6e3328ae1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q87/simplified.txt @@ -1,71 +1,69 @@ WholeStageCodegen (4) HashAggregate [count] [count(1),count(1),count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (3) + HashAggregate [count,count] + Project + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + ColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [d_date_sk,d_date] #4 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #7 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_date] #4 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #2 + CometExchange [c_last_name,c_first_name,d_date] #9 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ss_customer_sk,d_date] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #4 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #7 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [cs_bill_customer_sk,d_date] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] - CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #9 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ws_bill_customer_sk,d_date] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_date] #4 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt index cd9e4817c..1dface7ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/explain.txt @@ -8,7 +8,7 @@ : : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) : : : : : : :- * ColumnarToRow (25) : : : : : : : +- CometHashAggregate (24) -: : : : : : : +- CometColumnarExchange (23) +: : : : : : : +- CometExchange (23) : : : : : : : +- CometHashAggregate (22) : : : : : : : +- CometProject (21) : : : : : : : +- CometBroadcastHashJoin (20) @@ -34,7 +34,7 @@ : : : : : : +- BroadcastExchange (45) : : : : : : +- * ColumnarToRow (44) : : : : : : +- CometHashAggregate (43) -: : : : : : +- CometColumnarExchange (42) +: : : : : : +- CometExchange (42) : : : : : : +- CometHashAggregate (41) : : : : : : +- CometProject (40) : : : : : : +- CometBroadcastHashJoin (39) @@ -54,7 +54,7 @@ : : : : : +- BroadcastExchange (66) : : : : : +- * ColumnarToRow (65) : : : : : +- CometHashAggregate (64) -: : : : : +- CometColumnarExchange (63) +: : : : : +- CometExchange (63) : : : : : +- CometHashAggregate (62) : : : : : +- CometProject (61) : : : : : +- CometBroadcastHashJoin (60) @@ -74,7 +74,7 @@ : : : : +- BroadcastExchange (87) : : : : +- * ColumnarToRow (86) : : : : +- CometHashAggregate (85) -: : : : +- CometColumnarExchange (84) +: : : : +- CometExchange (84) : : : : +- CometHashAggregate (83) : : : : +- CometProject (82) : : : : +- CometBroadcastHashJoin (81) @@ -94,7 +94,7 @@ : : : +- BroadcastExchange (108) : : : +- * ColumnarToRow (107) : : : +- CometHashAggregate (106) -: : : +- CometColumnarExchange (105) +: : : +- CometExchange (105) : : : +- CometHashAggregate (104) : : : +- CometProject (103) : : : +- CometBroadcastHashJoin (102) @@ -114,7 +114,7 @@ : : +- BroadcastExchange (129) : : +- * ColumnarToRow (128) : : +- CometHashAggregate (127) -: : +- CometColumnarExchange (126) +: : +- CometExchange (126) : : +- CometHashAggregate (125) : : +- CometProject (124) : : +- CometBroadcastHashJoin (123) @@ -134,7 +134,7 @@ : +- BroadcastExchange (150) : +- * ColumnarToRow (149) : +- CometHashAggregate (148) -: +- CometColumnarExchange (147) +: +- CometExchange (147) : +- CometHashAggregate (146) : +- CometProject (145) : +- CometBroadcastHashJoin (144) @@ -154,7 +154,7 @@ +- BroadcastExchange (171) +- * ColumnarToRow (170) +- CometHashAggregate (169) - +- CometColumnarExchange (168) + +- CometExchange (168) +- CometHashAggregate (167) +- CometProject (166) +- CometBroadcastHashJoin (165) @@ -276,9 +276,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(23) CometColumnarExchange +(23) CometExchange Input [1]: [count#13] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [1]: [count#13] @@ -359,9 +359,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(42) CometColumnarExchange +(42) CometExchange Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (43) CometHashAggregate Input [1]: [count#24] @@ -450,9 +450,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(63) CometColumnarExchange +(63) CometExchange Input [1]: [count#35] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (64) CometHashAggregate Input [1]: [count#35] @@ -541,9 +541,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(84) CometColumnarExchange +(84) CometExchange Input [1]: [count#46] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (85) CometHashAggregate Input [1]: [count#46] @@ -632,9 +632,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(105) CometColumnarExchange +(105) CometExchange Input [1]: [count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (106) CometHashAggregate Input [1]: [count#57] @@ -723,9 +723,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(126) CometColumnarExchange +(126) CometExchange Input [1]: [count#68] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (127) CometHashAggregate Input [1]: [count#68] @@ -814,9 +814,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(147) CometColumnarExchange +(147) CometExchange Input [1]: [count#79] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (148) CometHashAggregate Input [1]: [count#79] @@ -905,9 +905,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(168) CometColumnarExchange +(168) CometExchange Input [1]: [count#90] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (169) CometHashAggregate Input [1]: [count#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt index b01685ed0..1bb61b6c9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q88/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h8_30_to_9,count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -38,7 +38,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h9_to_9_30,count,count(1)] - CometColumnarExchange #6 + CometExchange #6 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -61,7 +61,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h9_30_to_10,count,count(1)] - CometColumnarExchange #9 + CometExchange #9 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -84,7 +84,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h10_to_10_30,count,count(1)] - CometColumnarExchange #12 + CometExchange #12 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -107,7 +107,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h10_30_to_11,count,count(1)] - CometColumnarExchange #15 + CometExchange #15 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -130,7 +130,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h11_to_11_30,count,count(1)] - CometColumnarExchange #18 + CometExchange #18 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -153,7 +153,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h11_30_to_12,count,count(1)] - CometColumnarExchange #21 + CometExchange #21 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -176,7 +176,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h12_to_12_30,count,count(1)] - CometColumnarExchange #24 + CometExchange #24 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt index e4dca6fb4..07761b14b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/explain.txt @@ -1,36 +1,32 @@ == Physical Plan == -TakeOrderedAndProject (32) -+- * Project (31) - +- * Filter (30) - +- Window (29) - +- * ColumnarToRow (28) - +- CometSort (27) - +- CometColumnarExchange (26) - +- RowToColumnar (25) - +- * HashAggregate (24) - +- * ColumnarToRow (23) - +- CometColumnarExchange (22) - +- RowToColumnar (21) - +- * HashAggregate (20) - +- * ColumnarToRow (19) - +- CometProject (18) - +- CometBroadcastHashJoin (17) - :- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (7) - : : +- CometBroadcastHashJoin (6) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.item (1) - : : +- CometBroadcastExchange (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.store_sales (3) - : +- CometBroadcastExchange (11) - : +- CometProject (10) - : +- CometFilter (9) - : +- CometScan parquet spark_catalog.default.date_dim (8) - +- CometBroadcastExchange (16) - +- CometFilter (15) - +- CometScan parquet spark_catalog.default.store (14) +TakeOrderedAndProject (28) ++- * Project (27) + +- * Filter (26) + +- Window (25) + +- * Sort (24) + +- Exchange (23) + +- * HashAggregate (22) + +- Exchange (21) + +- * HashAggregate (20) + +- * ColumnarToRow (19) + +- CometProject (18) + +- CometBroadcastHashJoin (17) + :- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (7) + : : +- CometBroadcastHashJoin (6) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.item (1) + : : +- CometBroadcastExchange (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.store_sales (3) + : +- CometBroadcastExchange (11) + : +- CometProject (10) + : +- CometFilter (9) + : +- CometScan parquet spark_catalog.default.date_dim (8) + +- CometBroadcastExchange (16) + +- CometFilter (15) + +- CometScan parquet spark_catalog.default.store (14) (1) Scan parquet spark_catalog.default.item @@ -131,82 +127,70 @@ Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] Aggregate Attributes [1]: [sum#16] Results [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum#17] -(21) RowToColumnar +(21) Exchange Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum#17] +Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(22) CometColumnarExchange -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum#17] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(23) ColumnarToRow [codegen id : 2] -Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum#17] - -(24) HashAggregate [codegen id : 2] +(22) HashAggregate [codegen id : 2] Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum#17] Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12] Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#7))#18] Results [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#18,17,2) AS sum_sales#19, MakeDecimal(sum(UnscaledValue(ss_sales_price#7))#18,17,2) AS _w0#20] -(25) RowToColumnar -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, _w0#20] - -(26) CometColumnarExchange -Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, _w0#20] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(27) CometSort +(23) Exchange Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, _w0#20] -Arguments: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, _w0#20], [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST] +Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(28) ColumnarToRow [codegen id : 3] +(24) Sort [codegen id : 3] Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, _w0#20] +Arguments: [i_category#4 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST, s_company_name#15 ASC NULLS FIRST], false, 0 -(29) Window +(25) Window Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, _w0#20] Arguments: [avg(_w0#20) windowspecdefinition(i_category#4, i_brand#2, s_store_name#14, s_company_name#15, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#4, i_brand#2, s_store_name#14, s_company_name#15] -(30) Filter [codegen id : 4] +(26) Filter [codegen id : 4] Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, _w0#20, avg_monthly_sales#21] Condition : CASE WHEN NOT (avg_monthly_sales#21 = 0.000000) THEN ((abs((sum_sales#19 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END -(31) Project [codegen id : 4] +(27) Project [codegen id : 4] Output [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, avg_monthly_sales#21] Input [9]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, _w0#20, avg_monthly_sales#21] -(32) TakeOrderedAndProject +(28) TakeOrderedAndProject Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, avg_monthly_sales#21] Arguments: 100, [(sum_sales#19 - avg_monthly_sales#21) ASC NULLS FIRST, s_store_name#14 ASC NULLS FIRST], [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#19, avg_monthly_sales#21] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (37) -+- * ColumnarToRow (36) - +- CometProject (35) - +- CometFilter (34) - +- CometScan parquet spark_catalog.default.date_dim (33) +BroadcastExchange (33) ++- * ColumnarToRow (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan parquet spark_catalog.default.date_dim (29) -(33) Scan parquet spark_catalog.default.date_dim +(29) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(34) CometFilter +(30) CometFilter Input [3]: [d_date_sk#10, d_year#11, d_moy#12] Condition : ((isnotnull(d_year#11) AND (d_year#11 = 1999)) AND isnotnull(d_date_sk#10)) -(35) CometProject +(31) CometProject Input [3]: [d_date_sk#10, d_year#11, d_moy#12] Arguments: [d_date_sk#10, d_moy#12], [d_date_sk#10, d_moy#12] -(36) ColumnarToRow [codegen id : 1] +(32) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#10, d_moy#12] -(37) BroadcastExchange +(33) BroadcastExchange Input [2]: [d_date_sk#10, d_moy#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt index ec9252b0f..2bfff90d3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q89/simplified.txt @@ -5,44 +5,40 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla InputAdapter Window [_w0,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (3) - ColumnarToRow + Sort [i_category,i_brand,s_store_name,s_company_name] InputAdapter - CometSort [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] - CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] - CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] - CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] - CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_moy] #5 - CometProject [d_date_sk,d_moy] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 - CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + Exchange [i_category,i_brand,s_store_name,s_company_name] #1 + WholeStageCodegen (2) + HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + InputAdapter + Exchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,ss_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy] + CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_moy] + CometProject [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_class,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometFilter [i_item_sk,i_brand,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk,d_moy] #5 + CometProject [d_date_sk,d_moy] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt index b4a81109d..69c23211b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/explain.txt @@ -26,16 +26,14 @@ Input [1]: [r_reason_sk#1] ===== Subqueries ===== Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#2, [id=#3] -* Project (14) -+- * HashAggregate (13) - +- * ColumnarToRow (12) - +- CometColumnarExchange (11) - +- RowToColumnar (10) - +- * HashAggregate (9) - +- * ColumnarToRow (8) - +- CometProject (7) - +- CometFilter (6) - +- CometScan parquet spark_catalog.default.store_sales (5) +* Project (12) ++- * HashAggregate (11) + +- Exchange (10) + +- * HashAggregate (9) + +- * ColumnarToRow (8) + +- CometProject (7) + +- CometFilter (6) + +- CometScan parquet spark_catalog.default.store_sales (5) (5) Scan parquet spark_catalog.default.store_sales @@ -63,24 +61,18 @@ Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt# Aggregate Attributes [5]: [count#21, sum#22, count#23, sum#24, count#25] Results [5]: [count#26, sum#27, count#28, sum#29, count#30] -(10) RowToColumnar +(10) Exchange Input [5]: [count#26, sum#27, count#28, sum#29, count#30] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1] -(11) CometColumnarExchange -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(12) ColumnarToRow [codegen id : 2] -Input [5]: [count#26, sum#27, count#28, sum#29, count#30] - -(13) HashAggregate [codegen id : 2] +(11) HashAggregate [codegen id : 2] Input [5]: [count#26, sum#27, count#28, sum#29, count#30] Keys: [] Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#18)), avg(UnscaledValue(ss_net_paid#19))] Aggregate Attributes [3]: [count(1)#31, avg(UnscaledValue(ss_ext_discount_amt#18))#32, avg(UnscaledValue(ss_net_paid#19))#33] Results [3]: [count(1)#31 AS count(1)#34, cast((avg(UnscaledValue(ss_ext_discount_amt#18))#32 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#35, cast((avg(UnscaledValue(ss_net_paid#19))#33 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#36] -(14) Project [codegen id : 2] +(12) Project [codegen id : 2] Output [1]: [named_struct(count(1), count(1)#34, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#35, avg(ss_net_paid), avg(ss_net_paid)#36) AS mergedValue#37] Input [3]: [count(1)#34, avg(ss_ext_discount_amt)#35, avg(ss_net_paid)#36] @@ -89,61 +81,53 @@ Subquery:2 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery Subquery:3 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#2, [id=#3] Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#5, [id=#6] -* Project (24) -+- * HashAggregate (23) - +- * ColumnarToRow (22) - +- CometColumnarExchange (21) - +- RowToColumnar (20) - +- * HashAggregate (19) - +- * ColumnarToRow (18) - +- CometProject (17) - +- CometFilter (16) - +- CometScan parquet spark_catalog.default.store_sales (15) - - -(15) Scan parquet spark_catalog.default.store_sales +* Project (20) ++- * HashAggregate (19) + +- Exchange (18) + +- * HashAggregate (17) + +- * ColumnarToRow (16) + +- CometProject (15) + +- CometFilter (14) + +- CometScan parquet spark_catalog.default.store_sales (13) + + +(13) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,40)] ReadSchema: struct -(16) CometFilter +(14) CometFilter Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] Condition : ((isnotnull(ss_quantity#38) AND (ss_quantity#38 >= 21)) AND (ss_quantity#38 <= 40)) -(17) CometProject +(15) CometProject Input [4]: [ss_quantity#38, ss_ext_discount_amt#39, ss_net_paid#40, ss_sold_date_sk#41] Arguments: [ss_ext_discount_amt#39, ss_net_paid#40], [ss_ext_discount_amt#39, ss_net_paid#40] -(18) ColumnarToRow [codegen id : 1] +(16) ColumnarToRow [codegen id : 1] Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] -(19) HashAggregate [codegen id : 1] +(17) HashAggregate [codegen id : 1] Input [2]: [ss_ext_discount_amt#39, ss_net_paid#40] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#39)), partial_avg(UnscaledValue(ss_net_paid#40))] Aggregate Attributes [5]: [count#42, sum#43, count#44, sum#45, count#46] Results [5]: [count#47, sum#48, count#49, sum#50, count#51] -(20) RowToColumnar -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] - -(21) CometColumnarExchange +(18) Exchange Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(22) ColumnarToRow [codegen id : 2] -Input [5]: [count#47, sum#48, count#49, sum#50, count#51] - -(23) HashAggregate [codegen id : 2] +(19) HashAggregate [codegen id : 2] Input [5]: [count#47, sum#48, count#49, sum#50, count#51] Keys: [] Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#39)), avg(UnscaledValue(ss_net_paid#40))] Aggregate Attributes [3]: [count(1)#52, avg(UnscaledValue(ss_ext_discount_amt#39))#53, avg(UnscaledValue(ss_net_paid#40))#54] Results [3]: [count(1)#52 AS count(1)#55, cast((avg(UnscaledValue(ss_ext_discount_amt#39))#53 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#56, cast((avg(UnscaledValue(ss_net_paid#40))#54 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#57] -(24) Project [codegen id : 2] +(20) Project [codegen id : 2] Output [1]: [named_struct(count(1), count(1)#55, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#56, avg(ss_net_paid), avg(ss_net_paid)#57) AS mergedValue#58] Input [3]: [count(1)#55, avg(ss_ext_discount_amt)#56, avg(ss_net_paid)#57] @@ -152,61 +136,53 @@ Subquery:5 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery Subquery:6 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#5, [id=#6] Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#8, [id=#9] -* Project (34) -+- * HashAggregate (33) - +- * ColumnarToRow (32) - +- CometColumnarExchange (31) - +- RowToColumnar (30) - +- * HashAggregate (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.store_sales (25) - - -(25) Scan parquet spark_catalog.default.store_sales +* Project (28) ++- * HashAggregate (27) + +- Exchange (26) + +- * HashAggregate (25) + +- * ColumnarToRow (24) + +- CometProject (23) + +- CometFilter (22) + +- CometScan parquet spark_catalog.default.store_sales (21) + + +(21) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,41), LessThanOrEqual(ss_quantity,60)] ReadSchema: struct -(26) CometFilter +(22) CometFilter Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] Condition : ((isnotnull(ss_quantity#59) AND (ss_quantity#59 >= 41)) AND (ss_quantity#59 <= 60)) -(27) CometProject +(23) CometProject Input [4]: [ss_quantity#59, ss_ext_discount_amt#60, ss_net_paid#61, ss_sold_date_sk#62] Arguments: [ss_ext_discount_amt#60, ss_net_paid#61], [ss_ext_discount_amt#60, ss_net_paid#61] -(28) ColumnarToRow [codegen id : 1] +(24) ColumnarToRow [codegen id : 1] Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] -(29) HashAggregate [codegen id : 1] +(25) HashAggregate [codegen id : 1] Input [2]: [ss_ext_discount_amt#60, ss_net_paid#61] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#60)), partial_avg(UnscaledValue(ss_net_paid#61))] Aggregate Attributes [5]: [count#63, sum#64, count#65, sum#66, count#67] Results [5]: [count#68, sum#69, count#70, sum#71, count#72] -(30) RowToColumnar -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] - -(31) CometColumnarExchange -Input [5]: [count#68, sum#69, count#70, sum#71, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(32) ColumnarToRow [codegen id : 2] +(26) Exchange Input [5]: [count#68, sum#69, count#70, sum#71, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] -(33) HashAggregate [codegen id : 2] +(27) HashAggregate [codegen id : 2] Input [5]: [count#68, sum#69, count#70, sum#71, count#72] Keys: [] Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#60)), avg(UnscaledValue(ss_net_paid#61))] Aggregate Attributes [3]: [count(1)#73, avg(UnscaledValue(ss_ext_discount_amt#60))#74, avg(UnscaledValue(ss_net_paid#61))#75] Results [3]: [count(1)#73 AS count(1)#76, cast((avg(UnscaledValue(ss_ext_discount_amt#60))#74 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#77, cast((avg(UnscaledValue(ss_net_paid#61))#75 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#78] -(34) Project [codegen id : 2] +(28) Project [codegen id : 2] Output [1]: [named_struct(count(1), count(1)#76, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#77, avg(ss_net_paid), avg(ss_net_paid)#78) AS mergedValue#79] Input [3]: [count(1)#76, avg(ss_ext_discount_amt)#77, avg(ss_net_paid)#78] @@ -215,61 +191,53 @@ Subquery:8 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery Subquery:9 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#8, [id=#9] Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* Project (44) -+- * HashAggregate (43) - +- * ColumnarToRow (42) - +- CometColumnarExchange (41) - +- RowToColumnar (40) - +- * HashAggregate (39) - +- * ColumnarToRow (38) - +- CometProject (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.store_sales (35) +* Project (36) ++- * HashAggregate (35) + +- Exchange (34) + +- * HashAggregate (33) + +- * ColumnarToRow (32) + +- CometProject (31) + +- CometFilter (30) + +- CometScan parquet spark_catalog.default.store_sales (29) -(35) Scan parquet spark_catalog.default.store_sales +(29) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,61), LessThanOrEqual(ss_quantity,80)] ReadSchema: struct -(36) CometFilter +(30) CometFilter Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] Condition : ((isnotnull(ss_quantity#80) AND (ss_quantity#80 >= 61)) AND (ss_quantity#80 <= 80)) -(37) CometProject +(31) CometProject Input [4]: [ss_quantity#80, ss_ext_discount_amt#81, ss_net_paid#82, ss_sold_date_sk#83] Arguments: [ss_ext_discount_amt#81, ss_net_paid#82], [ss_ext_discount_amt#81, ss_net_paid#82] -(38) ColumnarToRow [codegen id : 1] +(32) ColumnarToRow [codegen id : 1] Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] -(39) HashAggregate [codegen id : 1] +(33) HashAggregate [codegen id : 1] Input [2]: [ss_ext_discount_amt#81, ss_net_paid#82] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#81)), partial_avg(UnscaledValue(ss_net_paid#82))] Aggregate Attributes [5]: [count#84, sum#85, count#86, sum#87, count#88] Results [5]: [count#89, sum#90, count#91, sum#92, count#93] -(40) RowToColumnar -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] - -(41) CometColumnarExchange +(34) Exchange Input [5]: [count#89, sum#90, count#91, sum#92, count#93] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] -(42) ColumnarToRow [codegen id : 2] -Input [5]: [count#89, sum#90, count#91, sum#92, count#93] - -(43) HashAggregate [codegen id : 2] +(35) HashAggregate [codegen id : 2] Input [5]: [count#89, sum#90, count#91, sum#92, count#93] Keys: [] Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#81)), avg(UnscaledValue(ss_net_paid#82))] Aggregate Attributes [3]: [count(1)#94, avg(UnscaledValue(ss_ext_discount_amt#81))#95, avg(UnscaledValue(ss_net_paid#82))#96] Results [3]: [count(1)#94 AS count(1)#97, cast((avg(UnscaledValue(ss_ext_discount_amt#81))#95 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#98, cast((avg(UnscaledValue(ss_net_paid#82))#96 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#99] -(44) Project [codegen id : 2] +(36) Project [codegen id : 2] Output [1]: [named_struct(count(1), count(1)#97, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#98, avg(ss_net_paid), avg(ss_net_paid)#99) AS mergedValue#100] Input [3]: [count(1)#97, avg(ss_ext_discount_amt)#98, avg(ss_net_paid)#99] @@ -278,61 +246,53 @@ Subquery:11 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery Subquery:12 Hosting operator id = 4 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery#14, [id=#15] -* Project (54) -+- * HashAggregate (53) - +- * ColumnarToRow (52) - +- CometColumnarExchange (51) - +- RowToColumnar (50) - +- * HashAggregate (49) - +- * ColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan parquet spark_catalog.default.store_sales (45) - - -(45) Scan parquet spark_catalog.default.store_sales +* Project (44) ++- * HashAggregate (43) + +- Exchange (42) + +- * HashAggregate (41) + +- * ColumnarToRow (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan parquet spark_catalog.default.store_sales (37) + + +(37) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,81), LessThanOrEqual(ss_quantity,100)] ReadSchema: struct -(46) CometFilter +(38) CometFilter Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] Condition : ((isnotnull(ss_quantity#101) AND (ss_quantity#101 >= 81)) AND (ss_quantity#101 <= 100)) -(47) CometProject +(39) CometProject Input [4]: [ss_quantity#101, ss_ext_discount_amt#102, ss_net_paid#103, ss_sold_date_sk#104] Arguments: [ss_ext_discount_amt#102, ss_net_paid#103], [ss_ext_discount_amt#102, ss_net_paid#103] -(48) ColumnarToRow [codegen id : 1] +(40) ColumnarToRow [codegen id : 1] Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] -(49) HashAggregate [codegen id : 1] +(41) HashAggregate [codegen id : 1] Input [2]: [ss_ext_discount_amt#102, ss_net_paid#103] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#102)), partial_avg(UnscaledValue(ss_net_paid#103))] Aggregate Attributes [5]: [count#105, sum#106, count#107, sum#108, count#109] Results [5]: [count#110, sum#111, count#112, sum#113, count#114] -(50) RowToColumnar -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] - -(51) CometColumnarExchange +(42) Exchange Input [5]: [count#110, sum#111, count#112, sum#113, count#114] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] -(52) ColumnarToRow [codegen id : 2] -Input [5]: [count#110, sum#111, count#112, sum#113, count#114] - -(53) HashAggregate [codegen id : 2] +(43) HashAggregate [codegen id : 2] Input [5]: [count#110, sum#111, count#112, sum#113, count#114] Keys: [] Functions [3]: [count(1), avg(UnscaledValue(ss_ext_discount_amt#102)), avg(UnscaledValue(ss_net_paid#103))] Aggregate Attributes [3]: [count(1)#115, avg(UnscaledValue(ss_ext_discount_amt#102))#116, avg(UnscaledValue(ss_net_paid#103))#117] Results [3]: [count(1)#115 AS count(1)#118, cast((avg(UnscaledValue(ss_ext_discount_amt#102))#116 / 100.0) as decimal(11,6)) AS avg(ss_ext_discount_amt)#119, cast((avg(UnscaledValue(ss_net_paid#103))#117 / 100.0) as decimal(11,6)) AS avg(ss_net_paid)#120] -(54) Project [codegen id : 2] +(44) Project [codegen id : 2] Output [1]: [named_struct(count(1), count(1)#118, avg(ss_ext_discount_amt), avg(ss_ext_discount_amt)#119, avg(ss_net_paid), avg(ss_net_paid)#120) AS mergedValue#121] Input [3]: [count(1)#118, avg(ss_ext_discount_amt)#119, avg(ss_net_paid)#120] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt index 79896e05b..581739cc6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q9/simplified.txt @@ -4,85 +4,75 @@ WholeStageCodegen (1) WholeStageCodegen (2) Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + InputAdapter + Exchange #1 + WholeStageCodegen (1) + HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #1 ReusedSubquery [mergedValue] #1 Subquery #2 WholeStageCodegen (2) Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + InputAdapter + Exchange #2 + WholeStageCodegen (1) + HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #2 ReusedSubquery [mergedValue] #2 Subquery #3 WholeStageCodegen (2) Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + InputAdapter + Exchange #3 + WholeStageCodegen (1) + HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #3 ReusedSubquery [mergedValue] #3 Subquery #4 WholeStageCodegen (2) Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #4 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + InputAdapter + Exchange #4 + WholeStageCodegen (1) + HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #4 ReusedSubquery [mergedValue] #4 Subquery #5 WholeStageCodegen (2) Project [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] HashAggregate [count,sum,count,sum,count] [count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid)),count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #5 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_ext_discount_amt,ss_net_paid] - CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + InputAdapter + Exchange #5 + WholeStageCodegen (1) + HashAggregate [ss_ext_discount_amt,ss_net_paid] [count,sum,count,sum,count,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ss_ext_discount_amt,ss_net_paid] + CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] ReusedSubquery [mergedValue] #5 ReusedSubquery [mergedValue] #5 ColumnarToRow diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt index ab17ec1d0..b55193e7e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/explain.txt @@ -3,7 +3,7 @@ +- * BroadcastNestedLoopJoin Inner BuildRight (46) :- * ColumnarToRow (25) : +- CometHashAggregate (24) - : +- CometColumnarExchange (23) + : +- CometExchange (23) : +- CometHashAggregate (22) : +- CometProject (21) : +- CometBroadcastHashJoin (20) @@ -29,7 +29,7 @@ +- BroadcastExchange (45) +- * ColumnarToRow (44) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometProject (40) +- CometBroadcastHashJoin (39) @@ -151,9 +151,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(23) CometColumnarExchange +(23) CometExchange Input [1]: [count#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [1]: [count#11] @@ -234,9 +234,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(42) CometColumnarExchange +(42) CometExchange Input [1]: [count#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (43) CometHashAggregate Input [1]: [count#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt index 65860a550..95fd73d86 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q90/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [amc,count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] @@ -33,7 +33,7 @@ WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [pmc,count,count(1)] - CometColumnarExchange #6 + CometExchange #6 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt index 7866655a3..9b9e36b3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/explain.txt @@ -1,49 +1,47 @@ == Physical Plan == -* ColumnarToRow (45) -+- CometSort (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * ColumnarToRow (40) - +- CometColumnarExchange (39) - +- RowToColumnar (38) - +- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometProject (35) - +- CometBroadcastHashJoin (34) - :- CometProject (29) - : +- CometBroadcastHashJoin (28) - : :- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (13) - : : : : +- CometBroadcastHashJoin (12) - : : : : :- CometProject (7) - : : : : : +- CometBroadcastHashJoin (6) - : : : : : :- CometFilter (2) - : : : : : : +- CometScan parquet spark_catalog.default.call_center (1) - : : : : : +- CometBroadcastExchange (5) - : : : : : +- CometFilter (4) - : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (3) - : : : : +- CometBroadcastExchange (11) - : : : : +- CometProject (10) - : : : : +- CometFilter (9) - : : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : : +- CometBroadcastExchange (16) - : : : +- CometFilter (15) - : : : +- CometScan parquet spark_catalog.default.customer (14) - : : +- CometBroadcastExchange (22) - : : +- CometProject (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.customer_address (19) - : +- CometBroadcastExchange (27) - : +- CometFilter (26) - : +- CometScan parquet spark_catalog.default.customer_demographics (25) - +- CometBroadcastExchange (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.household_demographics (30) +* ColumnarToRow (43) ++- CometSort (42) + +- CometColumnarExchange (41) + +- RowToColumnar (40) + +- * HashAggregate (39) + +- Exchange (38) + +- * HashAggregate (37) + +- * ColumnarToRow (36) + +- CometProject (35) + +- CometBroadcastHashJoin (34) + :- CometProject (29) + : +- CometBroadcastHashJoin (28) + : :- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (13) + : : : : +- CometBroadcastHashJoin (12) + : : : : :- CometProject (7) + : : : : : +- CometBroadcastHashJoin (6) + : : : : : :- CometFilter (2) + : : : : : : +- CometScan parquet spark_catalog.default.call_center (1) + : : : : : +- CometBroadcastExchange (5) + : : : : : +- CometFilter (4) + : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (3) + : : : : +- CometBroadcastExchange (11) + : : : : +- CometProject (10) + : : : : +- CometFilter (9) + : : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : : +- CometBroadcastExchange (16) + : : : +- CometFilter (15) + : : : +- CometScan parquet spark_catalog.default.customer (14) + : : +- CometBroadcastExchange (22) + : : +- CometProject (21) + : : +- CometFilter (20) + : : +- CometScan parquet spark_catalog.default.customer_address (19) + : +- CometBroadcastExchange (27) + : +- CometFilter (26) + : +- CometScan parquet spark_catalog.default.customer_demographics (25) + +- CometBroadcastExchange (33) + +- CometProject (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.household_demographics (30) (1) Scan parquet spark_catalog.default.call_center @@ -224,66 +222,60 @@ Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] Aggregate Attributes [1]: [sum#24] Results [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, sum#25] -(38) RowToColumnar +(38) Exchange Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, sum#25] +Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(39) CometColumnarExchange -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, sum#25] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(40) ColumnarToRow [codegen id : 2] -Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, sum#25] - -(41) HashAggregate [codegen id : 2] +(39) HashAggregate [codegen id : 2] Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, sum#25] Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21] Functions [1]: [sum(UnscaledValue(cr_net_loss#7))] Aggregate Attributes [1]: [sum(UnscaledValue(cr_net_loss#7))#26] Results [4]: [cc_call_center_id#2 AS Call_Center#27, cc_name#3 AS Call_Center_Name#28, cc_manager#4 AS Manager#29, MakeDecimal(sum(UnscaledValue(cr_net_loss#7))#26,17,2) AS Returns_Loss#30] -(42) RowToColumnar +(40) RowToColumnar Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] -(43) CometColumnarExchange +(41) CometColumnarExchange Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] Arguments: rangepartitioning(Returns_Loss#30 DESC NULLS LAST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] -(44) CometSort +(42) CometSort Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] Arguments: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30], [Returns_Loss#30 DESC NULLS LAST] -(45) ColumnarToRow [codegen id : 3] +(43) ColumnarToRow [codegen id : 3] Input [4]: [Call_Center#27, Call_Center_Name#28, Manager#29, Returns_Loss#30] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = cr_returned_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (50) -+- * ColumnarToRow (49) - +- CometProject (48) - +- CometFilter (47) - +- CometScan parquet spark_catalog.default.date_dim (46) +BroadcastExchange (48) ++- * ColumnarToRow (47) + +- CometProject (46) + +- CometFilter (45) + +- CometScan parquet spark_catalog.default.date_dim (44) -(46) Scan parquet spark_catalog.default.date_dim +(44) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#10, d_year#11, d_moy#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,1998), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(47) CometFilter +(45) CometFilter Input [3]: [d_date_sk#10, d_year#11, d_moy#12] Condition : ((((isnotnull(d_year#11) AND isnotnull(d_moy#12)) AND (d_year#11 = 1998)) AND (d_moy#12 = 11)) AND isnotnull(d_date_sk#10)) -(48) CometProject +(46) CometProject Input [3]: [d_date_sk#10, d_year#11, d_moy#12] Arguments: [d_date_sk#10], [d_date_sk#10] -(49) ColumnarToRow [codegen id : 1] +(47) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#10] -(50) BroadcastExchange +(48) BroadcastExchange Input [1]: [d_date_sk#10] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt index c55d2962c..1684e1803 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q91/simplified.txt @@ -6,54 +6,52 @@ WholeStageCodegen (3) RowToColumnar WholeStageCodegen (2) HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum] [sum(UnscaledValue(cr_net_loss)),Call_Center,Call_Center_Name,Manager,Returns_Loss,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,cr_net_loss] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,ca_address_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] - CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] - CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] - CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 - CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #6 - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk] #7 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_gmt_offset] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] - CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #8 - CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] - CometBroadcastExchange [hd_demo_sk] #9 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + InputAdapter + Exchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 + WholeStageCodegen (1) + HashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,cr_net_loss] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,cd_demo_sk,cd_marital_status,cd_education_status] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,ca_address_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss] + CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometProject [cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_net_loss,cr_returned_date_sk] + CometBroadcastHashJoin [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager,cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometFilter [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_call_center_id,cc_name,cc_manager] + CometBroadcastExchange [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] #3 + CometFilter [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_returning_customer_sk,cr_call_center_sk,cr_net_loss,cr_returned_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] #6 + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk] #7 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_gmt_offset] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_gmt_offset] + CometBroadcastExchange [cd_demo_sk,cd_marital_status,cd_education_status] #8 + CometFilter [cd_demo_sk,cd_marital_status,cd_education_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status,cd_education_status] + CometBroadcastExchange [hd_demo_sk] #9 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt index 2825d25af..e8d482e9b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/explain.txt @@ -1,39 +1,35 @@ == Physical Plan == -* HashAggregate (35) -+- * ColumnarToRow (34) - +- CometColumnarExchange (33) - +- RowToColumnar (32) - +- * HashAggregate (31) - +- * Project (30) - +- * BroadcastHashJoin Inner BuildRight (29) - :- * Project (27) - : +- * BroadcastHashJoin Inner BuildRight (26) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.item (3) - : +- BroadcastExchange (25) - : +- * Filter (24) - : +- * HashAggregate (23) - : +- * ColumnarToRow (22) - : +- CometColumnarExchange (21) - : +- RowToColumnar (20) - : +- * HashAggregate (19) - : +- * ColumnarToRow (18) - : +- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometFilter (11) - : : +- CometScan parquet spark_catalog.default.web_sales (10) - : +- CometBroadcastExchange (15) - : +- CometProject (14) - : +- CometFilter (13) - : +- CometScan parquet spark_catalog.default.date_dim (12) - +- ReusedExchange (28) +* HashAggregate (31) ++- Exchange (30) + +- * HashAggregate (29) + +- * Project (28) + +- * BroadcastHashJoin Inner BuildRight (27) + :- * Project (25) + : +- * BroadcastHashJoin Inner BuildRight (24) + : :- * ColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.item (3) + : +- BroadcastExchange (23) + : +- * Filter (22) + : +- * HashAggregate (21) + : +- Exchange (20) + : +- * HashAggregate (19) + : +- * ColumnarToRow (18) + : +- CometProject (17) + : +- CometBroadcastHashJoin (16) + : :- CometFilter (11) + : : +- CometScan parquet spark_catalog.default.web_sales (10) + : +- CometBroadcastExchange (15) + : +- CometProject (14) + : +- CometFilter (13) + : +- CometScan parquet spark_catalog.default.date_dim (12) + +- ReusedExchange (26) (1) Scan parquet spark_catalog.default.web_sales @@ -129,72 +125,60 @@ Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#8))] Aggregate Attributes [2]: [sum#13, count#14] Results [3]: [ws_item_sk#7, sum#15, count#16] -(20) RowToColumnar +(20) Exchange Input [3]: [ws_item_sk#7, sum#15, count#16] +Arguments: hashpartitioning(ws_item_sk#7, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(21) CometColumnarExchange -Input [3]: [ws_item_sk#7, sum#15, count#16] -Arguments: hashpartitioning(ws_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(22) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#7, sum#15, count#16] - -(23) HashAggregate [codegen id : 2] +(21) HashAggregate [codegen id : 2] Input [3]: [ws_item_sk#7, sum#15, count#16] Keys [1]: [ws_item_sk#7] Functions [1]: [avg(UnscaledValue(ws_ext_discount_amt#8))] Aggregate Attributes [1]: [avg(UnscaledValue(ws_ext_discount_amt#8))#17] Results [2]: [(1.3 * cast((avg(UnscaledValue(ws_ext_discount_amt#8))#17 / 100.0) as decimal(11,6))) AS (1.3 * avg(ws_ext_discount_amt))#18, ws_item_sk#7] -(24) Filter [codegen id : 2] +(22) Filter [codegen id : 2] Input [2]: [(1.3 * avg(ws_ext_discount_amt))#18, ws_item_sk#7] Condition : isnotnull((1.3 * avg(ws_ext_discount_amt))#18) -(25) BroadcastExchange +(23) BroadcastExchange Input [2]: [(1.3 * avg(ws_ext_discount_amt))#18, ws_item_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[1, int, true] as bigint)),false), [plan_id=2] -(26) BroadcastHashJoin [codegen id : 4] +(24) BroadcastHashJoin [codegen id : 4] Left keys [1]: [i_item_sk#5] Right keys [1]: [ws_item_sk#7] Join type: Inner Join condition: (cast(ws_ext_discount_amt#2 as decimal(14,7)) > (1.3 * avg(ws_ext_discount_amt))#18) -(27) Project [codegen id : 4] +(25) Project [codegen id : 4] Output [2]: [ws_ext_discount_amt#2, ws_sold_date_sk#3] Input [5]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, i_item_sk#5, (1.3 * avg(ws_ext_discount_amt))#18, ws_item_sk#7] -(28) ReusedExchange [Reuses operator id: 40] +(26) ReusedExchange [Reuses operator id: 36] Output [1]: [d_date_sk#19] -(29) BroadcastHashJoin [codegen id : 4] +(27) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ws_sold_date_sk#3] Right keys [1]: [d_date_sk#19] Join type: Inner Join condition: None -(30) Project [codegen id : 4] +(28) Project [codegen id : 4] Output [1]: [ws_ext_discount_amt#2] Input [3]: [ws_ext_discount_amt#2, ws_sold_date_sk#3, d_date_sk#19] -(31) HashAggregate [codegen id : 4] +(29) HashAggregate [codegen id : 4] Input [1]: [ws_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] Aggregate Attributes [1]: [sum#20] Results [1]: [sum#21] -(32) RowToColumnar -Input [1]: [sum#21] - -(33) CometColumnarExchange -Input [1]: [sum#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(34) ColumnarToRow [codegen id : 5] +(30) Exchange Input [1]: [sum#21] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] -(35) HashAggregate [codegen id : 5] +(31) HashAggregate [codegen id : 5] Input [1]: [sum#21] Keys: [] Functions [1]: [sum(UnscaledValue(ws_ext_discount_amt#2))] @@ -204,32 +188,32 @@ Results [1]: [MakeDecimal(sum(UnscaledValue(ws_ext_discount_amt#2))#22,17,2) AS ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (40) -+- * ColumnarToRow (39) - +- CometProject (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.date_dim (36) +BroadcastExchange (36) ++- * ColumnarToRow (35) + +- CometProject (34) + +- CometFilter (33) + +- CometScan parquet spark_catalog.default.date_dim (32) -(36) Scan parquet spark_catalog.default.date_dim +(32) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#19, d_date#24] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-01-27), LessThanOrEqual(d_date,2000-04-26), IsNotNull(d_date_sk)] ReadSchema: struct -(37) CometFilter +(33) CometFilter Input [2]: [d_date_sk#19, d_date#24] Condition : (((isnotnull(d_date#24) AND (d_date#24 >= 2000-01-27)) AND (d_date#24 <= 2000-04-26)) AND isnotnull(d_date_sk#19)) -(38) CometProject +(34) CometProject Input [2]: [d_date_sk#19, d_date#24] Arguments: [d_date_sk#19], [d_date_sk#19] -(39) ColumnarToRow [codegen id : 1] +(35) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#19] -(40) BroadcastExchange +(36) BroadcastExchange Input [1]: [d_date_sk#19] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt index df7b1ca66..aa24a049b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q92/simplified.txt @@ -1,54 +1,50 @@ WholeStageCodegen (5) HashAggregate [sum] [sum(UnscaledValue(ws_ext_discount_amt)),Excess Discount Amount ,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [ws_ext_discount_amt] [sum,sum] - Project [ws_ext_discount_amt] - BroadcastHashJoin [ws_sold_date_sk,d_date_sk] - Project [ws_ext_discount_amt,ws_sold_date_sk] - BroadcastHashJoin [i_item_sk,ws_item_sk,ws_ext_discount_amt,(1.3 * avg(ws_ext_discount_amt))] - ColumnarToRow - InputAdapter - CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] - CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk] #3 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] - InputAdapter - BroadcastExchange #4 - WholeStageCodegen (2) - Filter [(1.3 * avg(ws_ext_discount_amt))] - HashAggregate [ws_item_sk,sum,count] [avg(UnscaledValue(ws_ext_discount_amt)),(1.3 * avg(ws_ext_discount_amt)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ws_item_sk] #5 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ws_item_sk,ws_ext_discount_amt] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_ext_discount_amt] - CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (4) + HashAggregate [ws_ext_discount_amt] [sum,sum] + Project [ws_ext_discount_amt] + BroadcastHashJoin [ws_sold_date_sk,d_date_sk] + Project [ws_ext_discount_amt,ws_sold_date_sk] + BroadcastHashJoin [i_item_sk,ws_item_sk,ws_ext_discount_amt,(1.3 * avg(ws_ext_discount_amt))] + ColumnarToRow + InputAdapter + CometProject [ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] + CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,i_item_sk] + CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [i_item_sk] #3 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_manufact_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_manufact_id] InputAdapter - ReusedExchange [d_date_sk] #2 + BroadcastExchange #4 + WholeStageCodegen (2) + Filter [(1.3 * avg(ws_ext_discount_amt))] + HashAggregate [ws_item_sk,sum,count] [avg(UnscaledValue(ws_ext_discount_amt)),(1.3 * avg(ws_ext_discount_amt)),sum,count] + InputAdapter + Exchange [ws_item_sk] #5 + WholeStageCodegen (1) + HashAggregate [ws_item_sk,ws_ext_discount_amt] [sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ws_item_sk,ws_ext_discount_amt] + CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + ReusedExchange [d_date_sk] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt index 4634a45a1..aa92d095a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q93/explain.txt @@ -1,28 +1,26 @@ == Physical Plan == -TakeOrderedAndProject (24) -+- * HashAggregate (23) - +- * ColumnarToRow (22) - +- CometColumnarExchange (21) - +- RowToColumnar (20) - +- * HashAggregate (19) - +- * ColumnarToRow (18) - +- CometProject (17) - +- CometBroadcastHashJoin (16) - :- CometProject (11) - : +- CometSortMergeJoin (10) - : :- CometSort (4) - : : +- CometColumnarExchange (3) - : : +- CometProject (2) - : : +- CometScan parquet spark_catalog.default.store_sales (1) - : +- CometSort (9) - : +- CometColumnarExchange (8) - : +- CometProject (7) - : +- CometFilter (6) - : +- CometScan parquet spark_catalog.default.store_returns (5) - +- CometBroadcastExchange (15) - +- CometProject (14) - +- CometFilter (13) - +- CometScan parquet spark_catalog.default.reason (12) +TakeOrderedAndProject (22) ++- * HashAggregate (21) + +- Exchange (20) + +- * HashAggregate (19) + +- * ColumnarToRow (18) + +- CometProject (17) + +- CometBroadcastHashJoin (16) + :- CometProject (11) + : +- CometSortMergeJoin (10) + : :- CometSort (4) + : : +- CometExchange (3) + : : +- CometProject (2) + : : +- CometScan parquet spark_catalog.default.store_sales (1) + : +- CometSort (9) + : +- CometExchange (8) + : +- CometProject (7) + : +- CometFilter (6) + : +- CometScan parquet spark_catalog.default.store_returns (5) + +- CometBroadcastExchange (15) + +- CometProject (14) + +- CometFilter (13) + +- CometScan parquet spark_catalog.default.reason (12) (1) Scan parquet spark_catalog.default.store_sales @@ -35,9 +33,9 @@ ReadSchema: struct Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order_number#10] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [ws_warehouse_sk#9, ws_order_number#10] @@ -115,9 +113,9 @@ ReadSchema: struct Input [2]: [wr_order_number#12, wr_returned_date_sk#13] Arguments: [wr_order_number#12], [wr_order_number#12] -(16) CometColumnarExchange +(16) CometExchange Input [1]: [wr_order_number#12] -Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (17) CometSort Input [1]: [wr_order_number#12] @@ -249,17 +247,11 @@ Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] Results [3]: [sum#22, sum#23, count#25] -(44) RowToColumnar +(44) Exchange Input [3]: [sum#22, sum#23, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(45) CometColumnarExchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(46) ColumnarToRow [codegen id : 9] -Input [3]: [sum#22, sum#23, count#25] - -(47) HashAggregate [codegen id : 9] +(45) HashAggregate [codegen id : 9] Input [3]: [sum#22, sum#23, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt index 3f324f9f2..16295d42c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q94/simplified.txt @@ -1,70 +1,68 @@ WholeStageCodegen (9) HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (8) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (3) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometColumnarExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - CometColumnarExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [wr_order_number] - CometColumnarExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (8) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_web_site_sk,web_site_sk] + Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] + Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_date_sk,d_date_sk] + SortMergeJoin [ws_order_number,wr_order_number] + InputAdapter + WholeStageCodegen (3) + Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) + BroadcastExchange #5 + WholeStageCodegen (5) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) + BroadcastExchange #6 + WholeStageCodegen (6) ColumnarToRow InputAdapter - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt index f58d42056..348ee49e5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/explain.txt @@ -1,67 +1,65 @@ == Physical Plan == -* HashAggregate (63) -+- * ColumnarToRow (62) - +- CometColumnarExchange (61) - +- RowToColumnar (60) - +- * HashAggregate (59) - +- * HashAggregate (58) - +- * HashAggregate (57) - +- * Project (56) - +- * BroadcastHashJoin Inner BuildRight (55) - :- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (42) - : : +- * BroadcastHashJoin Inner BuildRight (41) - : : :- * SortMergeJoin LeftSemi (35) - : : : :- * SortMergeJoin LeftSemi (18) - : : : : :- * ColumnarToRow (6) - : : : : : +- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- * Project (17) - : : : : +- * SortMergeJoin Inner (16) - : : : : :- * ColumnarToRow (12) - : : : : : +- CometSort (11) - : : : : : +- CometColumnarExchange (10) - : : : : : +- CometProject (9) - : : : : : +- CometFilter (8) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (7) - : : : : +- * ColumnarToRow (15) - : : : : +- CometSort (14) - : : : : +- ReusedExchange (13) - : : : +- * Project (34) - : : : +- * SortMergeJoin Inner (33) - : : : :- * ColumnarToRow (24) - : : : : +- CometSort (23) - : : : : +- CometColumnarExchange (22) - : : : : +- CometProject (21) - : : : : +- CometFilter (20) - : : : : +- CometScan parquet spark_catalog.default.web_returns (19) - : : : +- * Project (32) - : : : +- * SortMergeJoin Inner (31) - : : : :- * ColumnarToRow (27) - : : : : +- CometSort (26) - : : : : +- ReusedExchange (25) - : : : +- * ColumnarToRow (30) - : : : +- CometSort (29) - : : : +- ReusedExchange (28) - : : +- BroadcastExchange (40) - : : +- * ColumnarToRow (39) - : : +- CometProject (38) - : : +- CometFilter (37) - : : +- CometScan parquet spark_catalog.default.date_dim (36) - : +- BroadcastExchange (47) - : +- * ColumnarToRow (46) - : +- CometProject (45) - : +- CometFilter (44) - : +- CometScan parquet spark_catalog.default.customer_address (43) - +- BroadcastExchange (54) - +- * ColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.web_site (50) +* HashAggregate (61) ++- Exchange (60) + +- * HashAggregate (59) + +- * HashAggregate (58) + +- * HashAggregate (57) + +- * Project (56) + +- * BroadcastHashJoin Inner BuildRight (55) + :- * Project (49) + : +- * BroadcastHashJoin Inner BuildRight (48) + : :- * Project (42) + : : +- * BroadcastHashJoin Inner BuildRight (41) + : : :- * SortMergeJoin LeftSemi (35) + : : : :- * SortMergeJoin LeftSemi (18) + : : : : :- * ColumnarToRow (6) + : : : : : +- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : +- * Project (17) + : : : : +- * SortMergeJoin Inner (16) + : : : : :- * ColumnarToRow (12) + : : : : : +- CometSort (11) + : : : : : +- CometExchange (10) + : : : : : +- CometProject (9) + : : : : : +- CometFilter (8) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (7) + : : : : +- * ColumnarToRow (15) + : : : : +- CometSort (14) + : : : : +- ReusedExchange (13) + : : : +- * Project (34) + : : : +- * SortMergeJoin Inner (33) + : : : :- * ColumnarToRow (24) + : : : : +- CometSort (23) + : : : : +- CometExchange (22) + : : : : +- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometScan parquet spark_catalog.default.web_returns (19) + : : : +- * Project (32) + : : : +- * SortMergeJoin Inner (31) + : : : :- * ColumnarToRow (27) + : : : : +- CometSort (26) + : : : : +- ReusedExchange (25) + : : : +- * ColumnarToRow (30) + : : : +- CometSort (29) + : : : +- ReusedExchange (28) + : : +- BroadcastExchange (40) + : : +- * ColumnarToRow (39) + : : +- CometProject (38) + : : +- CometFilter (37) + : : +- CometScan parquet spark_catalog.default.date_dim (36) + : +- BroadcastExchange (47) + : +- * ColumnarToRow (46) + : +- CometProject (45) + : +- CometFilter (44) + : +- CometScan parquet spark_catalog.default.customer_address (43) + +- BroadcastExchange (54) + +- * ColumnarToRow (53) + +- CometProject (52) + +- CometFilter (51) + +- CometScan parquet spark_catalog.default.web_site (50) (1) Scan parquet spark_catalog.default.web_sales @@ -79,9 +77,9 @@ Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(4) CometColumnarExchange +(4) CometExchange Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] @@ -105,9 +103,9 @@ Condition : (isnotnull(ws_order_number#9) AND isnotnull(ws_warehouse_sk#8)) Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_number#9] -(10) CometColumnarExchange +(10) CometExchange Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (11) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] @@ -157,9 +155,9 @@ Condition : isnotnull(wr_order_number#13) Input [2]: [wr_order_number#13, wr_returned_date_sk#14] Arguments: [wr_order_number#13], [wr_order_number#13] -(22) CometColumnarExchange +(22) CometExchange Input [1]: [wr_order_number#13] -Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (23) CometSort Input [1]: [wr_order_number#13] @@ -331,17 +329,11 @@ Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#25, sum(UnscaledValue(ws_net_profit#6))#26, count(ws_order_number#4)#29] Results [3]: [sum#27, sum#28, count#30] -(60) RowToColumnar +(60) Exchange Input [3]: [sum#27, sum#28, count#30] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(61) CometColumnarExchange -Input [3]: [sum#27, sum#28, count#30] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(62) ColumnarToRow [codegen id : 15] -Input [3]: [sum#27, sum#28, count#30] - -(63) HashAggregate [codegen id : 15] +(61) HashAggregate [codegen id : 15] Input [3]: [sum#27, sum#28, count#30] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt index 2c736e6e5..5cfa7827a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q95/simplified.txt @@ -1,101 +1,99 @@ WholeStageCodegen (15) HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (14) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (5) - SortMergeJoin [ws_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometColumnarExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - CometColumnarExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - WholeStageCodegen (10) - Project [wr_order_number] - SortMergeJoin [wr_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometSort [wr_order_number] - CometColumnarExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometFilter [wr_order_number,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + InputAdapter + Exchange #1 + WholeStageCodegen (14) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_web_site_sk,web_site_sk] + Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] + Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_date_sk,d_date_sk] + SortMergeJoin [ws_order_number,wr_order_number] + InputAdapter + WholeStageCodegen (5) + SortMergeJoin [ws_order_number,ws_order_number] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow InputAdapter - WholeStageCodegen (9) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (4) + Project [ws_order_number] + SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + InputAdapter + WholeStageCodegen (2) + ColumnarToRow InputAdapter - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (3) + ColumnarToRow InputAdapter - WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + InputAdapter + WholeStageCodegen (10) + Project [wr_order_number] + SortMergeJoin [wr_order_number,ws_order_number] + InputAdapter + WholeStageCodegen (6) + ColumnarToRow + InputAdapter + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometFilter [wr_order_number,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + InputAdapter + WholeStageCodegen (9) + Project [ws_order_number] + SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + InputAdapter + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + InputAdapter + WholeStageCodegen (8) + ColumnarToRow + InputAdapter + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 InputAdapter - BroadcastExchange #6 - WholeStageCodegen (12) + BroadcastExchange #5 + WholeStageCodegen (11) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (13) + BroadcastExchange #6 + WholeStageCodegen (12) ColumnarToRow InputAdapter - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (13) + ColumnarToRow + InputAdapter + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt index 6f9d1a3f2..c4b2cf973 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometBroadcastHashJoin (20) @@ -129,9 +129,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(23) CometColumnarExchange +(23) CometExchange Input [1]: [count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [1]: [count#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt index b8b3e211d..d9a87aa3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q96/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [count(1),count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt index 6c90e74cb..9576c4ef6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/explain.txt @@ -1,31 +1,29 @@ == Physical Plan == -* HashAggregate (27) -+- * ColumnarToRow (26) - +- CometColumnarExchange (25) - +- RowToColumnar (24) - +- * HashAggregate (23) - +- * ColumnarToRow (22) - +- CometProject (21) - +- CometSortMergeJoin (20) - :- CometSort (11) - : +- CometHashAggregate (10) - : +- CometColumnarExchange (9) - : +- CometHashAggregate (8) - : +- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometScan parquet spark_catalog.default.store_sales (1) - : +- CometBroadcastExchange (5) - : +- CometProject (4) - : +- CometFilter (3) - : +- CometScan parquet spark_catalog.default.date_dim (2) - +- CometSort (19) - +- CometHashAggregate (18) - +- CometColumnarExchange (17) - +- CometHashAggregate (16) - +- CometProject (15) - +- CometBroadcastHashJoin (14) - :- CometScan parquet spark_catalog.default.catalog_sales (12) - +- ReusedExchange (13) +* HashAggregate (25) ++- Exchange (24) + +- * HashAggregate (23) + +- * ColumnarToRow (22) + +- CometProject (21) + +- CometSortMergeJoin (20) + :- CometSort (11) + : +- CometHashAggregate (10) + : +- CometExchange (9) + : +- CometHashAggregate (8) + : +- CometProject (7) + : +- CometBroadcastHashJoin (6) + : :- CometScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (5) + : +- CometProject (4) + : +- CometFilter (3) + : +- CometScan parquet spark_catalog.default.date_dim (2) + +- CometSort (19) + +- CometHashAggregate (18) + +- CometExchange (17) + +- CometHashAggregate (16) + +- CometProject (15) + +- CometBroadcastHashJoin (14) + :- CometScan parquet spark_catalog.default.catalog_sales (12) + +- ReusedExchange (13) (1) Scan parquet spark_catalog.default.store_sales @@ -68,9 +66,9 @@ Input [2]: [ss_item_sk#1, ss_customer_sk#2] Keys [2]: [ss_customer_sk#2, ss_item_sk#1] Functions: [] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometHashAggregate Input [2]: [ss_customer_sk#2, ss_item_sk#1] @@ -105,9 +103,9 @@ Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] Keys [2]: [cs_bill_customer_sk#9, cs_item_sk#10] Functions: [] -(17) CometColumnarExchange +(17) CometExchange Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Arguments: hashpartitioning(cs_bill_customer_sk#9, cs_item_sk#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_bill_customer_sk#9, cs_item_sk#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometHashAggregate Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] @@ -137,17 +135,11 @@ Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(custo Aggregate Attributes [3]: [sum#16, sum#17, sum#18] Results [3]: [sum#19, sum#20, sum#21] -(24) RowToColumnar +(24) Exchange Input [3]: [sum#19, sum#20, sum#21] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] -(25) CometColumnarExchange -Input [3]: [sum#19, sum#20, sum#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(26) ColumnarToRow [codegen id : 2] -Input [3]: [sum#19, sum#20, sum#21] - -(27) HashAggregate [codegen id : 2] +(25) HashAggregate [codegen id : 2] Input [3]: [sum#19, sum#20, sum#21] Keys: [] Functions [3]: [sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(customer_sk#14)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END), sum(CASE WHEN (isnotnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END)] @@ -157,32 +149,32 @@ Results [3]: [sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(customer_sk#14) ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (32) -+- * ColumnarToRow (31) - +- CometProject (30) - +- CometFilter (29) - +- CometScan parquet spark_catalog.default.date_dim (28) +BroadcastExchange (30) ++- * ColumnarToRow (29) + +- CometProject (28) + +- CometFilter (27) + +- CometScan parquet spark_catalog.default.date_dim (26) -(28) Scan parquet spark_catalog.default.date_dim +(26) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(29) CometFilter +(27) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) -(30) CometProject +(28) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(31) ColumnarToRow [codegen id : 1] +(29) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(32) BroadcastExchange +(30) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt index 5e09fecd5..a9a3e66eb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q97/simplified.txt @@ -1,40 +1,38 @@ WholeStageCodegen (2) HashAggregate [sum,sum,sum] [sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),store_only,catalog_only,store_and_catalog,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [customer_sk,customer_sk] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [customer_sk,customer_sk] - CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] - CometSort [customer_sk,item_sk] - CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] - CometColumnarExchange [ss_customer_sk,ss_item_sk] #2 - CometHashAggregate [ss_customer_sk,ss_item_sk] - CometProject [ss_item_sk,ss_customer_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometSort [customer_sk,item_sk] - CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] - CometColumnarExchange [cs_bill_customer_sk,cs_item_sk] #5 - CometHashAggregate [cs_bill_customer_sk,cs_item_sk] - CometProject [cs_bill_customer_sk,cs_item_sk] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + InputAdapter + Exchange #1 + WholeStageCodegen (1) + HashAggregate [customer_sk,customer_sk] [sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [customer_sk,customer_sk] + CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] + CometSort [customer_sk,item_sk] + CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] + CometExchange [ss_customer_sk,ss_item_sk] #2 + CometHashAggregate [ss_customer_sk,ss_item_sk] + CometProject [ss_item_sk,ss_customer_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometSort [customer_sk,item_sk] + CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] + CometExchange [cs_bill_customer_sk,cs_item_sk] #5 + CometHashAggregate [cs_bill_customer_sk,cs_item_sk] + CometProject [cs_bill_customer_sk,cs_item_sk] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt index cb1e99486..90f3e0ae9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/explain.txt @@ -1,34 +1,30 @@ == Physical Plan == -* ColumnarToRow (30) -+- CometProject (29) - +- CometSort (28) - +- CometColumnarExchange (27) - +- RowToColumnar (26) - +- * Project (25) - +- Window (24) - +- * ColumnarToRow (23) - +- CometSort (22) - +- CometColumnarExchange (21) - +- RowToColumnar (20) - +- * HashAggregate (19) - +- * ColumnarToRow (18) - +- CometColumnarExchange (17) - +- RowToColumnar (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.store_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) +* ColumnarToRow (26) ++- CometProject (25) + +- CometSort (24) + +- CometColumnarExchange (23) + +- RowToColumnar (22) + +- * Project (21) + +- Window (20) + +- * Sort (19) + +- Exchange (18) + +- * HashAggregate (17) + +- Exchange (16) + +- * HashAggregate (15) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometBroadcastHashJoin (12) + :- CometProject (7) + : +- CometBroadcastHashJoin (6) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (11) + +- CometProject (10) + +- CometFilter (9) + +- CometScan parquet spark_catalog.default.date_dim (8) (1) Scan parquet spark_catalog.default.store_sales @@ -105,92 +101,80 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] Aggregate Attributes [1]: [sum#13] Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -(16) RowToColumnar +(16) Exchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 2] -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] - -(19) HashAggregate [codegen id : 2] +(17) HashAggregate [codegen id : 2] Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#15] Results [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#15,17,2) AS itemrevenue#16, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#15,17,2) AS _w0#17, i_item_id#6] -(20) RowToColumnar -Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] - -(21) CometColumnarExchange -Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(22) CometSort +(18) Exchange Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] -Arguments: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6], [i_class#9 ASC NULLS FIRST] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(23) ColumnarToRow [codegen id : 3] +(19) Sort [codegen id : 3] Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] +Arguments: [i_class#9 ASC NULLS FIRST], false, 0 -(24) Window +(20) Window Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6] Arguments: [sum(_w0#17) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#9] -(25) Project [codegen id : 4] +(21) Project [codegen id : 4] Output [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19, i_item_id#6] Input [8]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, i_item_id#6, _we0#18] -(26) RowToColumnar +(22) RowToColumnar Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19, i_item_id#6] -(27) CometColumnarExchange +(23) CometColumnarExchange Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19, i_item_id#6] Arguments: rangepartitioning(i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(28) CometSort +(24) CometSort Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19, i_item_id#6] Arguments: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19, i_item_id#6], [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] -(29) CometProject +(25) CometProject Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19, i_item_id#6] Arguments: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19], [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] -(30) ColumnarToRow [codegen id : 5] +(26) ColumnarToRow [codegen id : 5] Input [6]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (35) -+- * ColumnarToRow (34) - +- CometProject (33) - +- CometFilter (32) - +- CometScan parquet spark_catalog.default.date_dim (31) +BroadcastExchange (31) ++- * ColumnarToRow (30) + +- CometProject (29) + +- CometFilter (28) + +- CometScan parquet spark_catalog.default.date_dim (27) -(31) Scan parquet spark_catalog.default.date_dim +(27) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(32) CometFilter +(28) CometFilter Input [2]: [d_date_sk#11, d_date#12] Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 1999-02-22)) AND (d_date#12 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(33) CometProject +(29) CometProject Input [2]: [d_date_sk#11, d_date#12] Arguments: [d_date_sk#11], [d_date_sk#11] -(34) ColumnarToRow [codegen id : 1] +(30) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#11] -(35) BroadcastExchange +(31) BroadcastExchange Input [1]: [d_date_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt index 60e9051c7..11bd0f16e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q98/simplified.txt @@ -10,39 +10,35 @@ WholeStageCodegen (5) InputAdapter Window [_w0,i_class] WholeStageCodegen (3) - ColumnarToRow + Sort [i_class] InputAdapter - CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometColumnarExchange [i_class] #2 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + Exchange [i_class] #2 + WholeStageCodegen (2) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] + InputAdapter + Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt index 100603c4e..a6282e50c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/explain.txt @@ -1,34 +1,32 @@ == Physical Plan == -TakeOrderedAndProject (30) -+- * HashAggregate (29) - +- * ColumnarToRow (28) - +- CometColumnarExchange (27) - +- RowToColumnar (26) - +- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometProject (23) - +- CometBroadcastHashJoin (22) - :- CometProject (17) - : +- CometBroadcastHashJoin (16) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.warehouse (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.ship_mode (8) - : +- CometBroadcastExchange (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.call_center (13) - +- CometBroadcastExchange (21) - +- CometProject (20) - +- CometFilter (19) - +- CometScan parquet spark_catalog.default.date_dim (18) +TakeOrderedAndProject (28) ++- * HashAggregate (27) + +- Exchange (26) + +- * HashAggregate (25) + +- * ColumnarToRow (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometProject (17) + : +- CometBroadcastHashJoin (16) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.warehouse (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.ship_mode (8) + : +- CometBroadcastExchange (15) + : +- CometFilter (14) + : +- CometScan parquet spark_catalog.default.call_center (13) + +- CometBroadcastExchange (21) + +- CometProject (20) + +- CometFilter (19) + +- CometScan parquet spark_catalog.default.date_dim (18) (1) Scan parquet spark_catalog.default.catalog_sales @@ -152,24 +150,18 @@ Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) < Aggregate Attributes [5]: [sum#15, sum#16, sum#17, sum#18, sum#19] Results [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -(26) RowToColumnar +(26) Exchange Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] +Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(27) CometColumnarExchange -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(28) ColumnarToRow [codegen id : 2] -Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] - -(29) HashAggregate [codegen id : 2] +(27) HashAggregate [codegen id : 2] Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#20, sum#21, sum#22, sum#23, sum#24] Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] Functions [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] Aggregate Attributes [5]: [sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29] Results [8]: [_groupingexpression#14 AS substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END)#25 AS 30 days #31, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END)#26 AS 31 - 60 days #32, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END)#27 AS 61 - 90 days #33, sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END)#28 AS 91 - 120 days #34, sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)#29 AS >120 days #35] -(30) TakeOrderedAndProject +(28) TakeOrderedAndProject Input [8]: [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] Arguments: 100, [substr(w_warehouse_name, 1, 20)#30 ASC NULLS FIRST, sm_type#9 ASC NULLS FIRST, cc_name#11 ASC NULLS FIRST], [substr(w_warehouse_name, 1, 20)#30, sm_type#9, cc_name#11, 30 days #31, 31 - 60 days #32, 61 - 90 days #33, 91 - 120 days #34, >120 days #35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/simplified.txt index 26e56c432..25ff3a2fe 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4-spark4_0/q99/simplified.txt @@ -1,34 +1,32 @@ TakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] WholeStageCodegen (2) HashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum] [sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END),substr(w_warehouse_name, 1, 20),30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [_groupingexpression,sm_type,cc_name] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [_groupingexpression,sm_type,cc_name,cs_ship_date_sk,cs_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] - CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] - CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_call_center_sk,cc_name] - CometProject [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] - CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] - CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 - CometFilter [sm_ship_mode_sk,sm_type] - CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] - CometBroadcastExchange [cc_call_center_sk,cc_name] #4 - CometFilter [cc_call_center_sk,cc_name] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + Exchange [_groupingexpression,sm_type,cc_name] #1 + WholeStageCodegen (1) + HashAggregate [_groupingexpression,sm_type,cc_name,cs_ship_date_sk,cs_sold_date_sk] [sum,sum,sum,sum,sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] + CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] + CometProject [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_call_center_sk,cc_name] + CometProject [cs_ship_date_sk,cs_call_center_sk,cs_sold_date_sk,w_warehouse_name,sm_type] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name,sm_ship_mode_sk,sm_type] + CometProject [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_sold_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk,w_warehouse_sk,w_warehouse_name] + CometFilter [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_call_center_sk,cs_ship_mode_sk,cs_warehouse_sk,cs_sold_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #2 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [sm_ship_mode_sk,sm_type] #3 + CometFilter [sm_ship_mode_sk,sm_type] + CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_type] + CometBroadcastExchange [cc_call_center_sk,cc_name] #4 + CometFilter [cc_call_center_sk,cc_name] + CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt index 4c34d3b3e..747377eb6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/explain.txt @@ -9,7 +9,7 @@ : : +- CometBroadcastHashJoin (26) : : :- CometFilter (12) : : : +- CometHashAggregate (11) - : : : +- CometColumnarExchange (10) + : : : +- CometExchange (10) : : : +- CometHashAggregate (9) : : : +- CometProject (8) : : : +- CometBroadcastHashJoin (7) @@ -22,10 +22,10 @@ : : +- CometBroadcastExchange (25) : : +- CometFilter (24) : : +- CometHashAggregate (23) - : : +- CometColumnarExchange (22) + : : +- CometExchange (22) : : +- CometHashAggregate (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -86,9 +86,9 @@ Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -(10) CometColumnarExchange +(10) CometExchange Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#8] @@ -128,9 +128,9 @@ Input [3]: [sr_customer_sk#1, sr_store_sk#2, sr_return_amt#3] Keys [2]: [sr_customer_sk#1, sr_store_sk#2] Functions [1]: [partial_sum(UnscaledValue(sr_return_amt#3))] -(19) CometColumnarExchange +(19) CometExchange Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#13] -Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_customer_sk#1, sr_store_sk#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometHashAggregate Input [3]: [sr_customer_sk#1, sr_store_sk#2, sum#13] @@ -142,9 +142,9 @@ Input [2]: [ctr_store_sk#10, ctr_total_return#11] Keys [1]: [ctr_store_sk#10] Functions [1]: [partial_avg(ctr_total_return#11)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ctr_store_sk#10, sum#14, count#15] -Arguments: hashpartitioning(ctr_store_sk#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ctr_store_sk#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (23) CometHashAggregate Input [3]: [ctr_store_sk#10, sum#14, count#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt index eb1ae9719..10d410761 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q1/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ctr_customer_sk,ctr_store_sk,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_store_sk] CometFilter [ctr_customer_sk,ctr_store_sk,ctr_total_return] CometHashAggregate [ctr_customer_sk,ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum,sum(UnscaledValue(sr_return_amt))] - CometColumnarExchange [sr_customer_sk,sr_store_sk] #1 + CometExchange [sr_customer_sk,sr_store_sk] #1 CometHashAggregate [sr_customer_sk,sr_store_sk,sum,sr_return_amt] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] @@ -31,10 +31,10 @@ WholeStageCodegen (1) CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_store_sk] #4 CometFilter [(avg(ctr_total_return) * 1.2),ctr_store_sk] CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_store_sk,ctr_store_sk,sum,count,avg(ctr_total_return)] - CometColumnarExchange [ctr_store_sk] #5 + CometExchange [ctr_store_sk] #5 CometHashAggregate [ctr_store_sk,sum,count,ctr_total_return] CometHashAggregate [ctr_store_sk,ctr_total_return,sr_customer_sk,sr_store_sk,sum,sum(UnscaledValue(sr_return_amt))] - CometColumnarExchange [sr_customer_sk,sr_store_sk] #6 + CometExchange [sr_customer_sk,sr_store_sk] #6 CometHashAggregate [sr_customer_sk,sr_store_sk,sum,sr_return_amt] CometProject [sr_customer_sk,sr_store_sk,sr_return_amt] CometBroadcastHashJoin [sr_customer_sk,sr_store_sk,sr_return_amt,sr_returned_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt index 6f8c539fb..4a29b7260 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/explain.txt @@ -1,51 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (47) -+- * HashAggregate (46) - +- * ColumnarToRow (45) - +- CometColumnarExchange (44) - +- RowToColumnar (43) - +- * HashAggregate (42) - +- * Project (41) - +- * BroadcastHashJoin Inner BuildRight (40) - :- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (33) - : +- * ColumnarToRow (32) - : +- CometProject (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (39) - +- * ColumnarToRow (38) - +- CometFilter (37) - +- CometScan parquet spark_catalog.default.customer_demographics (36) +TakeOrderedAndProject (45) ++- * HashAggregate (44) + +- Exchange (43) + +- * HashAggregate (42) + +- * Project (41) + +- * BroadcastHashJoin Inner BuildRight (40) + :- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (33) + : +- * ColumnarToRow (32) + : +- CometProject (31) + : +- CometFilter (30) + : +- CometScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (39) + +- * ColumnarToRow (38) + +- CometFilter (37) + +- CometScan parquet spark_catalog.default.customer_demographics (36) (1) Scan parquet spark_catalog.default.customer @@ -245,56 +243,50 @@ Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#31] Results [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] -(43) RowToColumnar +(43) Exchange Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(44) CometColumnarExchange -Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] -Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(45) ColumnarToRow [codegen id : 6] -Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] - -(46) HashAggregate [codegen id : 6] +(44) HashAggregate [codegen id : 6] Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#32] Keys [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#33] Results [14]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, count(1)#33 AS cnt1#34, cd_purchase_estimate#26, count(1)#33 AS cnt2#35, cd_credit_rating#27, count(1)#33 AS cnt3#36, cd_dep_count#28, count(1)#33 AS cnt4#37, cd_dep_employed_count#29, count(1)#33 AS cnt5#38, cd_dep_college_count#30, count(1)#33 AS cnt6#39] -(47) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [14]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#34, cd_purchase_estimate#26, cnt2#35, cd_credit_rating#27, cnt3#36, cd_dep_count#28, cnt4#37, cd_dep_employed_count#29, cnt5#38, cd_dep_college_count#30, cnt6#39] Arguments: 100, [cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_education_status#25 ASC NULLS FIRST, cd_purchase_estimate#26 ASC NULLS FIRST, cd_credit_rating#27 ASC NULLS FIRST, cd_dep_count#28 ASC NULLS FIRST, cd_dep_employed_count#29 ASC NULLS FIRST, cd_dep_college_count#30 ASC NULLS FIRST], [cd_gender#23, cd_marital_status#24, cd_education_status#25, cnt1#34, cd_purchase_estimate#26, cnt2#35, cd_credit_rating#27, cnt3#36, cd_dep_count#28, cnt4#37, cd_dep_employed_count#29, cnt5#38, cd_dep_college_count#30, cnt6#39] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (52) -+- * ColumnarToRow (51) - +- CometProject (50) - +- CometFilter (49) - +- CometScan parquet spark_catalog.default.date_dim (48) +BroadcastExchange (50) ++- * ColumnarToRow (49) + +- CometProject (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(48) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2002), GreaterThanOrEqual(d_moy,1), LessThanOrEqual(d_moy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(49) CometFilter +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Condition : (((((isnotnull(d_year#10) AND isnotnull(d_moy#11)) AND (d_year#10 = 2002)) AND (d_moy#11 >= 1)) AND (d_moy#11 <= 4)) AND isnotnull(d_date_sk#9)) -(50) CometProject +(48) CometProject Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Arguments: [d_date_sk#9], [d_date_sk#9] -(51) ColumnarToRow [codegen id : 1] +(49) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(52) BroadcastExchange +(50) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt index a3ab6bbbb..efd4b187d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q10/simplified.txt @@ -1,73 +1,71 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,cnt2,cnt3,cnt4,cnt5,cnt6] WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] [count(1),cnt1,cnt2,cnt3,cnt4,cnt5,cnt6,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + InputAdapter + Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_county] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_county] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt index ac0d82f80..0346efa54 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/explain.txt @@ -9,7 +9,7 @@ : : +- CometBroadcastHashJoin (33) : : :- CometFilter (16) : : : +- CometHashAggregate (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometHashAggregate (13) : : : +- CometProject (12) : : : +- CometBroadcastHashJoin (11) @@ -25,7 +25,7 @@ : : : +- CometScan parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometProject (28) : : +- CometBroadcastHashJoin (27) @@ -42,7 +42,7 @@ : +- CometBroadcastExchange (49) : +- CometFilter (48) : +- CometHashAggregate (47) - : +- CometColumnarExchange (46) + : +- CometExchange (46) : +- CometHashAggregate (45) : +- CometProject (44) : +- CometBroadcastHashJoin (43) @@ -56,7 +56,7 @@ : +- ReusedExchange (42) +- CometBroadcastExchange (65) +- CometHashAggregate (64) - +- CometColumnarExchange (63) + +- CometExchange (63) +- CometHashAggregate (62) +- CometProject (61) +- CometBroadcastHashJoin (60) @@ -135,9 +135,9 @@ Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_fl Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -(14) CometColumnarExchange +(14) CometExchange Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] @@ -213,9 +213,9 @@ Input [10]: [c_customer_id#20, c_first_name#21, c_last_name#22, c_preferred_cust Keys [8]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#29 - ss_ext_discount_amt#28)))] -(30) CometColumnarExchange +(30) CometExchange Input [9]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, sum#34] -Arguments: hashpartitioning(c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate Input [9]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, sum#34] @@ -288,9 +288,9 @@ Input [10]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust Keys [8]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#52] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#48 - ws_ext_discount_amt#47)))] -(46) CometColumnarExchange +(46) CometExchange Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#52, sum#53] -Arguments: hashpartitioning(c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#52, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#52, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (47) CometHashAggregate Input [9]: [c_customer_id#39, c_first_name#40, c_last_name#41, c_preferred_cust_flag#42, c_birth_country#43, c_login#44, c_email_address#45, d_year#52, sum#53] @@ -367,9 +367,9 @@ Input [10]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust Keys [8]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#70] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#66 - ws_ext_discount_amt#65)))] -(63) CometColumnarExchange +(63) CometExchange Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#70, sum#71] -Arguments: hashpartitioning(c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#70, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#70, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (64) CometHashAggregate Input [9]: [c_customer_id#57, c_first_name#58, c_last_name#59, c_preferred_cust_flag#60, c_birth_country#61, c_login#62, c_email_address#63, d_year#70, sum#71] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt index 83ffbd0a6..e509c8c3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q11/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_preferred_cust_flag,year_total] CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -33,7 +33,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_preferred_cust_flag,year_total] #5 CometHashAggregate [customer_id,customer_preferred_cust_flag,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -57,7 +57,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -72,7 +72,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt index 40c6660a9..572f500c9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (22) +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -92,18 +92,18 @@ Input [6]: [ws_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt index 3e0fd16a4..e81a50713 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q12/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c ColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometColumnarExchange [i_class] #1 + CometExchange [i_class] #1 CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt index b05df2927..7d035666a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (33) +- CometHashAggregate (32) - +- CometColumnarExchange (31) + +- CometExchange (31) +- CometHashAggregate (30) +- CometProject (29) +- CometBroadcastHashJoin (28) @@ -179,9 +179,9 @@ Input [3]: [ss_quantity#5, ss_ext_sales_price#7, ss_ext_wholesale_cost#8] Keys: [] Functions [4]: [partial_avg(ss_quantity#5), partial_avg(UnscaledValue(ss_ext_sales_price#7)), partial_avg(UnscaledValue(ss_ext_wholesale_cost#8)), partial_sum(UnscaledValue(ss_ext_wholesale_cost#8))] -(31) CometColumnarExchange +(31) CometExchange Input [7]: [sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [7]: [sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt index 0f438f153..46ed9b636 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q13/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [avg(ss_quantity),avg(ss_ext_sales_price),avg(ss_ext_wholesale_cost),sum(ss_ext_wholesale_cost),sum,count,sum,count,sum,count,sum,avg(ss_quantity),avg(UnscaledValue(ss_ext_sales_price)),avg(UnscaledValue(ss_ext_wholesale_cost)),sum(UnscaledValue(ss_ext_wholesale_cost))] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,count,sum,count,sum,count,sum,ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] CometProject [ss_quantity,ss_ext_sales_price,ss_ext_wholesale_cost] CometBroadcastHashJoin [ss_hdemo_sk,ss_quantity,ss_sales_price,ss_ext_sales_price,ss_ext_wholesale_cost,cd_marital_status,cd_education_status,hd_demo_sk,hd_dep_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt index 6789d55ce..c7dc4b3c0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/explain.txt @@ -2,14 +2,14 @@ * ColumnarToRow (102) +- CometTakeOrderedAndProject (101) +- CometHashAggregate (100) - +- CometColumnarExchange (99) + +- CometExchange (99) +- CometHashAggregate (98) +- CometExpand (97) +- CometUnion (96) :- CometProject (65) : +- CometFilter (64) : +- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometProject (60) : +- CometBroadcastHashJoin (59) @@ -26,7 +26,7 @@ : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) - : : : : +- CometColumnarExchange (31) + : : : : +- CometExchange (31) : : : : +- CometHashAggregate (30) : : : : +- CometProject (29) : : : : +- CometBroadcastHashJoin (28) @@ -74,7 +74,7 @@ :- CometProject (80) : +- CometFilter (79) : +- CometHashAggregate (78) - : +- CometColumnarExchange (77) + : +- CometExchange (77) : +- CometHashAggregate (76) : +- CometProject (75) : +- CometBroadcastHashJoin (74) @@ -89,7 +89,7 @@ +- CometProject (95) +- CometFilter (94) +- CometHashAggregate (93) - +- CometColumnarExchange (92) + +- CometExchange (92) +- CometHashAggregate (91) +- CometProject (90) +- CometBroadcastHashJoin (89) @@ -252,9 +252,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -393,9 +393,9 @@ Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_cate Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(62) CometColumnarExchange +(62) CometExchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] @@ -459,9 +459,9 @@ Input [5]: [cs_quantity#58, cs_list_price#59, i_brand_id#63, i_class_id#64, i_ca Keys [3]: [i_brand_id#63, i_class_id#64, i_category_id#65] Functions [2]: [partial_sum((cast(cs_quantity#58 as decimal(10,0)) * cs_list_price#59)), partial_count(1)] -(77) CometColumnarExchange +(77) CometExchange Input [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#67, isEmpty#68, count#69] -Arguments: hashpartitioning(i_brand_id#63, i_class_id#64, i_category_id#65, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_brand_id#63, i_class_id#64, i_category_id#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (78) CometHashAggregate Input [6]: [i_brand_id#63, i_class_id#64, i_category_id#65, sum#67, isEmpty#68, count#69] @@ -525,9 +525,9 @@ Input [5]: [ws_quantity#74, ws_list_price#75, i_brand_id#79, i_class_id#80, i_ca Keys [3]: [i_brand_id#79, i_class_id#80, i_category_id#81] Functions [2]: [partial_sum((cast(ws_quantity#74 as decimal(10,0)) * ws_list_price#75)), partial_count(1)] -(92) CometColumnarExchange +(92) CometExchange Input [6]: [i_brand_id#79, i_class_id#80, i_category_id#81, sum#83, isEmpty#84, count#85] -Arguments: hashpartitioning(i_brand_id#79, i_class_id#80, i_category_id#81, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_brand_id#79, i_class_id#80, i_category_id#81, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (93) CometHashAggregate Input [6]: [i_brand_id#79, i_class_id#80, i_category_id#81, sum#83, isEmpty#84, count#85] @@ -556,9 +556,9 @@ Input [7]: [sales#49, number_sales#50, channel#89, i_brand_id#90, i_class_id#91, Keys [5]: [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93] Functions [2]: [partial_sum(sales#49), partial_sum(number_sales#50)] -(99) CometColumnarExchange +(99) CometExchange Input [8]: [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93, sum#94, isEmpty#95, sum#96] -Arguments: hashpartitioning(channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (100) CometHashAggregate Input [8]: [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, spark_grouping_id#93, sum#94, isEmpty#95, sum#96] @@ -577,7 +577,7 @@ Input [6]: [channel#89, i_brand_id#90, i_class_id#91, i_category_id#92, sum(sale Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#51, [id=#52] * ColumnarToRow (119) +- CometHashAggregate (118) - +- CometColumnarExchange (117) + +- CometExchange (117) +- CometHashAggregate (116) +- CometUnion (115) :- CometProject (106) @@ -661,9 +661,9 @@ Input [2]: [quantity#104, list_price#105] Keys: [] Functions [1]: [partial_avg((cast(quantity#104 as decimal(10,0)) * list_price#105))] -(117) CometColumnarExchange +(117) CometExchange Input [2]: [sum#120, count#121] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (118) CometHashAggregate Input [2]: [sum#120, count#121] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt index bb69fc92a..852af8ad8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14a/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales)] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sales),sum(number_sales),spark_grouping_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] - CometColumnarExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 + CometExchange [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] #1 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id,sum,isEmpty,sum,sales,number_sales] CometExpand [channel,i_brand_id,i_class_id,i_category_id] [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,spark_grouping_id] CometUnion [sales,number_sales,channel,i_brand_id,i_class_id,i_category_id] @@ -14,7 +14,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometColumnarExchange #15 + CometExchange #15 CometHashAggregate [sum,count,quantity,list_price] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] @@ -33,7 +33,7 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #2 ReusedExchange [d_date_sk] #11 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #2 + CometExchange [i_brand_id,i_class_id,i_category_id] #2 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -58,7 +58,7 @@ WholeStageCodegen (1) CometBroadcastExchange [brand_id,class_id,category_id] #5 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #6 + CometExchange [brand_id,class_id,category_id] #6 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -117,7 +117,7 @@ WholeStageCodegen (1) CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #16 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,cs_quantity,cs_list_price] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -134,7 +134,7 @@ WholeStageCodegen (1) CometFilter [i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #17 + CometExchange [i_brand_id,i_class_id,i_category_id] #17 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ws_quantity,ws_list_price] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt index 199939cd8..9585c3214 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/explain.txt @@ -4,7 +4,7 @@ +- CometBroadcastHashJoin (83) :- CometFilter (64) : +- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometProject (60) : +- CometBroadcastHashJoin (59) @@ -21,7 +21,7 @@ : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) - : : : : +- CometColumnarExchange (31) + : : : : +- CometExchange (31) : : : : +- CometHashAggregate (30) : : : : +- CometProject (29) : : : : +- CometBroadcastHashJoin (28) @@ -69,7 +69,7 @@ +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) - +- CometColumnarExchange (79) + +- CometExchange (79) +- CometHashAggregate (78) +- CometProject (77) +- CometBroadcastHashJoin (76) @@ -235,9 +235,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -376,9 +376,9 @@ Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_cate Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(62) CometColumnarExchange +(62) CometExchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] @@ -454,9 +454,9 @@ Input [5]: [ss_quantity#56, ss_list_price#57, i_brand_id#61, i_class_id#62, i_ca Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] Functions [2]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), partial_count(1)] -(79) CometColumnarExchange +(79) CometExchange Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#68, isEmpty#69, count#70] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (80) CometHashAggregate Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#68, isEmpty#69, count#70] @@ -488,7 +488,7 @@ Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#5 Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#53, [id=#54] * ColumnarToRow (102) +- CometHashAggregate (101) - +- CometColumnarExchange (100) + +- CometExchange (100) +- CometHashAggregate (99) +- CometUnion (98) :- CometProject (89) @@ -572,9 +572,9 @@ Input [2]: [quantity#79, list_price#80] Keys: [] Functions [1]: [partial_avg((cast(quantity#79 as decimal(10,0)) * list_price#80))] -(100) CometColumnarExchange +(100) CometExchange Input [2]: [sum#95, count#96] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (101) CometHashAggregate Input [2]: [sum#95, count#96] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt index a51d1d007..bb4be4d3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q14b/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometColumnarExchange #14 + CometExchange #14 CometHashAggregate [sum,count,quantity,list_price] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] @@ -28,7 +28,7 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #1 + CometExchange [i_brand_id,i_class_id,i_category_id] #1 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -60,7 +60,7 @@ WholeStageCodegen (1) CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #5 + CometExchange [brand_id,class_id,category_id] #5 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -120,7 +120,7 @@ WholeStageCodegen (1) CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #16 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt index af9c0909f..842c3e576 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (23) +- CometTakeOrderedAndProject (22) +- CometHashAggregate (21) - +- CometColumnarExchange (20) + +- CometExchange (20) +- CometHashAggregate (19) +- CometProject (18) +- CometBroadcastHashJoin (17) @@ -117,9 +117,9 @@ Input [2]: [cs_sales_price#2, ca_zip#9] Keys [1]: [ca_zip#9] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#2))] -(20) CometColumnarExchange +(20) CometExchange Input [2]: [ca_zip#9, sum#13] -Arguments: hashpartitioning(ca_zip#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ca_zip#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [2]: [ca_zip#9, sum#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt index 989ea0856..d417f1c68 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q15/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [ca_zip,sum(cs_sales_price)] CometHashAggregate [ca_zip,sum(cs_sales_price),sum,sum(UnscaledValue(cs_sales_price))] - CometColumnarExchange [ca_zip] #1 + CometExchange [ca_zip] #1 CometHashAggregate [ca_zip,sum,cs_sales_price] CometProject [cs_sales_price,ca_zip] CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,ca_zip,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt index ebff95d06..4890b4436 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/explain.txt @@ -1,51 +1,49 @@ == Physical Plan == -* HashAggregate (47) -+- * ColumnarToRow (46) - +- CometColumnarExchange (45) - +- RowToColumnar (44) - +- * HashAggregate (43) - +- * HashAggregate (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (33) - : +- * BroadcastHashJoin Inner BuildRight (32) - : :- * Project (26) - : : +- * BroadcastHashJoin Inner BuildRight (25) - : : :- * SortMergeJoin LeftAnti (19) - : : : :- * Project (13) - : : : : +- * SortMergeJoin LeftSemi (12) - : : : : :- * ColumnarToRow (6) - : : : : : +- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : +- * ColumnarToRow (11) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (7) - : : : +- * ColumnarToRow (18) - : : : +- CometSort (17) - : : : +- CometColumnarExchange (16) - : : : +- CometProject (15) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (14) - : : +- BroadcastExchange (24) - : : +- * ColumnarToRow (23) - : : +- CometProject (22) - : : +- CometFilter (21) - : : +- CometScan parquet spark_catalog.default.date_dim (20) - : +- BroadcastExchange (31) - : +- * ColumnarToRow (30) - : +- CometProject (29) - : +- CometFilter (28) - : +- CometScan parquet spark_catalog.default.customer_address (27) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometProject (36) - +- CometFilter (35) - +- CometScan parquet spark_catalog.default.call_center (34) +* HashAggregate (45) ++- Exchange (44) + +- * HashAggregate (43) + +- * HashAggregate (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (33) + : +- * BroadcastHashJoin Inner BuildRight (32) + : :- * Project (26) + : : +- * BroadcastHashJoin Inner BuildRight (25) + : : :- * SortMergeJoin LeftAnti (19) + : : : :- * Project (13) + : : : : +- * SortMergeJoin LeftSemi (12) + : : : : :- * ColumnarToRow (6) + : : : : : +- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- * ColumnarToRow (11) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (7) + : : : +- * ColumnarToRow (18) + : : : +- CometSort (17) + : : : +- CometExchange (16) + : : : +- CometProject (15) + : : : +- CometScan parquet spark_catalog.default.catalog_returns (14) + : : +- BroadcastExchange (24) + : : +- * ColumnarToRow (23) + : : +- CometProject (22) + : : +- CometFilter (21) + : : +- CometScan parquet spark_catalog.default.date_dim (20) + : +- BroadcastExchange (31) + : +- * ColumnarToRow (30) + : +- CometProject (29) + : +- CometFilter (28) + : +- CometScan parquet spark_catalog.default.customer_address (27) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometProject (36) + +- CometFilter (35) + +- CometScan parquet spark_catalog.default.call_center (34) (1) Scan parquet spark_catalog.default.catalog_sales @@ -63,9 +61,9 @@ Condition : ((isnotnull(cs_ship_date_sk#1) AND isnotnull(cs_ship_addr_sk#2)) AND Input [8]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7, cs_sold_date_sk#8] Arguments: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7], [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -(4) CometColumnarExchange +(4) CometExchange Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] -Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [7]: [cs_ship_date_sk#1, cs_ship_addr_sk#2, cs_call_center_sk#3, cs_warehouse_sk#4, cs_order_number#5, cs_ext_ship_cost#6, cs_net_profit#7] @@ -84,9 +82,9 @@ ReadSchema: struct Input [3]: [cs_warehouse_sk#9, cs_order_number#10, cs_sold_date_sk#11] Arguments: [cs_warehouse_sk#9, cs_order_number#10], [cs_warehouse_sk#9, cs_order_number#10] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [cs_warehouse_sk#9, cs_order_number#10] -Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [cs_warehouse_sk#9, cs_order_number#10] @@ -115,9 +113,9 @@ ReadSchema: struct Input [2]: [cr_order_number#12, cr_returned_date_sk#13] Arguments: [cr_order_number#12], [cr_order_number#12] -(16) CometColumnarExchange +(16) CometExchange Input [1]: [cr_order_number#12] -Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (17) CometSort Input [1]: [cr_order_number#12] @@ -249,17 +247,11 @@ Functions [3]: [merge_sum(UnscaledValue(cs_ext_ship_cost#6)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_ship_cost#6))#20, sum(UnscaledValue(cs_net_profit#7))#21, count(cs_order_number#5)#24] Results [3]: [sum#22, sum#23, count#25] -(44) RowToColumnar +(44) Exchange Input [3]: [sum#22, sum#23, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(45) CometColumnarExchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(46) ColumnarToRow [codegen id : 9] -Input [3]: [sum#22, sum#23, count#25] - -(47) HashAggregate [codegen id : 9] +(45) HashAggregate [codegen id : 9] Input [3]: [sum#22, sum#23, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(cs_ext_ship_cost#6)), sum(UnscaledValue(cs_net_profit#7)), count(distinct cs_order_number#5)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt index 605ce3703..18ba39326 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q16/simplified.txt @@ -1,70 +1,68 @@ WholeStageCodegen (9) HashAggregate [sum,sum,count] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (8) - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] - HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] - Project [cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] - Project [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk] - Project [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk] - SortMergeJoin [cs_order_number,cr_order_number] - InputAdapter - WholeStageCodegen (3) - Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometColumnarExchange [cs_order_number] #2 - CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] - CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [cs_warehouse_sk,cs_order_number] - CometColumnarExchange [cs_order_number] #3 - CometProject [cs_warehouse_sk,cs_order_number] - CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [cr_order_number] - CometColumnarExchange [cr_order_number] #4 - CometProject [cr_order_number] - CometScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (8) + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),count(cs_order_number),sum,sum,count,sum,sum,count] + HashAggregate [cs_order_number] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + HashAggregate [cs_order_number,cs_ext_ship_cost,cs_net_profit] [sum(UnscaledValue(cs_ext_ship_cost)),sum(UnscaledValue(cs_net_profit)),sum,sum,sum,sum] + Project [cs_order_number,cs_ext_ship_cost,cs_net_profit] + BroadcastHashJoin [cs_call_center_sk,cc_call_center_sk] + Project [cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + BroadcastHashJoin [cs_ship_addr_sk,ca_address_sk] + Project [cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + BroadcastHashJoin [cs_ship_date_sk,d_date_sk] + SortMergeJoin [cs_order_number,cr_order_number] + InputAdapter + WholeStageCodegen (3) + Project [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + SortMergeJoin [cs_order_number,cs_order_number,cs_warehouse_sk,cs_warehouse_sk] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometExchange [cs_order_number] #2 + CometProject [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit] + CometFilter [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_ship_addr_sk,cs_call_center_sk,cs_warehouse_sk,cs_order_number,cs_ext_ship_cost,cs_net_profit,cs_sold_date_sk] + InputAdapter + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometSort [cs_warehouse_sk,cs_order_number] + CometExchange [cs_order_number] #3 + CometProject [cs_warehouse_sk,cs_order_number] + CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_order_number,cs_sold_date_sk] + InputAdapter + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [cr_order_number] + CometExchange [cr_order_number] #4 + CometProject [cr_order_number] + CometScan parquet spark_catalog.default.catalog_returns [cr_order_number,cr_returned_date_sk] InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) + BroadcastExchange #5 + WholeStageCodegen (5) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) + BroadcastExchange #6 + WholeStageCodegen (6) ColumnarToRow InputAdapter - CometProject [cc_call_center_sk] - CometFilter [cc_call_center_sk,cc_county] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometProject [cc_call_center_sk] + CometFilter [cc_call_center_sk,cc_county] + CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt index ddf62fe94..69f720e6f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (42) +- CometTakeOrderedAndProject (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -226,9 +226,9 @@ Input [6]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_state#25, i_ Keys [3]: [i_item_id#27, i_item_desc#28, s_state#25] Functions [9]: [partial_count(ss_quantity#5), partial_avg(ss_quantity#5), partial_stddev_samp(cast(ss_quantity#5 as double)), partial_count(sr_return_quantity#11), partial_avg(sr_return_quantity#11), partial_stddev_samp(cast(sr_return_quantity#11 as double)), partial_count(cs_quantity#16), partial_avg(cs_quantity#16), partial_stddev_samp(cast(cs_quantity#16 as double))] -(39) CometColumnarExchange +(39) CometExchange Input [21]: [i_item_id#27, i_item_desc#28, s_state#25, count#29, sum#30, count#31, n#32, avg#33, m2#34, count#35, sum#36, count#37, n#38, avg#39, m2#40, count#41, sum#42, count#43, n#44, avg#45, m2#46] -Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_state#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#27, i_item_desc#28, s_state#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate Input [21]: [i_item_id#27, i_item_desc#28, s_state#25, count#29, sum#30, count#31, n#32, avg#33, m2#34, count#35, sum#36, count#37, n#38, avg#39, m2#40, count#41, sum#42, count#43, n#44, avg#45, m2#46] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt index e350dfe51..a3adaf172 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q17/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov] CometHashAggregate [i_item_id,i_item_desc,s_state,store_sales_quantitycount,store_sales_quantityave,store_sales_quantitystdev,store_sales_quantitycov,as_store_returns_quantitycount,as_store_returns_quantityave,as_store_returns_quantitystdev,store_returns_quantitycov,catalog_sales_quantitycount,catalog_sales_quantityave,catalog_sales_quantitystdev,catalog_sales_quantitycov,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count(ss_quantity),avg(ss_quantity),stddev_samp(cast(ss_quantity as double)),count(sr_return_quantity),avg(sr_return_quantity),stddev_samp(cast(sr_return_quantity as double)),count(cs_quantity),avg(cs_quantity),stddev_samp(cast(cs_quantity as double))] - CometColumnarExchange [i_item_id,i_item_desc,s_state] #1 + CometExchange [i_item_id,i_item_desc,s_state] #1 CometHashAggregate [i_item_id,i_item_desc,s_state,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,count,sum,count,n,avg,m2,ss_quantity,sr_return_quantity,cs_quantity] CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_state,i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt index 6c969939d..bd8c5eb57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (41) +- CometTakeOrderedAndProject (40) +- CometHashAggregate (39) - +- CometColumnarExchange (38) + +- CometExchange (38) +- CometHashAggregate (37) +- CometExpand (36) +- CometProject (35) @@ -219,9 +219,9 @@ Input [12]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, Keys [5]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33] Functions [7]: [partial_avg(cast(cs_quantity#4 as decimal(12,2))), partial_avg(cast(cs_list_price#5 as decimal(12,2))), partial_avg(cast(cs_coupon_amt#7 as decimal(12,2))), partial_avg(cast(cs_sales_price#6 as decimal(12,2))), partial_avg(cast(cs_net_profit#8 as decimal(12,2))), partial_avg(cast(c_birth_year#19 as decimal(12,2))), partial_avg(cast(cd_dep_count#14 as decimal(12,2)))] -(38) CometColumnarExchange +(38) CometExchange Input [19]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, sum#34, count#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47] -Arguments: hashpartitioning(i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (39) CometHashAggregate Input [19]: [i_item_id#29, ca_country#30, ca_state#31, ca_county#32, spark_grouping_id#33, sum#34, count#35, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt index 8182da775..e26491c76 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q18/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(cast(cs_quantity as decimal(12,2))),avg(cast(cs_list_price as decimal(12,2))),avg(cast(cs_coupon_amt as decimal(12,2))),avg(cast(cs_sales_price as decimal(12,2))),avg(cast(cs_net_profit as decimal(12,2))),avg(cast(c_birth_year as decimal(12,2))),avg(cast(cd_dep_count as decimal(12,2)))] - CometColumnarExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 + CometExchange [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] #1 CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] CometExpand [i_item_id,ca_country,ca_state,ca_county] [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county,spark_grouping_id] CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_id,ca_country,ca_state,ca_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt index 2b3480d80..2191317fd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (34) +- CometTakeOrderedAndProject (33) +- CometHashAggregate (32) - +- CometColumnarExchange (31) + +- CometExchange (31) +- CometHashAggregate (30) +- CometProject (29) +- CometBroadcastHashJoin (28) @@ -180,9 +180,9 @@ Input [5]: [ss_ext_sales_price#7, i_brand_id#10, i_brand#11, i_manufact_id#12, i Keys [4]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#7))] -(31) CometColumnarExchange +(31) CometExchange Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#21] -Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [5]: [i_brand#11, i_brand_id#10, i_manufact_id#12, i_manufact#13, sum#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt index 8f81e32ff..46b0d650a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q19/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [brand_id,brand,i_manufact_id,i_manufact,ext_price] CometHashAggregate [brand_id,brand,i_manufact_id,i_manufact,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 + CometExchange [i_brand,i_brand_id,i_manufact_id,i_manufact] #1 CometHashAggregate [i_brand,i_brand_id,i_manufact_id,i_manufact,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,i_brand_id,i_brand,i_manufact_id,i_manufact,ca_zip,s_store_sk,s_zip] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt index 10ce239de..b1f3430ff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/explain.txt @@ -7,7 +7,7 @@ :- CometProject (19) : +- CometBroadcastHashJoin (18) : :- CometHashAggregate (13) - : : +- CometColumnarExchange (12) + : : +- CometExchange (12) : : +- CometHashAggregate (11) : : +- CometProject (10) : : +- CometBroadcastHashJoin (9) @@ -89,9 +89,9 @@ Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] Keys [1]: [d_week_seq#10] Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday ) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday ) THEN sales_price#4 END))] -(12) CometColumnarExchange +(12) CometExchange Input [8]: [d_week_seq#10, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] -Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(d_week_seq#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (13) CometHashAggregate Input [8]: [d_week_seq#10, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17, sum#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt index c5c06a0b6..ed772bade 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q2/simplified.txt @@ -8,7 +8,7 @@ WholeStageCodegen (1) CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] [d_week_seq1,sun_sales1,mon_sales1,tue_sales1,wed_sales1,thu_sales1,fri_sales1,sat_sales1] CometBroadcastHashJoin [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,d_week_seq] CometHashAggregate [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN sales_price END))] - CometColumnarExchange [d_week_seq] #2 + CometExchange [d_week_seq] #2 CometHashAggregate [d_week_seq,sum,sum,sum,sum,sum,sum,sum,d_day_name,sales_price] CometProject [sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [sold_date_sk,sales_price,d_date_sk,d_week_seq,d_day_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt index c6982f073..1af8f20f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (22) +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -92,18 +92,18 @@ Input [6]: [cs_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt index 9c30689f6..9a906085b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q20/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c ColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometColumnarExchange [i_class] #1 + CometExchange [i_class] #1 CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt index ff0f63b74..dce72db26 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (23) +- CometFilter (22) +- CometHashAggregate (21) - +- CometColumnarExchange (20) + +- CometExchange (20) +- CometHashAggregate (19) +- CometProject (18) +- CometBroadcastHashJoin (17) @@ -118,9 +118,9 @@ Input [4]: [inv_quantity_on_hand#3, w_warehouse_name#7, i_item_id#9, d_date#12] Keys [2]: [w_warehouse_name#7, i_item_id#9] Functions [2]: [partial_sum(CASE WHEN (d_date#12 < 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END), partial_sum(CASE WHEN (d_date#12 >= 2000-03-11) THEN inv_quantity_on_hand#3 ELSE 0 END)] -(20) CometColumnarExchange +(20) CometExchange Input [4]: [w_warehouse_name#7, i_item_id#9, sum#13, sum#14] -Arguments: hashpartitioning(w_warehouse_name#7, i_item_id#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(w_warehouse_name#7, i_item_id#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [4]: [w_warehouse_name#7, i_item_id#9, sum#13, sum#14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt index 1a4f175c4..6e483f604 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q21/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [w_warehouse_name,i_item_id,inv_before,inv_after] CometFilter [w_warehouse_name,i_item_id,inv_before,inv_after] CometHashAggregate [w_warehouse_name,i_item_id,inv_before,inv_after,sum,sum,sum(CASE WHEN (d_date < 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN inv_quantity_on_hand ELSE 0 END)] - CometColumnarExchange [w_warehouse_name,i_item_id] #1 + CometExchange [w_warehouse_name,i_item_id] #1 CometHashAggregate [w_warehouse_name,i_item_id,sum,sum,d_date,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,w_warehouse_name,i_item_id,d_date] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,w_warehouse_name,i_item_id,d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt index 1f7209c62..c3f3c8002 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (24) +- CometTakeOrderedAndProject (23) +- CometHashAggregate (22) - +- CometColumnarExchange (21) + +- CometExchange (21) +- CometHashAggregate (20) +- CometExpand (19) +- CometProject (18) @@ -122,9 +122,9 @@ Input [6]: [inv_quantity_on_hand#3, i_product_name#14, i_brand#15, i_class#16, i Keys [5]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -(21) CometColumnarExchange +(21) CometExchange Input [7]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, sum#19, count#20] -Arguments: hashpartitioning(i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate Input [7]: [i_product_name#14, i_brand#15, i_class#16, i_category#17, spark_grouping_id#18, sum#19, count#20] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt index 05d02283b..ed997b5d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q22/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_product_name,i_brand,i_class,i_category,qoh] CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,spark_grouping_id,sum,count,avg(inv_quantity_on_hand)] - CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + CometExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 CometHashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count,inv_quantity_on_hand] CometExpand [i_product_name,i_brand,i_class,i_category] [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category,spark_grouping_id] CometProject [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt index 107640c7a..7d42d368a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (67) +- CometHashAggregate (66) - +- CometColumnarExchange (65) + +- CometExchange (65) +- CometHashAggregate (64) +- CometUnion (63) :- CometProject (46) @@ -9,7 +9,7 @@ : :- CometProject (40) : : +- CometSortMergeJoin (39) : : :- CometSort (24) - : : : +- CometColumnarExchange (23) + : : : +- CometExchange (23) : : : +- CometProject (22) : : : +- CometBroadcastHashJoin (21) : : : :- CometScan parquet spark_catalog.default.catalog_sales (1) @@ -17,7 +17,7 @@ : : : +- CometProject (19) : : : +- CometFilter (18) : : : +- CometHashAggregate (17) - : : : +- CometColumnarExchange (16) + : : : +- CometExchange (16) : : : +- CometHashAggregate (15) : : : +- CometProject (14) : : : +- CometBroadcastHashJoin (13) @@ -36,7 +36,7 @@ : : +- CometProject (37) : : +- CometFilter (36) : : +- CometHashAggregate (35) - : : +- CometColumnarExchange (34) + : : +- CometExchange (34) : : +- CometHashAggregate (33) : : +- CometProject (32) : : +- CometBroadcastHashJoin (31) @@ -55,7 +55,7 @@ :- CometProject (59) : +- CometSortMergeJoin (58) : :- CometSort (52) - : : +- CometColumnarExchange (51) + : : +- CometExchange (51) : : +- CometProject (50) : : +- CometBroadcastHashJoin (49) : : :- CometScan parquet spark_catalog.default.web_sales (47) @@ -144,9 +144,9 @@ Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] Functions [1]: [partial_count(1)] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] @@ -174,9 +174,9 @@ Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(23) CometColumnarExchange +(23) CometExchange Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -226,9 +226,9 @@ Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] Keys [1]: [c_customer_sk#23] Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -(34) CometColumnarExchange +(34) CometExchange Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (35) CometHashAggregate Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] @@ -303,9 +303,9 @@ Arguments: [ws_item_sk#33], [item_sk#17], LeftSemi, BuildRight Input [5]: [ws_item_sk#33, ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] Arguments: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37], [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -(51) CometColumnarExchange +(51) CometExchange Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] -Arguments: hashpartitioning(ws_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ws_bill_customer_sk#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (52) CometSort Input [4]: [ws_bill_customer_sk#34, ws_quantity#35, ws_list_price#36, ws_sold_date_sk#37] @@ -361,9 +361,9 @@ Input [1]: [sales#32] Keys: [] Functions [1]: [partial_sum(sales#32)] -(65) CometColumnarExchange +(65) CometExchange Input [2]: [sum#41, isEmpty#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (66) CometHashAggregate Input [2]: [sum#41, isEmpty#42] @@ -438,10 +438,10 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:3 Hosting operator id = 36 Hosting Expression = Subquery scalar-subquery#27, [id=#28] * ColumnarToRow (95) +- CometHashAggregate (94) - +- CometColumnarExchange (93) + +- CometExchange (93) +- CometHashAggregate (92) +- CometHashAggregate (91) - +- CometColumnarExchange (90) + +- CometExchange (90) +- CometHashAggregate (89) +- CometProject (88) +- CometBroadcastHashJoin (87) @@ -513,9 +513,9 @@ Input [3]: [ss_quantity#45, ss_sales_price#46, c_customer_sk#49] Keys [1]: [c_customer_sk#49] Functions [1]: [partial_sum((cast(ss_quantity#45 as decimal(10,0)) * ss_sales_price#46))] -(90) CometColumnarExchange +(90) CometExchange Input [3]: [c_customer_sk#49, sum#52, isEmpty#53] -Arguments: hashpartitioning(c_customer_sk#49, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(c_customer_sk#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (91) CometHashAggregate Input [3]: [c_customer_sk#49, sum#52, isEmpty#53] @@ -527,9 +527,9 @@ Input [1]: [csales#54] Keys: [] Functions [1]: [partial_max(csales#54)] -(93) CometColumnarExchange +(93) CometExchange Input [1]: [max#55] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (94) CometHashAggregate Input [1]: [max#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt index 18cf2b2c6..7606f105f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23a/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [sum(sales),sum,isEmpty,sum(sales)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,isEmpty,sales] CometUnion [sales] CometProject [cs_quantity,cs_list_price] [sales] @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometProject [cs_quantity,cs_list_price,cs_sold_date_sk] CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometColumnarExchange [cs_bill_customer_sk] #2 + CometExchange [cs_bill_customer_sk] #2 CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] @@ -26,7 +26,7 @@ WholeStageCodegen (1) CometProject [item_sk] CometFilter [item_sk,cnt] CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometColumnarExchange [_groupingexpression,i_item_sk,d_date] #5 + CometExchange [_groupingexpression,i_item_sk,d_date] #5 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] @@ -57,10 +57,10 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [tpcds_cmax,max,max(csales)] - CometColumnarExchange #11 + CometExchange #11 CometHashAggregate [max,csales] CometHashAggregate [csales,c_customer_sk,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometColumnarExchange [c_customer_sk] #12 + CometExchange [c_customer_sk] #12 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] @@ -82,7 +82,7 @@ WholeStageCodegen (1) CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometColumnarExchange [c_customer_sk] #9 + CometExchange [c_customer_sk] #9 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] @@ -101,7 +101,7 @@ WholeStageCodegen (1) CometProject [ws_quantity,ws_list_price,ws_sold_date_sk] CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometColumnarExchange [ws_bill_customer_sk] #16 + CometExchange [ws_bill_customer_sk] #16 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt index 0f36e0d18..cd97d73b1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (86) +- CometUnion (85) :- CometHashAggregate (62) - : +- CometColumnarExchange (61) + : +- CometExchange (61) : +- CometHashAggregate (60) : +- CometProject (59) : +- CometBroadcastHashJoin (58) @@ -11,7 +11,7 @@ : : +- CometBroadcastHashJoin (52) : : :- CometSortMergeJoin (40) : : : :- CometSort (25) - : : : : +- CometColumnarExchange (24) + : : : : +- CometExchange (24) : : : : +- CometProject (23) : : : : +- CometBroadcastHashJoin (22) : : : : :- CometFilter (2) @@ -20,7 +20,7 @@ : : : : +- CometProject (20) : : : : +- CometFilter (19) : : : : +- CometHashAggregate (18) - : : : : +- CometColumnarExchange (17) + : : : : +- CometExchange (17) : : : : +- CometHashAggregate (16) : : : : +- CometProject (15) : : : : +- CometBroadcastHashJoin (14) @@ -39,7 +39,7 @@ : : : +- CometProject (38) : : : +- CometFilter (37) : : : +- CometHashAggregate (36) - : : : +- CometColumnarExchange (35) + : : : +- CometExchange (35) : : : +- CometHashAggregate (34) : : : +- CometProject (33) : : : +- CometBroadcastHashJoin (32) @@ -52,7 +52,7 @@ : : +- CometBroadcastExchange (51) : : +- CometSortMergeJoin (50) : : :- CometSort (44) - : : : +- CometColumnarExchange (43) + : : : +- CometExchange (43) : : : +- CometFilter (42) : : : +- CometScan parquet spark_catalog.default.customer (41) : : +- CometSort (49) @@ -65,7 +65,7 @@ : +- CometFilter (55) : +- CometScan parquet spark_catalog.default.date_dim (54) +- CometHashAggregate (84) - +- CometColumnarExchange (83) + +- CometExchange (83) +- CometHashAggregate (82) +- CometProject (81) +- CometBroadcastHashJoin (80) @@ -73,7 +73,7 @@ : +- CometBroadcastHashJoin (77) : :- CometSortMergeJoin (75) : : :- CometSort (69) - : : : +- CometColumnarExchange (68) + : : : +- CometExchange (68) : : : +- CometProject (67) : : : +- CometBroadcastHashJoin (66) : : : :- CometFilter (64) @@ -169,9 +169,9 @@ Input [3]: [d_date#11, i_item_sk#13, _groupingexpression#15] Keys [3]: [_groupingexpression#15, i_item_sk#13, d_date#11] Functions [1]: [partial_count(1)] -(17) CometColumnarExchange +(17) CometExchange Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] -Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(_groupingexpression#15, i_item_sk#13, d_date#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [4]: [_groupingexpression#15, i_item_sk#13, d_date#11, count#16] @@ -199,9 +199,9 @@ Arguments: [cs_item_sk#2], [item_sk#17], LeftSemi, BuildRight Input [5]: [cs_bill_customer_sk#1, cs_item_sk#2, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] Arguments: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5], [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -(24) CometColumnarExchange +(24) CometExchange Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] -Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_bill_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (25) CometSort Input [4]: [cs_bill_customer_sk#1, cs_quantity#3, cs_list_price#4, cs_sold_date_sk#5] @@ -251,9 +251,9 @@ Input [3]: [ss_quantity#20, ss_sales_price#21, c_customer_sk#23] Keys [1]: [c_customer_sk#23] Functions [1]: [partial_sum((cast(ss_quantity#20 as decimal(10,0)) * ss_sales_price#21))] -(35) CometColumnarExchange +(35) CometExchange Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] -Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (36) CometHashAggregate Input [3]: [c_customer_sk#23, sum#24, isEmpty#25] @@ -288,9 +288,9 @@ ReadSchema: struct Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] Condition : isnotnull(c_customer_sk#29) -(43) CometColumnarExchange +(43) CometExchange Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] -Arguments: hashpartitioning(c_customer_sk#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_sk#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (44) CometSort Input [3]: [c_customer_sk#29, c_first_name#30, c_last_name#31] @@ -367,9 +367,9 @@ Input [4]: [cs_quantity#3, cs_list_price#4, c_first_name#30, c_last_name#31] Keys [2]: [c_last_name#31, c_first_name#30] Functions [1]: [partial_sum((cast(cs_quantity#3 as decimal(10,0)) * cs_list_price#4))] -(61) CometColumnarExchange +(61) CometExchange Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] -Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(c_last_name#31, c_first_name#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (62) CometHashAggregate Input [4]: [c_last_name#31, c_first_name#30, sum#35, isEmpty#36] @@ -400,9 +400,9 @@ Arguments: [ws_item_sk#37], [item_sk#17], LeftSemi, BuildRight Input [5]: [ws_item_sk#37, ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] Arguments: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41], [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] -(68) CometColumnarExchange +(68) CometExchange Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] -Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ws_bill_customer_sk#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (69) CometSort Input [4]: [ws_bill_customer_sk#38, ws_quantity#39, ws_list_price#40, ws_sold_date_sk#41] @@ -462,9 +462,9 @@ Input [4]: [ws_quantity#39, ws_list_price#40, c_first_name#44, c_last_name#45] Keys [2]: [c_last_name#45, c_first_name#44] Functions [1]: [partial_sum((cast(ws_quantity#39 as decimal(10,0)) * ws_list_price#40))] -(83) CometColumnarExchange +(83) CometExchange Input [4]: [c_last_name#45, c_first_name#44, sum#47, isEmpty#48] -Arguments: hashpartitioning(c_last_name#45, c_first_name#44, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(c_last_name#45, c_first_name#44, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (84) CometHashAggregate Input [4]: [c_last_name#45, c_first_name#44, sum#47, isEmpty#48] @@ -547,10 +547,10 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:3 Hosting operator id = 37 Hosting Expression = Subquery scalar-subquery#27, [id=#28] * ColumnarToRow (115) +- CometHashAggregate (114) - +- CometColumnarExchange (113) + +- CometExchange (113) +- CometHashAggregate (112) +- CometHashAggregate (111) - +- CometColumnarExchange (110) + +- CometExchange (110) +- CometHashAggregate (109) +- CometProject (108) +- CometBroadcastHashJoin (107) @@ -622,9 +622,9 @@ Input [3]: [ss_quantity#52, ss_sales_price#53, c_customer_sk#56] Keys [1]: [c_customer_sk#56] Functions [1]: [partial_sum((cast(ss_quantity#52 as decimal(10,0)) * ss_sales_price#53))] -(110) CometColumnarExchange +(110) CometExchange Input [3]: [c_customer_sk#56, sum#59, isEmpty#60] -Arguments: hashpartitioning(c_customer_sk#56, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(c_customer_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (111) CometHashAggregate Input [3]: [c_customer_sk#56, sum#59, isEmpty#60] @@ -636,9 +636,9 @@ Input [1]: [csales#61] Keys: [] Functions [1]: [partial_max(csales#61)] -(113) CometColumnarExchange +(113) CometExchange Input [1]: [max#62] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (114) CometHashAggregate Input [1]: [max#62] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt index 55f17ca34..d503bb68a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q23b/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [c_last_name,c_first_name,sales] CometUnion [c_last_name,c_first_name,sales] CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price))] - CometColumnarExchange [c_last_name,c_first_name] #1 + CometExchange [c_last_name,c_first_name] #1 CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,cs_quantity,cs_list_price] CometProject [cs_quantity,cs_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,c_first_name,c_last_name,d_date_sk] @@ -12,7 +12,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk,c_customer_sk] CometSort [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometColumnarExchange [cs_bill_customer_sk] #2 + CometExchange [cs_bill_customer_sk] #2 CometProject [cs_bill_customer_sk,cs_quantity,cs_list_price,cs_sold_date_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,item_sk] CometFilter [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] @@ -29,7 +29,7 @@ WholeStageCodegen (1) CometProject [item_sk] CometFilter [item_sk,cnt] CometHashAggregate [item_sk,cnt,_groupingexpression,i_item_sk,d_date,count,count(1)] - CometColumnarExchange [_groupingexpression,i_item_sk,d_date] #5 + CometExchange [_groupingexpression,i_item_sk,d_date] #5 CometHashAggregate [_groupingexpression,i_item_sk,d_date,count] CometProject [i_item_desc] [d_date,i_item_sk,_groupingexpression] CometBroadcastHashJoin [ss_item_sk,d_date,i_item_sk,i_item_desc] @@ -60,10 +60,10 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [tpcds_cmax,max,max(csales)] - CometColumnarExchange #11 + CometExchange #11 CometHashAggregate [max,csales] CometHashAggregate [csales,c_customer_sk,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometColumnarExchange [c_customer_sk] #12 + CometExchange [c_customer_sk] #12 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_quantity,ss_sales_price,ss_sold_date_sk,c_customer_sk,d_date_sk] @@ -85,7 +85,7 @@ WholeStageCodegen (1) CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometHashAggregate [c_customer_sk,ssales,sum,isEmpty,sum((cast(ss_quantity as decimal(10,0)) * ss_sales_price))] - CometColumnarExchange [c_customer_sk] #9 + CometExchange [c_customer_sk] #9 CometHashAggregate [c_customer_sk,sum,isEmpty,ss_quantity,ss_sales_price] CometProject [ss_quantity,ss_sales_price,c_customer_sk] CometBroadcastHashJoin [ss_customer_sk,ss_quantity,ss_sales_price,c_customer_sk] @@ -98,7 +98,7 @@ WholeStageCodegen (1) CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #15 CometSortMergeJoin [c_customer_sk,c_first_name,c_last_name,c_customer_sk] CometSort [c_customer_sk,c_first_name,c_last_name] - CometColumnarExchange [c_customer_sk] #16 + CometExchange [c_customer_sk] #16 CometFilter [c_customer_sk,c_first_name,c_last_name] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometSort [c_customer_sk] @@ -112,7 +112,7 @@ WholeStageCodegen (1) CometFilter [d_date_sk,d_year,d_moy] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] CometHashAggregate [c_last_name,c_first_name,sales,sum,isEmpty,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price))] - CometColumnarExchange [c_last_name,c_first_name] #18 + CometExchange [c_last_name,c_first_name] #18 CometHashAggregate [c_last_name,c_first_name,sum,isEmpty,ws_quantity,ws_list_price] CometProject [ws_quantity,ws_list_price,c_first_name,c_last_name] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,c_first_name,c_last_name,d_date_sk] @@ -120,7 +120,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk,c_first_name,c_last_name] CometSortMergeJoin [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,c_customer_sk] CometSort [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometColumnarExchange [ws_bill_customer_sk] #19 + CometExchange [ws_bill_customer_sk] #19 CometProject [ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk,item_sk] CometFilter [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_list_price,ws_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt index 549a3ae5a..d643cf7a3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/explain.txt @@ -1,50 +1,46 @@ == Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometColumnarExchange (38) - +- RowToColumnar (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) +* Filter (42) ++- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (21) + : : +- CometFilter (20) + : : +- CometScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.customer (24) + +- BroadcastExchange (33) + +- * ColumnarToRow (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.customer_address (30) (1) Scan parquet spark_catalog.default.store_sales @@ -62,9 +58,9 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(4) CometColumnarExchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -85,9 +81,9 @@ Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -216,216 +212,188 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#28] Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -(37) RowToColumnar +(37) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(38) CometColumnarExchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(39) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(40) HashAggregate [codegen id : 3] +(38) HashAggregate [codegen id : 3] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [partial_sum(netpaid#31)] Aggregate Attributes [2]: [sum#32, isEmpty#33] Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -(42) RowToColumnar -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(43) CometColumnarExchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [sum(netpaid#31)] Aggregate Attributes [1]: [sum(netpaid#31)#36] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (77) -+- * ColumnarToRow (76) - +- CometColumnarExchange (75) - +- RowToColumnar (74) - +- * HashAggregate (73) - +- * HashAggregate (72) - +- * ColumnarToRow (71) - +- CometColumnarExchange (70) - +- RowToColumnar (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * ColumnarToRow (64) - : +- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometProject (52) - : : : : +- CometSortMergeJoin (51) - : : : : :- CometSort (48) - : : : : : +- ReusedExchange (47) - : : : : +- CometSort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- CometBroadcastExchange (58) - : : +- CometFilter (57) - : : +- CometScan parquet spark_catalog.default.item (56) - : +- ReusedExchange (61) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 4] +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] +* HashAggregate (69) ++- Exchange (68) + +- * HashAggregate (67) + +- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * BroadcastHashJoin Inner BuildRight (62) + :- * ColumnarToRow (60) + : +- CometProject (59) + : +- CometBroadcastHashJoin (58) + : :- CometProject (56) + : : +- CometBroadcastHashJoin (55) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometProject (48) + : : : : +- CometSortMergeJoin (47) + : : : : :- CometSort (44) + : : : : : +- ReusedExchange (43) + : : : : +- CometSort (46) + : : : : +- ReusedExchange (45) + : : : +- ReusedExchange (49) + : : +- CometBroadcastExchange (54) + : : +- CometFilter (53) + : : +- CometScan parquet spark_catalog.default.item (52) + : +- ReusedExchange (57) + +- ReusedExchange (61) + + +(43) ReusedExchange [Reuses operator id: 4] Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(48) CometSort +(44) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(49) ReusedExchange [Reuses operator id: 9] +(45) ReusedExchange [Reuses operator id: 9] Output [2]: [sr_item_sk#7, sr_ticket_number#8] -(50) CometSort +(46) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] -(51) CometSortMergeJoin +(47) CometSortMergeJoin Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Right output [2]: [sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner -(52) CometProject +(48) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(53) ReusedExchange [Reuses operator id: 16] +(49) ReusedExchange [Reuses operator id: 16] Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(55) CometProject +(51) CometProject Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -(56) Scan parquet spark_catalog.default.item +(52) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) CometFilter +(53) CometFilter Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Condition : isnotnull(i_item_sk#15) -(58) CometBroadcastExchange +(54) CometBroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -(59) CometBroadcastHashJoin +(55) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight -(60) CometProject +(56) CometProject Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -(61) ReusedExchange [Reuses operator id: 26] +(57) ReusedExchange [Reuses operator id: 26] Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -(62) CometBroadcastHashJoin +(58) CometBroadcastHashJoin Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight -(63) CometProject +(59) CometProject Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -(64) ColumnarToRow [codegen id : 2] +(60) ColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -(65) ReusedExchange [Reuses operator id: 33] +(61) ReusedExchange [Reuses operator id: 33] Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -(66) BroadcastHashJoin [codegen id : 2] +(62) BroadcastHashJoin [codegen id : 2] Left keys [2]: [c_birth_country#24, s_zip#14] Right keys [2]: [upper(ca_country#27), ca_zip#26] Join type: Inner Join condition: None -(67) Project [codegen id : 2] +(63) Project [codegen id : 2] Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] -(68) HashAggregate [codegen id : 2] +(64) HashAggregate [codegen id : 2] Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#40] Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -(69) RowToColumnar +(65) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(70) CometColumnarExchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(71) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] - -(72) HashAggregate [codegen id : 3] +(66) HashAggregate [codegen id : 3] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] -(73) HashAggregate [codegen id : 3] +(67) HashAggregate [codegen id : 3] Input [1]: [netpaid#31] Keys: [] Functions [1]: [partial_avg(netpaid#31)] Aggregate Attributes [2]: [sum#42, count#43] Results [2]: [sum#44, count#45] -(74) RowToColumnar -Input [2]: [sum#44, count#45] - -(75) CometColumnarExchange -Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(76) ColumnarToRow [codegen id : 4] +(68) Exchange Input [2]: [sum#44, count#45] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(77) HashAggregate [codegen id : 4] +(69) HashAggregate [codegen id : 4] Input [2]: [sum#44, count#45] Keys: [] Functions [1]: [avg(netpaid#31)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt index 0b53b6fc9..bd14d8250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24a/simplified.txt @@ -3,92 +3,84 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #9 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 - RowToColumnar + InputAdapter + Exchange #9 WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [netpaid] [sum,count,sum,count] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (1) + ColumnarToRow InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt index 3e9d66921..2bc198436 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/explain.txt @@ -1,50 +1,46 @@ == Physical Plan == -* Filter (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometColumnarExchange (38) - +- RowToColumnar (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) +* Filter (42) ++- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (21) + : : +- CometFilter (20) + : : +- CometScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.customer (24) + +- BroadcastExchange (33) + +- * ColumnarToRow (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.customer_address (30) (1) Scan parquet spark_catalog.default.store_sales @@ -62,9 +58,9 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(4) CometColumnarExchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -85,9 +81,9 @@ Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -216,216 +212,188 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#28] Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -(37) RowToColumnar +(37) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(38) CometColumnarExchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(39) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] - -(40) HashAggregate [codegen id : 3] +(38) HashAggregate [codegen id : 3] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#29] Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, netpaid#31] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [partial_sum(netpaid#31)] Aggregate Attributes [2]: [sum#32, isEmpty#33] Results [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -(42) RowToColumnar -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] - -(43) CometColumnarExchange -Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [c_last_name#23, c_first_name#22, s_store_name#11, sum#34, isEmpty#35] Keys [3]: [c_last_name#23, c_first_name#22, s_store_name#11] Functions [1]: [sum(netpaid#31)] Aggregate Attributes [1]: [sum(netpaid#31)#36] Results [4]: [c_last_name#23, c_first_name#22, s_store_name#11, sum(netpaid#31)#36 AS paid#37] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [4]: [c_last_name#23, c_first_name#22, s_store_name#11, paid#37] Condition : (isnotnull(paid#37) AND (cast(paid#37 as decimal(33,8)) > cast(Subquery scalar-subquery#38, [id=#39] as decimal(33,8)))) ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#38, [id=#39] -* HashAggregate (77) -+- * ColumnarToRow (76) - +- CometColumnarExchange (75) - +- RowToColumnar (74) - +- * HashAggregate (73) - +- * HashAggregate (72) - +- * ColumnarToRow (71) - +- CometColumnarExchange (70) - +- RowToColumnar (69) - +- * HashAggregate (68) - +- * Project (67) - +- * BroadcastHashJoin Inner BuildRight (66) - :- * ColumnarToRow (64) - : +- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometProject (52) - : : : : +- CometSortMergeJoin (51) - : : : : :- CometSort (48) - : : : : : +- ReusedExchange (47) - : : : : +- CometSort (50) - : : : : +- ReusedExchange (49) - : : : +- ReusedExchange (53) - : : +- CometBroadcastExchange (58) - : : +- CometFilter (57) - : : +- CometScan parquet spark_catalog.default.item (56) - : +- ReusedExchange (61) - +- ReusedExchange (65) - - -(47) ReusedExchange [Reuses operator id: 4] +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#38, [id=#39] +* HashAggregate (69) ++- Exchange (68) + +- * HashAggregate (67) + +- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * BroadcastHashJoin Inner BuildRight (62) + :- * ColumnarToRow (60) + : +- CometProject (59) + : +- CometBroadcastHashJoin (58) + : :- CometProject (56) + : : +- CometBroadcastHashJoin (55) + : : :- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometProject (48) + : : : : +- CometSortMergeJoin (47) + : : : : :- CometSort (44) + : : : : : +- ReusedExchange (43) + : : : : +- CometSort (46) + : : : : +- ReusedExchange (45) + : : : +- ReusedExchange (49) + : : +- CometBroadcastExchange (54) + : : +- CometFilter (53) + : : +- CometScan parquet spark_catalog.default.item (52) + : +- ReusedExchange (57) + +- ReusedExchange (61) + + +(43) ReusedExchange [Reuses operator id: 4] Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(48) CometSort +(44) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(49) ReusedExchange [Reuses operator id: 9] +(45) ReusedExchange [Reuses operator id: 9] Output [2]: [sr_item_sk#7, sr_ticket_number#8] -(50) CometSort +(46) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] -(51) CometSortMergeJoin +(47) CometSortMergeJoin Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Right output [2]: [sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner -(52) CometProject +(48) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(53) ReusedExchange [Reuses operator id: 16] +(49) ReusedExchange [Reuses operator id: 16] Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(55) CometProject +(51) CometProject Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -(56) Scan parquet spark_catalog.default.item +(52) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(57) CometFilter +(53) CometFilter Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Condition : isnotnull(i_item_sk#15) -(58) CometBroadcastExchange +(54) CometBroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -(59) CometBroadcastHashJoin +(55) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight -(60) CometProject +(56) CometProject Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -(61) ReusedExchange [Reuses operator id: 26] +(57) ReusedExchange [Reuses operator id: 26] Output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] -(62) CometBroadcastHashJoin +(58) CometBroadcastHashJoin Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Right output [4]: [c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight -(63) CometProject +(59) CometProject Input [14]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_first_name#22, c_last_name#23, c_birth_country#24] Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -(64) ColumnarToRow [codegen id : 2] +(60) ColumnarToRow [codegen id : 2] Input [12]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24] -(65) ReusedExchange [Reuses operator id: 33] +(61) ReusedExchange [Reuses operator id: 33] Output [3]: [ca_state#25, ca_zip#26, ca_country#27] -(66) BroadcastHashJoin [codegen id : 2] +(62) BroadcastHashJoin [codegen id : 2] Left keys [2]: [c_birth_country#24, s_zip#14] Right keys [2]: [upper(ca_country#27), ca_zip#26] Join type: Inner Join condition: None -(67) Project [codegen id : 2] +(63) Project [codegen id : 2] Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] Input [15]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, c_birth_country#24, ca_state#25, ca_zip#26, ca_country#27] -(68) HashAggregate [codegen id : 2] +(64) HashAggregate [codegen id : 2] Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#22, c_last_name#23, ca_state#25] Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#40] Results [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -(69) RowToColumnar +(65) Exchange Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] +Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(70) CometColumnarExchange -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] -Arguments: hashpartitioning(c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(71) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] - -(72) HashAggregate [codegen id : 3] +(66) HashAggregate [codegen id : 3] Input [11]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#41] Keys [10]: [c_last_name#23, c_first_name#22, s_store_name#11, ca_state#25, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#30] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#30,17,2) AS netpaid#31] -(73) HashAggregate [codegen id : 3] +(67) HashAggregate [codegen id : 3] Input [1]: [netpaid#31] Keys: [] Functions [1]: [partial_avg(netpaid#31)] Aggregate Attributes [2]: [sum#42, count#43] Results [2]: [sum#44, count#45] -(74) RowToColumnar -Input [2]: [sum#44, count#45] - -(75) CometColumnarExchange -Input [2]: [sum#44, count#45] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(76) ColumnarToRow [codegen id : 4] +(68) Exchange Input [2]: [sum#44, count#45] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(77) HashAggregate [codegen id : 4] +(69) HashAggregate [codegen id : 4] Input [2]: [sum#44, count#45] Keys: [] Functions [1]: [avg(netpaid#31)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt index 0b53b6fc9..bd14d8250 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q24b/simplified.txt @@ -3,92 +3,84 @@ WholeStageCodegen (4) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #9 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #4 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - InputAdapter - ReusedExchange [ca_state,ca_zip,ca_country] #8 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name] #1 - RowToColumnar + InputAdapter + Exchange #9 WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [netpaid] [sum,count,sum,count] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 - CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #10 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #3 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #4 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #11 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + InputAdapter + ReusedExchange [ca_state,ca_zip,ca_country] #8 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name] #1 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #2 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_birth_country,s_zip,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #5 + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #6 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name,c_birth_country] #7 + CometFilter [c_customer_sk,c_first_name,c_last_name,c_birth_country] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (1) + ColumnarToRow InputAdapter - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] + CometFilter [ca_state,ca_zip,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt index f12717c3f..0d47bdcf1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (42) +- CometTakeOrderedAndProject (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -226,9 +226,9 @@ Input [7]: [ss_net_profit#5, sr_net_loss#11, cs_net_profit#16, s_store_id#27, s_ Keys [4]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28] Functions [3]: [partial_sum(UnscaledValue(ss_net_profit#5)), partial_sum(UnscaledValue(sr_net_loss#11)), partial_sum(UnscaledValue(cs_net_profit#16))] -(39) CometColumnarExchange +(39) CometExchange Input [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, sum#32, sum#33, sum#34] -Arguments: hashpartitioning(i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate Input [7]: [i_item_id#30, i_item_desc#31, s_store_id#27, s_store_name#28, sum#32, sum#33, sum#34] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt index c2c652e2b..13cb05053 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q25/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit] CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_profit,store_returns_loss,catalog_sales_profit,sum,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(sr_net_loss)),sum(UnscaledValue(cs_net_profit))] - CometColumnarExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum,ss_net_profit,sr_net_loss,cs_net_profit] CometProject [ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_net_profit,sr_net_loss,cs_net_profit,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt index cb4f53a0f..5ba7d13f3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (30) +- CometTakeOrderedAndProject (29) +- CometHashAggregate (28) - +- CometColumnarExchange (27) + +- CometExchange (27) +- CometHashAggregate (26) +- CometProject (25) +- CometBroadcastHashJoin (24) @@ -156,9 +156,9 @@ Input [5]: [cs_quantity#4, cs_list_price#5, cs_sales_price#6, cs_coupon_amt#7, i Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(cs_quantity#4), partial_avg(UnscaledValue(cs_list_price#5)), partial_avg(UnscaledValue(cs_coupon_amt#7)), partial_avg(UnscaledValue(cs_sales_price#6))] -(27) CometColumnarExchange +(27) CometExchange Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] -Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt index 08745014d..f643ff7b0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q26/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] CometHashAggregate [i_item_id,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(cs_quantity),avg(UnscaledValue(cs_list_price)),avg(UnscaledValue(cs_coupon_amt)),avg(UnscaledValue(cs_sales_price))] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price] CometProject [cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id] CometBroadcastHashJoin [cs_promo_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,i_item_id,p_promo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt index 129139b03..150b31fd3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (30) +- CometTakeOrderedAndProject (29) +- CometHashAggregate (28) - +- CometColumnarExchange (27) + +- CometExchange (27) +- CometHashAggregate (26) +- CometExpand (25) +- CometProject (24) @@ -156,9 +156,9 @@ Input [7]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i Keys [3]: [i_item_id#20, s_state#21, spark_grouping_id#22] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -(27) CometColumnarExchange +(27) CometExchange Input [11]: [i_item_id#20, s_state#21, spark_grouping_id#22, sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] -Arguments: hashpartitioning(i_item_id#20, s_state#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#20, s_state#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate Input [11]: [i_item_id#20, s_state#21, spark_grouping_id#22, sum#23, count#24, sum#25, count#26, sum#27, count#28, sum#29, count#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt index 0c2fe68c6..121f1be7c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q27/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_item_id,s_state,spark_grouping_id] #1 + CometExchange [i_item_id,s_state,spark_grouping_id] #1 CometHashAggregate [i_item_id,s_state,spark_grouping_id,sum,count,sum,count,sum,count,sum,count,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] CometExpand [i_item_id,s_state] [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state,spark_grouping_id] CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,s_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt index 8881b95aa..29af7273c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/explain.txt @@ -1,86 +1,74 @@ == Physical Plan == -* BroadcastNestedLoopJoin Inner BuildRight (82) -:- * BroadcastNestedLoopJoin Inner BuildRight (68) -: :- * BroadcastNestedLoopJoin Inner BuildRight (54) -: : :- * BroadcastNestedLoopJoin Inner BuildRight (40) -: : : :- * BroadcastNestedLoopJoin Inner BuildRight (26) -: : : : :- * ColumnarToRow (12) -: : : : : +- CometHashAggregate (11) -: : : : : +- CometColumnarExchange (10) -: : : : : +- RowToColumnar (9) -: : : : : +- * HashAggregate (8) -: : : : : +- * HashAggregate (7) -: : : : : +- * ColumnarToRow (6) -: : : : : +- CometColumnarExchange (5) -: : : : : +- CometHashAggregate (4) -: : : : : +- CometProject (3) -: : : : : +- CometFilter (2) -: : : : : +- CometScan parquet spark_catalog.default.store_sales (1) -: : : : +- BroadcastExchange (25) -: : : : +- * ColumnarToRow (24) -: : : : +- CometHashAggregate (23) -: : : : +- CometColumnarExchange (22) -: : : : +- RowToColumnar (21) -: : : : +- * HashAggregate (20) -: : : : +- * HashAggregate (19) -: : : : +- * ColumnarToRow (18) -: : : : +- CometColumnarExchange (17) -: : : : +- CometHashAggregate (16) -: : : : +- CometProject (15) -: : : : +- CometFilter (14) -: : : : +- CometScan parquet spark_catalog.default.store_sales (13) -: : : +- BroadcastExchange (39) -: : : +- * ColumnarToRow (38) -: : : +- CometHashAggregate (37) -: : : +- CometColumnarExchange (36) -: : : +- RowToColumnar (35) -: : : +- * HashAggregate (34) -: : : +- * HashAggregate (33) -: : : +- * ColumnarToRow (32) -: : : +- CometColumnarExchange (31) -: : : +- CometHashAggregate (30) -: : : +- CometProject (29) -: : : +- CometFilter (28) -: : : +- CometScan parquet spark_catalog.default.store_sales (27) -: : +- BroadcastExchange (53) -: : +- * ColumnarToRow (52) -: : +- CometHashAggregate (51) -: : +- CometColumnarExchange (50) -: : +- RowToColumnar (49) -: : +- * HashAggregate (48) -: : +- * HashAggregate (47) -: : +- * ColumnarToRow (46) -: : +- CometColumnarExchange (45) -: : +- CometHashAggregate (44) -: : +- CometProject (43) -: : +- CometFilter (42) -: : +- CometScan parquet spark_catalog.default.store_sales (41) -: +- BroadcastExchange (67) -: +- * ColumnarToRow (66) -: +- CometHashAggregate (65) -: +- CometColumnarExchange (64) -: +- RowToColumnar (63) -: +- * HashAggregate (62) -: +- * HashAggregate (61) -: +- * ColumnarToRow (60) -: +- CometColumnarExchange (59) -: +- CometHashAggregate (58) -: +- CometProject (57) -: +- CometFilter (56) -: +- CometScan parquet spark_catalog.default.store_sales (55) -+- BroadcastExchange (81) - +- * ColumnarToRow (80) - +- CometHashAggregate (79) - +- CometColumnarExchange (78) - +- RowToColumnar (77) - +- * HashAggregate (76) - +- * HashAggregate (75) - +- * ColumnarToRow (74) - +- CometColumnarExchange (73) - +- CometHashAggregate (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan parquet spark_catalog.default.store_sales (69) +* BroadcastNestedLoopJoin Inner BuildRight (70) +:- * BroadcastNestedLoopJoin Inner BuildRight (58) +: :- * BroadcastNestedLoopJoin Inner BuildRight (46) +: : :- * BroadcastNestedLoopJoin Inner BuildRight (34) +: : : :- * BroadcastNestedLoopJoin Inner BuildRight (22) +: : : : :- * HashAggregate (10) +: : : : : +- Exchange (9) +: : : : : +- * HashAggregate (8) +: : : : : +- * HashAggregate (7) +: : : : : +- * ColumnarToRow (6) +: : : : : +- CometExchange (5) +: : : : : +- CometHashAggregate (4) +: : : : : +- CometProject (3) +: : : : : +- CometFilter (2) +: : : : : +- CometScan parquet spark_catalog.default.store_sales (1) +: : : : +- BroadcastExchange (21) +: : : : +- * HashAggregate (20) +: : : : +- Exchange (19) +: : : : +- * HashAggregate (18) +: : : : +- * HashAggregate (17) +: : : : +- * ColumnarToRow (16) +: : : : +- CometExchange (15) +: : : : +- CometHashAggregate (14) +: : : : +- CometProject (13) +: : : : +- CometFilter (12) +: : : : +- CometScan parquet spark_catalog.default.store_sales (11) +: : : +- BroadcastExchange (33) +: : : +- * HashAggregate (32) +: : : +- Exchange (31) +: : : +- * HashAggregate (30) +: : : +- * HashAggregate (29) +: : : +- * ColumnarToRow (28) +: : : +- CometExchange (27) +: : : +- CometHashAggregate (26) +: : : +- CometProject (25) +: : : +- CometFilter (24) +: : : +- CometScan parquet spark_catalog.default.store_sales (23) +: : +- BroadcastExchange (45) +: : +- * HashAggregate (44) +: : +- Exchange (43) +: : +- * HashAggregate (42) +: : +- * HashAggregate (41) +: : +- * ColumnarToRow (40) +: : +- CometExchange (39) +: : +- CometHashAggregate (38) +: : +- CometProject (37) +: : +- CometFilter (36) +: : +- CometScan parquet spark_catalog.default.store_sales (35) +: +- BroadcastExchange (57) +: +- * HashAggregate (56) +: +- Exchange (55) +: +- * HashAggregate (54) +: +- * HashAggregate (53) +: +- * ColumnarToRow (52) +: +- CometExchange (51) +: +- CometHashAggregate (50) +: +- CometProject (49) +: +- CometFilter (48) +: +- CometScan parquet spark_catalog.default.store_sales (47) ++- BroadcastExchange (69) + +- * HashAggregate (68) + +- Exchange (67) + +- * HashAggregate (66) + +- * HashAggregate (65) + +- * ColumnarToRow (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometFilter (60) + +- CometScan parquet spark_catalog.default.store_sales (59) (1) Scan parquet spark_catalog.default.store_sales @@ -103,9 +91,9 @@ Input [1]: [ss_list_price#3] Keys [1]: [ss_list_price#3] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#3)), partial_count(ss_list_price#3)] -(5) CometColumnarExchange +(5) CometExchange Input [4]: [ss_list_price#3, sum#6, count#7, count#8] -Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_list_price#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (6) ColumnarToRow [codegen id : 1] Input [4]: [ss_list_price#3, sum#6, count#7, count#8] @@ -124,338 +112,314 @@ Functions [3]: [merge_avg(UnscaledValue(ss_list_price#3)), merge_count(ss_list_p Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] Results [4]: [sum#6, count#7, count#8, count#12] -(9) RowToColumnar +(9) Exchange Input [4]: [sum#6, count#7, count#8, count#12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(10) CometColumnarExchange -Input [4]: [sum#6, count#7, count#8, count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(11) CometHashAggregate +(10) HashAggregate [codegen id : 12] Input [4]: [sum#6, count#7, count#8, count#12] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#3)), count(ss_list_price#3), count(distinct ss_list_price#3)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#3))#9, count(ss_list_price#3)#10, count(ss_list_price#3)#11] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#3))#9 / 100.0) as decimal(11,6)) AS B1_LP#13, count(ss_list_price#3)#10 AS B1_CNT#14, count(ss_list_price#3)#11 AS B1_CNTD#15] -(12) ColumnarToRow [codegen id : 12] -Input [3]: [B1_LP#13, B1_CNT#14, B1_CNTD#15] - -(13) Scan parquet spark_catalog.default.store_sales +(11) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,6), LessThanOrEqual(ss_quantity,10), Or(Or(And(GreaterThanOrEqual(ss_list_price,90.00),LessThanOrEqual(ss_list_price,100.00)),And(GreaterThanOrEqual(ss_coupon_amt,2323.00),LessThanOrEqual(ss_coupon_amt,3323.00))),And(GreaterThanOrEqual(ss_wholesale_cost,31.00),LessThanOrEqual(ss_wholesale_cost,51.00)))] ReadSchema: struct -(14) CometFilter +(12) CometFilter Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Condition : (((isnotnull(ss_quantity#16) AND (ss_quantity#16 >= 6)) AND (ss_quantity#16 <= 10)) AND ((((ss_list_price#18 >= 90.00) AND (ss_list_price#18 <= 100.00)) OR ((ss_coupon_amt#19 >= 2323.00) AND (ss_coupon_amt#19 <= 3323.00))) OR ((ss_wholesale_cost#17 >= 31.00) AND (ss_wholesale_cost#17 <= 51.00)))) -(15) CometProject +(13) CometProject Input [5]: [ss_quantity#16, ss_wholesale_cost#17, ss_list_price#18, ss_coupon_amt#19, ss_sold_date_sk#20] Arguments: [ss_list_price#18], [ss_list_price#18] -(16) CometHashAggregate +(14) CometHashAggregate Input [1]: [ss_list_price#18] Keys [1]: [ss_list_price#18] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#18)), partial_count(ss_list_price#18)] -(17) CometColumnarExchange +(15) CometExchange Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_list_price#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] -(18) ColumnarToRow [codegen id : 2] +(16) ColumnarToRow [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] -(19) HashAggregate [codegen id : 2] +(17) HashAggregate [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys [1]: [ss_list_price#18] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25] Results [4]: [ss_list_price#18, sum#21, count#22, count#23] -(20) HashAggregate [codegen id : 2] +(18) HashAggregate [codegen id : 2] Input [4]: [ss_list_price#18, sum#21, count#22, count#23] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#18)), merge_count(ss_list_price#18), partial_count(distinct ss_list_price#18)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] Results [4]: [sum#21, count#22, count#23, count#27] -(21) RowToColumnar +(19) Exchange Input [4]: [sum#21, count#22, count#23, count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] -(22) CometColumnarExchange -Input [4]: [sum#21, count#22, count#23, count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(23) CometHashAggregate +(20) HashAggregate [codegen id : 3] Input [4]: [sum#21, count#22, count#23, count#27] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#18)), count(ss_list_price#18), count(distinct ss_list_price#18)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#18))#24, count(ss_list_price#18)#25, count(ss_list_price#18)#26] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#18))#24 / 100.0) as decimal(11,6)) AS B2_LP#28, count(ss_list_price#18)#25 AS B2_CNT#29, count(ss_list_price#18)#26 AS B2_CNTD#30] -(24) ColumnarToRow [codegen id : 3] -Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] - -(25) BroadcastExchange +(21) BroadcastExchange Input [3]: [B2_LP#28, B2_CNT#29, B2_CNTD#30] Arguments: IdentityBroadcastMode, [plan_id=5] -(26) BroadcastNestedLoopJoin [codegen id : 12] +(22) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(27) Scan parquet spark_catalog.default.store_sales +(23) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,11), LessThanOrEqual(ss_quantity,15), Or(Or(And(GreaterThanOrEqual(ss_list_price,142.00),LessThanOrEqual(ss_list_price,152.00)),And(GreaterThanOrEqual(ss_coupon_amt,12214.00),LessThanOrEqual(ss_coupon_amt,13214.00))),And(GreaterThanOrEqual(ss_wholesale_cost,79.00),LessThanOrEqual(ss_wholesale_cost,99.00)))] ReadSchema: struct -(28) CometFilter +(24) CometFilter Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Condition : (((isnotnull(ss_quantity#31) AND (ss_quantity#31 >= 11)) AND (ss_quantity#31 <= 15)) AND ((((ss_list_price#33 >= 142.00) AND (ss_list_price#33 <= 152.00)) OR ((ss_coupon_amt#34 >= 12214.00) AND (ss_coupon_amt#34 <= 13214.00))) OR ((ss_wholesale_cost#32 >= 79.00) AND (ss_wholesale_cost#32 <= 99.00)))) -(29) CometProject +(25) CometProject Input [5]: [ss_quantity#31, ss_wholesale_cost#32, ss_list_price#33, ss_coupon_amt#34, ss_sold_date_sk#35] Arguments: [ss_list_price#33], [ss_list_price#33] -(30) CometHashAggregate +(26) CometHashAggregate Input [1]: [ss_list_price#33] Keys [1]: [ss_list_price#33] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#33)), partial_count(ss_list_price#33)] -(31) CometColumnarExchange +(27) CometExchange Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ss_list_price#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(32) ColumnarToRow [codegen id : 4] +(28) ColumnarToRow [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] -(33) HashAggregate [codegen id : 4] +(29) HashAggregate [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys [1]: [ss_list_price#33] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40] Results [4]: [ss_list_price#33, sum#36, count#37, count#38] -(34) HashAggregate [codegen id : 4] +(30) HashAggregate [codegen id : 4] Input [4]: [ss_list_price#33, sum#36, count#37, count#38] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#33)), merge_count(ss_list_price#33), partial_count(distinct ss_list_price#33)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] Results [4]: [sum#36, count#37, count#38, count#42] -(35) RowToColumnar -Input [4]: [sum#36, count#37, count#38, count#42] - -(36) CometColumnarExchange +(31) Exchange Input [4]: [sum#36, count#37, count#38, count#42] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(37) CometHashAggregate +(32) HashAggregate [codegen id : 5] Input [4]: [sum#36, count#37, count#38, count#42] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#33)), count(ss_list_price#33), count(distinct ss_list_price#33)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#33))#39, count(ss_list_price#33)#40, count(ss_list_price#33)#41] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#33))#39 / 100.0) as decimal(11,6)) AS B3_LP#43, count(ss_list_price#33)#40 AS B3_CNT#44, count(ss_list_price#33)#41 AS B3_CNTD#45] -(38) ColumnarToRow [codegen id : 5] -Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] - -(39) BroadcastExchange +(33) BroadcastExchange Input [3]: [B3_LP#43, B3_CNT#44, B3_CNTD#45] Arguments: IdentityBroadcastMode, [plan_id=8] -(40) BroadcastNestedLoopJoin [codegen id : 12] +(34) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(41) Scan parquet spark_catalog.default.store_sales +(35) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,16), LessThanOrEqual(ss_quantity,20), Or(Or(And(GreaterThanOrEqual(ss_list_price,135.00),LessThanOrEqual(ss_list_price,145.00)),And(GreaterThanOrEqual(ss_coupon_amt,6071.00),LessThanOrEqual(ss_coupon_amt,7071.00))),And(GreaterThanOrEqual(ss_wholesale_cost,38.00),LessThanOrEqual(ss_wholesale_cost,58.00)))] ReadSchema: struct -(42) CometFilter +(36) CometFilter Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Condition : (((isnotnull(ss_quantity#46) AND (ss_quantity#46 >= 16)) AND (ss_quantity#46 <= 20)) AND ((((ss_list_price#48 >= 135.00) AND (ss_list_price#48 <= 145.00)) OR ((ss_coupon_amt#49 >= 6071.00) AND (ss_coupon_amt#49 <= 7071.00))) OR ((ss_wholesale_cost#47 >= 38.00) AND (ss_wholesale_cost#47 <= 58.00)))) -(43) CometProject +(37) CometProject Input [5]: [ss_quantity#46, ss_wholesale_cost#47, ss_list_price#48, ss_coupon_amt#49, ss_sold_date_sk#50] Arguments: [ss_list_price#48], [ss_list_price#48] -(44) CometHashAggregate +(38) CometHashAggregate Input [1]: [ss_list_price#48] Keys [1]: [ss_list_price#48] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#48)), partial_count(ss_list_price#48)] -(45) CometColumnarExchange +(39) CometExchange Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: hashpartitioning(ss_list_price#48, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] -(46) ColumnarToRow [codegen id : 6] +(40) ColumnarToRow [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] -(47) HashAggregate [codegen id : 6] +(41) HashAggregate [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys [1]: [ss_list_price#48] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55] Results [4]: [ss_list_price#48, sum#51, count#52, count#53] -(48) HashAggregate [codegen id : 6] +(42) HashAggregate [codegen id : 6] Input [4]: [ss_list_price#48, sum#51, count#52, count#53] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#48)), merge_count(ss_list_price#48), partial_count(distinct ss_list_price#48)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] Results [4]: [sum#51, count#52, count#53, count#57] -(49) RowToColumnar +(43) Exchange Input [4]: [sum#51, count#52, count#53, count#57] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] -(50) CometColumnarExchange -Input [4]: [sum#51, count#52, count#53, count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] - -(51) CometHashAggregate +(44) HashAggregate [codegen id : 7] Input [4]: [sum#51, count#52, count#53, count#57] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#48)), count(ss_list_price#48), count(distinct ss_list_price#48)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#48))#54, count(ss_list_price#48)#55, count(ss_list_price#48)#56] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#48))#54 / 100.0) as decimal(11,6)) AS B4_LP#58, count(ss_list_price#48)#55 AS B4_CNT#59, count(ss_list_price#48)#56 AS B4_CNTD#60] -(52) ColumnarToRow [codegen id : 7] -Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] - -(53) BroadcastExchange +(45) BroadcastExchange Input [3]: [B4_LP#58, B4_CNT#59, B4_CNTD#60] Arguments: IdentityBroadcastMode, [plan_id=11] -(54) BroadcastNestedLoopJoin [codegen id : 12] +(46) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(55) Scan parquet spark_catalog.default.store_sales +(47) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,21), LessThanOrEqual(ss_quantity,25), Or(Or(And(GreaterThanOrEqual(ss_list_price,122.00),LessThanOrEqual(ss_list_price,132.00)),And(GreaterThanOrEqual(ss_coupon_amt,836.00),LessThanOrEqual(ss_coupon_amt,1836.00))),And(GreaterThanOrEqual(ss_wholesale_cost,17.00),LessThanOrEqual(ss_wholesale_cost,37.00)))] ReadSchema: struct -(56) CometFilter +(48) CometFilter Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Condition : (((isnotnull(ss_quantity#61) AND (ss_quantity#61 >= 21)) AND (ss_quantity#61 <= 25)) AND ((((ss_list_price#63 >= 122.00) AND (ss_list_price#63 <= 132.00)) OR ((ss_coupon_amt#64 >= 836.00) AND (ss_coupon_amt#64 <= 1836.00))) OR ((ss_wholesale_cost#62 >= 17.00) AND (ss_wholesale_cost#62 <= 37.00)))) -(57) CometProject +(49) CometProject Input [5]: [ss_quantity#61, ss_wholesale_cost#62, ss_list_price#63, ss_coupon_amt#64, ss_sold_date_sk#65] Arguments: [ss_list_price#63], [ss_list_price#63] -(58) CometHashAggregate +(50) CometHashAggregate Input [1]: [ss_list_price#63] Keys [1]: [ss_list_price#63] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#63)), partial_count(ss_list_price#63)] -(59) CometColumnarExchange +(51) CometExchange Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: hashpartitioning(ss_list_price#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(60) ColumnarToRow [codegen id : 8] +(52) ColumnarToRow [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] -(61) HashAggregate [codegen id : 8] +(53) HashAggregate [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys [1]: [ss_list_price#63] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70] Results [4]: [ss_list_price#63, sum#66, count#67, count#68] -(62) HashAggregate [codegen id : 8] +(54) HashAggregate [codegen id : 8] Input [4]: [ss_list_price#63, sum#66, count#67, count#68] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#63)), merge_count(ss_list_price#63), partial_count(distinct ss_list_price#63)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] Results [4]: [sum#66, count#67, count#68, count#72] -(63) RowToColumnar +(55) Exchange Input [4]: [sum#66, count#67, count#68, count#72] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=13] -(64) CometColumnarExchange -Input [4]: [sum#66, count#67, count#68, count#72] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] - -(65) CometHashAggregate +(56) HashAggregate [codegen id : 9] Input [4]: [sum#66, count#67, count#68, count#72] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#63)), count(ss_list_price#63), count(distinct ss_list_price#63)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#63))#69, count(ss_list_price#63)#70, count(ss_list_price#63)#71] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#63))#69 / 100.0) as decimal(11,6)) AS B5_LP#73, count(ss_list_price#63)#70 AS B5_CNT#74, count(ss_list_price#63)#71 AS B5_CNTD#75] -(66) ColumnarToRow [codegen id : 9] -Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] - -(67) BroadcastExchange +(57) BroadcastExchange Input [3]: [B5_LP#73, B5_CNT#74, B5_CNTD#75] Arguments: IdentityBroadcastMode, [plan_id=14] -(68) BroadcastNestedLoopJoin [codegen id : 12] +(58) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None -(69) Scan parquet spark_catalog.default.store_sales +(59) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Batched: true Location [not included in comparison]/{warehouse_dir}/store_sales] PushedFilters: [IsNotNull(ss_quantity), GreaterThanOrEqual(ss_quantity,26), LessThanOrEqual(ss_quantity,30), Or(Or(And(GreaterThanOrEqual(ss_list_price,154.00),LessThanOrEqual(ss_list_price,164.00)),And(GreaterThanOrEqual(ss_coupon_amt,7326.00),LessThanOrEqual(ss_coupon_amt,8326.00))),And(GreaterThanOrEqual(ss_wholesale_cost,7.00),LessThanOrEqual(ss_wholesale_cost,27.00)))] ReadSchema: struct -(70) CometFilter +(60) CometFilter Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Condition : (((isnotnull(ss_quantity#76) AND (ss_quantity#76 >= 26)) AND (ss_quantity#76 <= 30)) AND ((((ss_list_price#78 >= 154.00) AND (ss_list_price#78 <= 164.00)) OR ((ss_coupon_amt#79 >= 7326.00) AND (ss_coupon_amt#79 <= 8326.00))) OR ((ss_wholesale_cost#77 >= 7.00) AND (ss_wholesale_cost#77 <= 27.00)))) -(71) CometProject +(61) CometProject Input [5]: [ss_quantity#76, ss_wholesale_cost#77, ss_list_price#78, ss_coupon_amt#79, ss_sold_date_sk#80] Arguments: [ss_list_price#78], [ss_list_price#78] -(72) CometHashAggregate +(62) CometHashAggregate Input [1]: [ss_list_price#78] Keys [1]: [ss_list_price#78] Functions [2]: [partial_avg(UnscaledValue(ss_list_price#78)), partial_count(ss_list_price#78)] -(73) CometColumnarExchange +(63) CometExchange Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] +Arguments: hashpartitioning(ss_list_price#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(74) ColumnarToRow [codegen id : 10] +(64) ColumnarToRow [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] -(75) HashAggregate [codegen id : 10] +(65) HashAggregate [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys [1]: [ss_list_price#78] Functions [2]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78)] Aggregate Attributes [2]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85] Results [4]: [ss_list_price#78, sum#81, count#82, count#83] -(76) HashAggregate [codegen id : 10] +(66) HashAggregate [codegen id : 10] Input [4]: [ss_list_price#78, sum#81, count#82, count#83] Keys: [] Functions [3]: [merge_avg(UnscaledValue(ss_list_price#78)), merge_count(ss_list_price#78), partial_count(distinct ss_list_price#78)] Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] Results [4]: [sum#81, count#82, count#83, count#87] -(77) RowToColumnar -Input [4]: [sum#81, count#82, count#83, count#87] - -(78) CometColumnarExchange +(67) Exchange Input [4]: [sum#81, count#82, count#83, count#87] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=16] -(79) CometHashAggregate +(68) HashAggregate [codegen id : 11] Input [4]: [sum#81, count#82, count#83, count#87] Keys: [] Functions [3]: [avg(UnscaledValue(ss_list_price#78)), count(ss_list_price#78), count(distinct ss_list_price#78)] +Aggregate Attributes [3]: [avg(UnscaledValue(ss_list_price#78))#84, count(ss_list_price#78)#85, count(ss_list_price#78)#86] +Results [3]: [cast((avg(UnscaledValue(ss_list_price#78))#84 / 100.0) as decimal(11,6)) AS B6_LP#88, count(ss_list_price#78)#85 AS B6_CNT#89, count(ss_list_price#78)#86 AS B6_CNTD#90] -(80) ColumnarToRow [codegen id : 11] -Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] - -(81) BroadcastExchange +(69) BroadcastExchange Input [3]: [B6_LP#88, B6_CNT#89, B6_CNTD#90] Arguments: IdentityBroadcastMode, [plan_id=17] -(82) BroadcastNestedLoopJoin [codegen id : 12] +(70) BroadcastNestedLoopJoin [codegen id : 12] Join type: Inner Join condition: None diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/simplified.txt index 73343f39e..4a547c4e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28/simplified.txt @@ -4,108 +4,96 @@ WholeStageCodegen (12) BroadcastNestedLoopJoin BroadcastNestedLoopJoin BroadcastNestedLoopJoin - ColumnarToRow + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count] InputAdapter - CometHashAggregate [B1_LP,B1_CNT,B1_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #2 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + Exchange #1 + WholeStageCodegen (1) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometExchange [ss_list_price] #2 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] InputAdapter BroadcastExchange #3 WholeStageCodegen (3) - ColumnarToRow + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count] InputAdapter - CometHashAggregate [B2_LP,B2_CNT,B2_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #4 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #5 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometHashAggregate [B3_LP,B3_CNT,B3_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #7 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] - HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_list_price] #8 - CometHashAggregate [ss_list_price,sum,count,count] - CometProject [ss_list_price] - CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometHashAggregate [B4_LP,B4_CNT,B4_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #10 - RowToColumnar - WholeStageCodegen (6) + Exchange #4 + WholeStageCodegen (2) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] ColumnarToRow InputAdapter - CometColumnarExchange [ss_list_price] #11 + CometExchange [ss_list_price] #5 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (9) - ColumnarToRow - InputAdapter - CometHashAggregate [B5_LP,B5_CNT,B5_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #13 - RowToColumnar - WholeStageCodegen (8) + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (5) + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count] + InputAdapter + Exchange #7 + WholeStageCodegen (4) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] ColumnarToRow InputAdapter - CometColumnarExchange [ss_list_price] #14 + CometExchange [ss_list_price] #8 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - InputAdapter - BroadcastExchange #15 - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometHashAggregate [B6_LP,B6_CNT,B6_CNTD,sum,count,count,count,avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price)] - CometColumnarExchange #16 - RowToColumnar - WholeStageCodegen (10) + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (7) + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count] + InputAdapter + Exchange #10 + WholeStageCodegen (6) HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] ColumnarToRow InputAdapter - CometColumnarExchange [ss_list_price] #17 + CometExchange [ss_list_price] #11 CometHashAggregate [ss_list_price,sum,count,count] CometProject [ss_list_price] CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #12 + WholeStageCodegen (9) + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count] + InputAdapter + Exchange #13 + WholeStageCodegen (8) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometExchange [ss_list_price] #14 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + InputAdapter + BroadcastExchange #15 + WholeStageCodegen (11) + HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count] + InputAdapter + Exchange #16 + WholeStageCodegen (10) + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count] + HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count] + ColumnarToRow + InputAdapter + CometExchange [ss_list_price] #17 + CometHashAggregate [ss_list_price,sum,count,count] + CometProject [ss_list_price] + CometFilter [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt index a3346549e..6248ddee4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (45) +- CometTakeOrderedAndProject (44) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometProject (40) +- CometBroadcastHashJoin (39) @@ -245,9 +245,9 @@ Input [7]: [ss_quantity#5, sr_return_quantity#11, cs_quantity#16, s_store_id#28, Keys [4]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29] Functions [3]: [partial_sum(ss_quantity#5), partial_sum(sr_return_quantity#11), partial_sum(cs_quantity#16)] -(42) CometColumnarExchange +(42) CometExchange Input [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, sum#33, sum#34, sum#35] -Arguments: hashpartitioning(i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (43) CometHashAggregate Input [7]: [i_item_id#31, i_item_desc#32, s_store_id#28, s_store_name#29, sum#33, sum#34, sum#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt index 3ca13e7b4..9398fcdf5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q29/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity] CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,store_sales_quantity,store_returns_quantity,catalog_sales_quantity,sum,sum,sum,sum(ss_quantity),sum(sr_return_quantity),sum(cs_quantity)] - CometColumnarExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 + CometExchange [i_item_id,i_item_desc,s_store_id,s_store_name] #1 CometHashAggregate [i_item_id,i_item_desc,s_store_id,s_store_name,sum,sum,sum,ss_quantity,sr_return_quantity,cs_quantity] CometProject [ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_id,i_item_desc] CometBroadcastHashJoin [ss_item_sk,ss_quantity,sr_return_quantity,cs_quantity,s_store_id,s_store_name,i_item_sk,i_item_id,i_item_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt index 9079e0886..9a63fffa1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -93,9 +93,9 @@ Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt index 65c80baed..f82fd24bd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q3/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [d_year,brand_id,brand,sum_agg] CometHashAggregate [d_year,brand_id,brand,sum_agg,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [d_year,i_brand,i_brand_id] #1 + CometExchange [d_year,i_brand,i_brand_id] #1 CometHashAggregate [d_year,i_brand,i_brand_id,sum,ss_ext_sales_price] CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt index f16b49407..695999eb8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/explain.txt @@ -9,7 +9,7 @@ : : +- CometBroadcastHashJoin (34) : : :- CometFilter (17) : : : +- CometHashAggregate (16) - : : : +- CometColumnarExchange (15) + : : : +- CometExchange (15) : : : +- CometHashAggregate (14) : : : +- CometProject (13) : : : +- CometBroadcastHashJoin (12) @@ -27,10 +27,10 @@ : : +- CometBroadcastExchange (33) : : +- CometFilter (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometHashAggregate (28) - : : +- CometColumnarExchange (27) + : : +- CometExchange (27) : : +- CometHashAggregate (26) : : +- CometProject (25) : : +- CometBroadcastHashJoin (24) @@ -118,9 +118,9 @@ Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#10] @@ -172,9 +172,9 @@ Input [3]: [wr_returning_customer_sk#1, wr_return_amt#3, ca_state#9] Keys [2]: [wr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(wr_return_amt#3))] -(27) CometColumnarExchange +(27) CometExchange Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#15] -Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(wr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [3]: [wr_returning_customer_sk#1, ca_state#9, sum#15] @@ -186,9 +186,9 @@ Input [2]: [ctr_state#12, ctr_total_return#13] Keys [1]: [ctr_state#12] Functions [1]: [partial_avg(ctr_total_return#13)] -(30) CometColumnarExchange +(30) CometExchange Input [3]: [ctr_state#12, sum#16, count#17] -Arguments: hashpartitioning(ctr_state#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ctr_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (31) CometHashAggregate Input [3]: [ctr_state#12, sum#16, count#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt index f1bb3a7ae..872bf5191 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q30/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] CometHashAggregate [ctr_customer_sk,ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(wr_return_amt))] - CometColumnarExchange [wr_returning_customer_sk,ca_state] #1 + CometExchange [wr_returning_customer_sk,ca_state] #1 CometHashAggregate [wr_returning_customer_sk,ca_state,sum,wr_return_amt] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] @@ -36,10 +36,10 @@ WholeStageCodegen (1) CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #5 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_state,ctr_state,sum,count,avg(ctr_total_return)] - CometColumnarExchange [ctr_state] #6 + CometExchange [ctr_state] #6 CometHashAggregate [ctr_state,sum,count,ctr_total_return] CometHashAggregate [ctr_state,ctr_total_return,wr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(wr_return_amt))] - CometColumnarExchange [wr_returning_customer_sk,ca_state] #7 + CometExchange [wr_returning_customer_sk,ca_state] #7 CometHashAggregate [wr_returning_customer_sk,ca_state,sum,wr_return_amt] CometProject [wr_returning_customer_sk,wr_return_amt,ca_state] CometBroadcastHashJoin [wr_returning_customer_sk,wr_returning_addr_sk,wr_return_amt,ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt index 7284a48b6..e5bc4e7ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/explain.txt @@ -11,7 +11,7 @@ : : : +- CometBroadcastHashJoin (45) : : : :- CometBroadcastHashJoin (30) : : : : :- CometHashAggregate (15) - : : : : : +- CometColumnarExchange (14) + : : : : : +- CometExchange (14) : : : : : +- CometHashAggregate (13) : : : : : +- CometProject (12) : : : : : +- CometBroadcastHashJoin (11) @@ -27,7 +27,7 @@ : : : : : +- CometScan parquet spark_catalog.default.customer_address (8) : : : : +- CometBroadcastExchange (29) : : : : +- CometHashAggregate (28) - : : : : +- CometColumnarExchange (27) + : : : : +- CometExchange (27) : : : : +- CometHashAggregate (26) : : : : +- CometProject (25) : : : : +- CometBroadcastHashJoin (24) @@ -41,7 +41,7 @@ : : : : +- ReusedExchange (23) : : : +- CometBroadcastExchange (44) : : : +- CometHashAggregate (43) - : : : +- CometColumnarExchange (42) + : : : +- CometExchange (42) : : : +- CometHashAggregate (41) : : : +- CometProject (40) : : : +- CometBroadcastHashJoin (39) @@ -55,7 +55,7 @@ : : : +- ReusedExchange (38) : : +- CometBroadcastExchange (58) : : +- CometHashAggregate (57) - : : +- CometColumnarExchange (56) + : : +- CometExchange (56) : : +- CometHashAggregate (55) : : +- CometProject (54) : : +- CometBroadcastHashJoin (53) @@ -67,7 +67,7 @@ : : +- ReusedExchange (52) : +- CometBroadcastExchange (71) : +- CometHashAggregate (70) - : +- CometColumnarExchange (69) + : +- CometExchange (69) : +- CometHashAggregate (68) : +- CometProject (67) : +- CometBroadcastHashJoin (66) @@ -79,7 +79,7 @@ : +- ReusedExchange (65) +- CometBroadcastExchange (85) +- CometHashAggregate (84) - +- CometColumnarExchange (83) + +- CometExchange (83) +- CometHashAggregate (82) +- CometProject (81) +- CometBroadcastHashJoin (80) @@ -156,9 +156,9 @@ Input [4]: [ss_ext_sales_price#2, d_year#6, d_qoy#7, ca_county#9] Keys [3]: [ca_county#9, d_qoy#7, d_year#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(14) CometColumnarExchange +(14) CometExchange Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] -Arguments: hashpartitioning(ca_county#9, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ca_county#9, d_qoy#7, d_year#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [4]: [ca_county#9, d_qoy#7, d_year#6, sum#10] @@ -218,9 +218,9 @@ Input [4]: [ss_ext_sales_price#12, d_year#16, d_qoy#17, ca_county#19] Keys [3]: [ca_county#19, d_qoy#17, d_year#16] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#12))] -(27) CometColumnarExchange +(27) CometExchange Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] -Arguments: hashpartitioning(ca_county#19, d_qoy#17, d_year#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ca_county#19, d_qoy#17, d_year#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [4]: [ca_county#19, d_qoy#17, d_year#16, sum#20] @@ -289,9 +289,9 @@ Input [4]: [ss_ext_sales_price#24, d_year#28, d_qoy#29, ca_county#31] Keys [3]: [ca_county#31, d_qoy#29, d_year#28] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#24))] -(42) CometColumnarExchange +(42) CometExchange Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#32] -Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ca_county#31, d_qoy#29, d_year#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (43) CometHashAggregate Input [4]: [ca_county#31, d_qoy#29, d_year#28, sum#32] @@ -352,9 +352,9 @@ Input [4]: [ws_ext_sales_price#35, d_year#39, d_qoy#40, ca_county#42] Keys [3]: [ca_county#42, d_qoy#40, d_year#39] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#35))] -(56) CometColumnarExchange +(56) CometExchange Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#43] -Arguments: hashpartitioning(ca_county#42, d_qoy#40, d_year#39, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ca_county#42, d_qoy#40, d_year#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (57) CometHashAggregate Input [4]: [ca_county#42, d_qoy#40, d_year#39, sum#43] @@ -411,9 +411,9 @@ Input [4]: [ws_ext_sales_price#46, d_year#50, d_qoy#51, ca_county#53] Keys [3]: [ca_county#53, d_qoy#51, d_year#50] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#46))] -(69) CometColumnarExchange +(69) CometExchange Input [4]: [ca_county#53, d_qoy#51, d_year#50, sum#54] -Arguments: hashpartitioning(ca_county#53, d_qoy#51, d_year#50, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ca_county#53, d_qoy#51, d_year#50, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (70) CometHashAggregate Input [4]: [ca_county#53, d_qoy#51, d_year#50, sum#54] @@ -474,9 +474,9 @@ Input [4]: [ws_ext_sales_price#57, d_year#61, d_qoy#62, ca_county#64] Keys [3]: [ca_county#64, d_qoy#62, d_year#61] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#57))] -(83) CometColumnarExchange +(83) CometExchange Input [4]: [ca_county#64, d_qoy#62, d_year#61, sum#65] -Arguments: hashpartitioning(ca_county#64, d_qoy#62, d_year#61, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ca_county#64, d_qoy#62, d_year#61, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (84) CometHashAggregate Input [4]: [ca_county#64, d_qoy#62, d_year#61, sum#65] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt index 6dcf5714d..d6615fe1c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q31/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales,ca_county,store_sales] CometBroadcastHashJoin [ca_county,d_year,store_sales,ca_county,store_sales] CometHashAggregate [ca_county,d_year,store_sales,d_qoy,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #2 + CometExchange [ca_county,d_qoy,d_year] #2 CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -35,7 +35,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county] CometBroadcastExchange [ca_county,store_sales] #6 CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #7 + CometExchange [ca_county,d_qoy,d_year] #7 CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -56,7 +56,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,store_sales] #10 CometHashAggregate [ca_county,store_sales,d_qoy,d_year,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #11 + CometExchange [ca_county,d_qoy,d_year] #11 CometHashAggregate [ca_county,d_qoy,d_year,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ss_addr_sk,ss_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -77,7 +77,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,web_sales] #14 CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #15 + CometExchange [ca_county,d_qoy,d_year] #15 CometHashAggregate [ca_county,d_qoy,d_year,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -90,7 +90,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,web_sales] #16 CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #17 + CometExchange [ca_county,d_qoy,d_year] #17 CometHashAggregate [ca_county,d_qoy,d_year,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] @@ -103,7 +103,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk,ca_county] #5 CometBroadcastExchange [ca_county,web_sales] #18 CometHashAggregate [ca_county,web_sales,d_qoy,d_year,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [ca_county,d_qoy,d_year] #19 + CometExchange [ca_county,d_qoy,d_year] #19 CometHashAggregate [ca_county,d_qoy,d_year,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,d_year,d_qoy,ca_county] CometBroadcastHashJoin [ws_bill_addr_sk,ws_ext_sales_price,d_year,d_qoy,ca_address_sk,ca_county] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt index 701a75e24..a0b604678 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (30) +- CometHashAggregate (29) - +- CometColumnarExchange (28) + +- CometExchange (28) +- CometHashAggregate (27) +- CometProject (26) +- CometBroadcastHashJoin (25) @@ -18,7 +18,7 @@ : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) - : +- CometColumnarExchange (18) + : +- CometExchange (18) : +- CometHashAggregate (17) : +- CometProject (16) : +- CometBroadcastHashJoin (15) @@ -116,9 +116,9 @@ Input [2]: [cs_item_sk#7, cs_ext_discount_amt#8] Keys [1]: [cs_item_sk#7] Functions [1]: [partial_avg(UnscaledValue(cs_ext_discount_amt#8))] -(18) CometColumnarExchange +(18) CometExchange Input [3]: [cs_item_sk#7, sum#13, count#14] -Arguments: hashpartitioning(cs_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (19) CometHashAggregate Input [3]: [cs_item_sk#7, sum#13, count#14] @@ -159,9 +159,9 @@ Input [1]: [cs_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(cs_ext_discount_amt#2))] -(28) CometColumnarExchange +(28) CometExchange Input [1]: [sum#17] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate Input [1]: [sum#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt index 19be8718e..14b2a34a8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q32/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [excess discount amount,sum,sum(UnscaledValue(cs_ext_discount_amt))] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,cs_ext_discount_amt] CometProject [cs_ext_discount_amt] CometBroadcastHashJoin [cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] @@ -27,7 +27,7 @@ WholeStageCodegen (1) CometBroadcastExchange [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] #4 CometFilter [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk] CometHashAggregate [(1.3 * avg(cs_ext_discount_amt)),cs_item_sk,sum,count,avg(UnscaledValue(cs_ext_discount_amt))] - CometColumnarExchange [cs_item_sk] #5 + CometExchange [cs_item_sk] #5 CometHashAggregate [cs_item_sk,sum,count,cs_ext_discount_amt] CometProject [cs_item_sk,cs_ext_discount_amt] CometBroadcastHashJoin [cs_item_sk,cs_ext_discount_amt,cs_sold_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt index 6fb8b05fb..a432ecd29 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/explain.txt @@ -2,11 +2,11 @@ * ColumnarToRow (61) +- CometTakeOrderedAndProject (60) +- CometHashAggregate (59) - +- CometColumnarExchange (58) + +- CometExchange (58) +- CometHashAggregate (57) +- CometUnion (56) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -33,7 +33,7 @@ : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.item (17) :- CometHashAggregate (41) - : +- CometColumnarExchange (40) + : +- CometExchange (40) : +- CometHashAggregate (39) : +- CometProject (38) : +- CometBroadcastHashJoin (37) @@ -47,7 +47,7 @@ : : +- ReusedExchange (33) : +- ReusedExchange (36) +- CometHashAggregate (55) - +- CometColumnarExchange (54) + +- CometExchange (54) +- CometHashAggregate (53) +- CometProject (52) +- CometBroadcastHashJoin (51) @@ -183,9 +183,9 @@ Input [2]: [ss_ext_sales_price#3, i_manufact_id#12] Keys [1]: [i_manufact_id#12] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(26) CometColumnarExchange +(26) CometExchange Input [2]: [i_manufact_id#12, sum#15] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [2]: [i_manufact_id#12, sum#15] @@ -245,9 +245,9 @@ Input [2]: [cs_ext_sales_price#18, i_manufact_id#24] Keys [1]: [i_manufact_id#24] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] -(40) CometColumnarExchange +(40) CometExchange Input [2]: [i_manufact_id#24, sum#25] -Arguments: hashpartitioning(i_manufact_id#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_manufact_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (41) CometHashAggregate Input [2]: [i_manufact_id#24, sum#25] @@ -307,9 +307,9 @@ Input [2]: [ws_ext_sales_price#28, i_manufact_id#34] Keys [1]: [i_manufact_id#34] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] -(54) CometColumnarExchange +(54) CometExchange Input [2]: [i_manufact_id#34, sum#35] -Arguments: hashpartitioning(i_manufact_id#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_manufact_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (55) CometHashAggregate Input [2]: [i_manufact_id#34, sum#35] @@ -326,9 +326,9 @@ Input [2]: [i_manufact_id#12, total_sales#36] Keys [1]: [i_manufact_id#12] Functions [1]: [partial_sum(total_sales#36)] -(58) CometColumnarExchange +(58) CometExchange Input [3]: [i_manufact_id#12, sum#39, isEmpty#40] -Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (59) CometHashAggregate Input [3]: [i_manufact_id#12, sum#39, isEmpty#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt index 964f250ba..9556aa785 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q33/simplified.txt @@ -3,11 +3,11 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_manufact_id,total_sales] CometHashAggregate [i_manufact_id,total_sales,sum,isEmpty,sum(total_sales)] - CometColumnarExchange [i_manufact_id] #1 + CometExchange [i_manufact_id] #1 CometHashAggregate [i_manufact_id,sum,isEmpty,total_sales] CometUnion [i_manufact_id,total_sales] CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_manufact_id] #2 + CometExchange [i_manufact_id] #2 CometHashAggregate [i_manufact_id,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_manufact_id] @@ -42,7 +42,7 @@ WholeStageCodegen (1) CometFilter [i_category,i_manufact_id] CometScan parquet spark_catalog.default.item [i_category,i_manufact_id] CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_manufact_id] #8 + CometExchange [i_manufact_id] #8 CometHashAggregate [i_manufact_id,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_manufact_id] @@ -57,7 +57,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk] #5 ReusedExchange [i_item_sk,i_manufact_id] #6 CometHashAggregate [i_manufact_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_manufact_id] #9 + CometExchange [i_manufact_id] #9 CometHashAggregate [i_manufact_id,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_manufact_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt index c33735fa4..dccad5750 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/explain.txt @@ -6,7 +6,7 @@ +- CometBroadcastHashJoin (28) :- CometFilter (24) : +- CometHashAggregate (23) - : +- CometColumnarExchange (22) + : +- CometExchange (22) : +- CometHashAggregate (21) : +- CometProject (20) : +- CometBroadcastHashJoin (19) @@ -134,9 +134,9 @@ Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt index c131bd9c0..779d4cbdd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q34/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometColumnarExchange [ss_ticket_number,ss_customer_sk] #2 + CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt index b35338d18..c06c1dd16 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/explain.txt @@ -1,50 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- Exchange (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * ColumnarToRow (31) + : +- CometFilter (30) + : +- CometScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometFilter (36) + +- CometScan parquet spark_catalog.default.customer_demographics (35) (1) Scan parquet spark_catalog.default.customer @@ -240,56 +238,50 @@ Functions [10]: [partial_count(1), partial_min(cd_dep_count#25), partial_max(cd_ Aggregate Attributes [13]: [count#28, min#29, max#30, sum#31, count#32, min#33, max#34, sum#35, count#36, min#37, max#38, sum#39, count#40] Results [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] -(42) RowToColumnar +(42) Exchange Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] +Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) CometColumnarExchange -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] -Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 6] -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] - -(45) HashAggregate [codegen id : 6] +(43) HashAggregate [codegen id : 6] Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, min#42, max#43, sum#44, count#45, min#46, max#47, sum#48, count#49, min#50, max#51, sum#52, count#53] Keys [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [10]: [count(1), min(cd_dep_count#25), max(cd_dep_count#25), avg(cd_dep_count#25), min(cd_dep_employed_count#26), max(cd_dep_employed_count#26), avg(cd_dep_employed_count#26), min(cd_dep_college_count#27), max(cd_dep_college_count#27), avg(cd_dep_college_count#27)] Aggregate Attributes [10]: [count(1)#54, min(cd_dep_count#25)#55, max(cd_dep_count#25)#56, avg(cd_dep_count#25)#57, min(cd_dep_employed_count#26)#58, max(cd_dep_employed_count#26)#59, avg(cd_dep_employed_count#26)#60, min(cd_dep_college_count#27)#61, max(cd_dep_college_count#27)#62, avg(cd_dep_college_count#27)#63] Results [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, count(1)#54 AS cnt1#64, min(cd_dep_count#25)#55 AS min(cd_dep_count)#65, max(cd_dep_count#25)#56 AS max(cd_dep_count)#66, avg(cd_dep_count#25)#57 AS avg(cd_dep_count)#67, cd_dep_employed_count#26, count(1)#54 AS cnt2#68, min(cd_dep_employed_count#26)#58 AS min(cd_dep_employed_count)#69, max(cd_dep_employed_count#26)#59 AS max(cd_dep_employed_count)#70, avg(cd_dep_employed_count#26)#60 AS avg(cd_dep_employed_count)#71, cd_dep_college_count#27, count(1)#54 AS cnt3#72, min(cd_dep_college_count#27)#61 AS min(cd_dep_college_count)#73, max(cd_dep_college_count#27)#62 AS max(cd_dep_college_count)#74, avg(cd_dep_college_count#27)#63 AS avg(cd_dep_college_count)#75, cd_dep_count#25] -(46) TakeOrderedAndProject +(44) TakeOrderedAndProject Input [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75, cd_dep_count#25] Arguments: 100, [ca_state#21 ASC NULLS FIRST, cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [ca_state#21, cd_gender#23, cd_marital_status#24, cnt1#64, min(cd_dep_count)#65, max(cd_dep_count)#66, avg(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, min(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, avg(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, min(cd_dep_college_count)#73, max(cd_dep_college_count)#74, avg(cd_dep_college_count)#75] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (51) -+- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.date_dim (47) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan parquet spark_catalog.default.date_dim (45) -(47) Scan parquet spark_catalog.default.date_dim +(45) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(48) CometFilter +(46) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) -(49) CometProject +(47) CometProject Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Arguments: [d_date_sk#9], [d_date_sk#9] -(50) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(51) BroadcastExchange +(49) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt index f9ef660d5..9deb51342 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q35/simplified.txt @@ -1,72 +1,70 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] [count(1),min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),cnt1,min(cd_dep_count),max(cd_dep_count),avg(cd_dep_count),cnt2,min(cd_dep_employed_count),max(cd_dep_employed_count),avg(cd_dep_employed_count),cnt3,min(cd_dep_college_count),max(cd_dep_college_count),avg(cd_dep_college_count),count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + InputAdapter + Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,min,max,sum,count,min,max,sum,count,min,max,sum,count,count,min,max,sum,count,min,max,sum,count,min,max,sum,count] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt index 9f22e37d6..68397f3a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (29) +- Window (27) +- * ColumnarToRow (26) +- CometSort (25) - +- CometColumnarExchange (24) + +- CometExchange (24) +- CometHashAggregate (23) - +- CometColumnarExchange (22) + +- CometExchange (22) +- CometHashAggregate (21) +- CometExpand (20) +- CometProject (19) @@ -131,18 +131,18 @@ Input [5]: [ss_ext_sales_price#3, ss_net_profit#4, i_category#14, i_class#15, sp Keys [3]: [i_category#14, i_class#15, spark_grouping_id#16] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(22) CometColumnarExchange +(22) CometExchange Input [5]: [i_category#14, i_class#15, spark_grouping_id#16, sum#17, sum#18] -Arguments: hashpartitioning(i_category#14, i_class#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#14, i_class#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [5]: [i_category#14, i_class#15, spark_grouping_id#16, sum#17, sum#18] Keys [3]: [i_category#14, i_class#15, spark_grouping_id#16] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] -(24) CometColumnarExchange +(24) CometExchange Input [7]: [gross_margin#19, i_category#14, i_class#15, lochierarchy#20, _w0#21, _w1#22, _w2#23] -Arguments: hashpartitioning(_w1#22, _w2#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(_w1#22, _w2#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (25) CometSort Input [7]: [gross_margin#19, i_category#14, i_class#15, lochierarchy#20, _w0#21, _w1#22, _w2#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt index d7beb4422..421db8dad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q36/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i ColumnarToRow InputAdapter CometSort [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2] - CometColumnarExchange [_w1,_w2] #1 + CometExchange [_w1,_w2] #1 CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,_w1,_w2,spark_grouping_id,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_category,i_class,spark_grouping_id] #2 + CometExchange [i_category,i_class,spark_grouping_id] #2 CometHashAggregate [i_category,i_class,spark_grouping_id,sum,sum,ss_net_profit,ss_ext_sales_price] CometExpand [i_category,i_class] [ss_ext_sales_price,ss_net_profit,i_category,i_class,spark_grouping_id] CometProject [ss_ext_sales_price,ss_net_profit,i_category,i_class] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt index 42dab0b38..bd5ca5ef6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometBroadcastHashJoin (20) @@ -132,9 +132,9 @@ Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Functions: [] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt index 39edfb519..85e86ca8a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q37/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometColumnarExchange [i_item_id,i_item_desc,i_current_price] #1 + CometExchange [i_item_id,i_item_desc,i_current_price] #1 CometHashAggregate [i_item_id,i_item_desc,i_current_price] CometProject [i_item_id,i_item_desc,i_current_price] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,cs_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt index 091883bd6..598dccaf0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == * ColumnarToRow (47) +- CometHashAggregate (46) - +- CometColumnarExchange (45) + +- CometExchange (45) +- CometHashAggregate (44) +- CometProject (43) +- CometBroadcastHashJoin (42) :- CometBroadcastHashJoin (29) : :- CometHashAggregate (16) - : : +- CometColumnarExchange (15) + : : +- CometExchange (15) : : +- CometHashAggregate (14) : : +- CometProject (13) : : +- CometBroadcastHashJoin (12) @@ -24,7 +24,7 @@ : : +- CometScan parquet spark_catalog.default.customer (9) : +- CometBroadcastExchange (28) : +- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -36,7 +36,7 @@ : +- ReusedExchange (22) +- CometBroadcastExchange (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -117,9 +117,9 @@ Input [3]: [c_last_name#9, c_first_name#8, d_date#5] Keys [3]: [c_last_name#9, c_first_name#8, d_date#5] Functions: [] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [c_last_name#9, c_first_name#8, d_date#5] -Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [c_last_name#9, c_first_name#8, d_date#5] @@ -167,9 +167,9 @@ Input [3]: [c_last_name#17, c_first_name#16, d_date#14] Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(26) CometColumnarExchange +(26) CometExchange Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate Input [3]: [c_last_name#17, c_first_name#16, d_date#14] @@ -226,9 +226,9 @@ Input [3]: [c_last_name#25, c_first_name#24, d_date#22] Keys [3]: [c_last_name#25, c_first_name#24, d_date#22] Functions: [] -(39) CometColumnarExchange +(39) CometExchange Input [3]: [c_last_name#25, c_first_name#24, d_date#22] -Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (40) CometHashAggregate Input [3]: [c_last_name#25, c_first_name#24, d_date#22] @@ -252,9 +252,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(45) CometColumnarExchange +(45) CometExchange Input [1]: [count#26] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (46) CometHashAggregate Input [1]: [count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt index 77970e8d3..8272260f5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q38/simplified.txt @@ -2,13 +2,13 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [count(1),count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] CometBroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #2 + CometExchange [c_last_name,c_first_name,d_date] #2 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -33,7 +33,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] CometBroadcastExchange [c_last_name,c_first_name,d_date] #6 CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #7 + CometExchange [c_last_name,c_first_name,d_date] #7 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] @@ -46,7 +46,7 @@ WholeStageCodegen (1) ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 CometBroadcastExchange [c_last_name,c_first_name,d_date] #8 CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #9 + CometExchange [c_last_name,c_first_name,d_date] #9 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt index 60c02c542..58e0e6649 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/explain.txt @@ -6,7 +6,7 @@ :- CometProject (23) : +- CometFilter (22) : +- CometHashAggregate (21) - : +- CometColumnarExchange (20) + : +- CometExchange (20) : +- CometHashAggregate (19) : +- CometProject (18) : +- CometBroadcastHashJoin (17) @@ -30,7 +30,7 @@ +- CometProject (42) +- CometFilter (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -141,9 +141,9 @@ Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_n Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -(20) CometColumnarExchange +(20) CometExchange Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] @@ -227,9 +227,9 @@ Input [5]: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehous Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#22 as double)), partial_avg(inv_quantity_on_hand#22)] -(39) CometColumnarExchange +(39) CometExchange Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) CometHashAggregate Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt index a4d1753f9..b789f3a8d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39a/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] @@ -39,7 +39,7 @@ WholeStageCodegen (1) CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt index d95f2c4cc..c246e93a5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/explain.txt @@ -6,7 +6,7 @@ :- CometProject (23) : +- CometFilter (22) : +- CometHashAggregate (21) - : +- CometColumnarExchange (20) + : +- CometExchange (20) : +- CometHashAggregate (19) : +- CometProject (18) : +- CometBroadcastHashJoin (17) @@ -30,7 +30,7 @@ +- CometProject (42) +- CometFilter (41) +- CometHashAggregate (40) - +- CometColumnarExchange (39) + +- CometExchange (39) +- CometHashAggregate (38) +- CometProject (37) +- CometBroadcastHashJoin (36) @@ -141,9 +141,9 @@ Input [5]: [inv_quantity_on_hand#3, i_item_sk#6, w_warehouse_sk#7, w_warehouse_n Keys [4]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#3 as double)), partial_avg(inv_quantity_on_hand#3)] -(20) CometColumnarExchange +(20) CometExchange Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] -Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [9]: [w_warehouse_name#8, w_warehouse_sk#7, i_item_sk#6, d_moy#11, n#12, avg#13, m2#14, sum#15, count#16] @@ -227,9 +227,9 @@ Input [5]: [inv_quantity_on_hand#22, i_item_sk#25, w_warehouse_sk#26, w_warehous Keys [4]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30] Functions [2]: [partial_stddev_samp(cast(inv_quantity_on_hand#22 as double)), partial_avg(inv_quantity_on_hand#22)] -(39) CometColumnarExchange +(39) CometExchange Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] -Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) CometHashAggregate Input [9]: [w_warehouse_name#27, w_warehouse_sk#26, i_item_sk#25, d_moy#30, n#31, avg#32, m2#33, sum#34, count#35] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt index a4d1753f9..b789f3a8d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q39b/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometProject [stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #2 CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] @@ -39,7 +39,7 @@ WholeStageCodegen (1) CometProject [mean,stdev] [w_warehouse_sk,i_item_sk,d_moy,mean,cov] CometFilter [w_warehouse_sk,i_item_sk,d_moy,stdev,mean] CometHashAggregate [w_warehouse_sk,i_item_sk,d_moy,stdev,mean,w_warehouse_name,n,avg,m2,sum,count,stddev_samp(cast(inv_quantity_on_hand as double)),avg(inv_quantity_on_hand)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 + CometExchange [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy] #8 CometHashAggregate [w_warehouse_name,w_warehouse_sk,i_item_sk,d_moy,n,avg,m2,sum,count,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,i_item_sk,w_warehouse_sk,w_warehouse_name,d_moy] CometBroadcastHashJoin [inv_quantity_on_hand,inv_date_sk,i_item_sk,w_warehouse_sk,w_warehouse_name,d_date_sk,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt index 960cb130c..b6387767d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/explain.txt @@ -12,7 +12,7 @@ : : : :- CometBroadcastHashJoin (33) : : : : :- CometFilter (16) : : : : : +- CometHashAggregate (15) - : : : : : +- CometColumnarExchange (14) + : : : : : +- CometExchange (14) : : : : : +- CometHashAggregate (13) : : : : : +- CometProject (12) : : : : : +- CometBroadcastHashJoin (11) @@ -28,7 +28,7 @@ : : : : : +- CometScan parquet spark_catalog.default.date_dim (8) : : : : +- CometBroadcastExchange (32) : : : : +- CometHashAggregate (31) - : : : : +- CometColumnarExchange (30) + : : : : +- CometExchange (30) : : : : +- CometHashAggregate (29) : : : : +- CometProject (28) : : : : +- CometBroadcastHashJoin (27) @@ -45,7 +45,7 @@ : : : +- CometBroadcastExchange (48) : : : +- CometFilter (47) : : : +- CometHashAggregate (46) - : : : +- CometColumnarExchange (45) + : : : +- CometExchange (45) : : : +- CometHashAggregate (44) : : : +- CometProject (43) : : : +- CometBroadcastHashJoin (42) @@ -59,7 +59,7 @@ : : : +- ReusedExchange (41) : : +- CometBroadcastExchange (64) : : +- CometHashAggregate (63) - : : +- CometColumnarExchange (62) + : : +- CometExchange (62) : : +- CometHashAggregate (61) : : +- CometProject (60) : : +- CometBroadcastHashJoin (59) @@ -74,7 +74,7 @@ : +- CometBroadcastExchange (81) : +- CometFilter (80) : +- CometHashAggregate (79) - : +- CometColumnarExchange (78) + : +- CometExchange (78) : +- CometHashAggregate (77) : +- CometProject (76) : +- CometBroadcastHashJoin (75) @@ -88,7 +88,7 @@ : +- ReusedExchange (74) +- CometBroadcastExchange (97) +- CometHashAggregate (96) - +- CometColumnarExchange (95) + +- CometExchange (95) +- CometHashAggregate (94) +- CometProject (93) +- CometBroadcastHashJoin (92) @@ -167,9 +167,9 @@ Input [12]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_fl Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17] Functions [1]: [partial_sum(((((ss_ext_list_price#13 - ss_ext_wholesale_cost#12) - ss_ext_discount_amt#10) + ss_ext_sales_price#11) / 2))] -(14) CometColumnarExchange +(14) CometExchange Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, sum#18, isEmpty#19] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, d_year#17, sum#18, isEmpty#19] @@ -245,9 +245,9 @@ Input [12]: [c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust Keys [8]: [c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26, c_birth_country#27, c_login#28, c_email_address#29, d_year#38] Functions [1]: [partial_sum(((((ss_ext_list_price#34 - ss_ext_wholesale_cost#33) - ss_ext_discount_amt#31) + ss_ext_sales_price#32) / 2))] -(30) CometColumnarExchange +(30) CometExchange Input [10]: [c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26, c_birth_country#27, c_login#28, c_email_address#29, d_year#38, sum#39, isEmpty#40] -Arguments: hashpartitioning(c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26, c_birth_country#27, c_login#28, c_email_address#29, d_year#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26, c_birth_country#27, c_login#28, c_email_address#29, d_year#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate Input [10]: [c_customer_id#23, c_first_name#24, c_last_name#25, c_preferred_cust_flag#26, c_birth_country#27, c_login#28, c_email_address#29, d_year#38, sum#39, isEmpty#40] @@ -316,9 +316,9 @@ Input [12]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust Keys [8]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#65] Functions [1]: [partial_sum(((((cs_ext_list_price#61 - cs_ext_wholesale_cost#60) - cs_ext_discount_amt#58) + cs_ext_sales_price#59) / 2))] -(45) CometColumnarExchange +(45) CometExchange Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#65, sum#66, isEmpty#67] -Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#65, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometHashAggregate Input [10]: [c_customer_id#50, c_first_name#51, c_last_name#52, c_preferred_cust_flag#53, c_birth_country#54, c_login#55, c_email_address#56, d_year#65, sum#66, isEmpty#67] @@ -395,9 +395,9 @@ Input [12]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust Keys [8]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#86] Functions [1]: [partial_sum(((((cs_ext_list_price#82 - cs_ext_wholesale_cost#81) - cs_ext_discount_amt#79) + cs_ext_sales_price#80) / 2))] -(62) CometColumnarExchange +(62) CometExchange Input [10]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#86, sum#87, isEmpty#88] -Arguments: hashpartitioning(c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#86, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#86, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (63) CometHashAggregate Input [10]: [c_customer_id#71, c_first_name#72, c_last_name#73, c_preferred_cust_flag#74, c_birth_country#75, c_login#76, c_email_address#77, d_year#86, sum#87, isEmpty#88] @@ -470,9 +470,9 @@ Input [12]: [c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust Keys [8]: [c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust_flag#95, c_birth_country#96, c_login#97, c_email_address#98, d_year#107] Functions [1]: [partial_sum(((((ws_ext_list_price#103 - ws_ext_wholesale_cost#102) - ws_ext_discount_amt#100) + ws_ext_sales_price#101) / 2))] -(78) CometColumnarExchange +(78) CometExchange Input [10]: [c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust_flag#95, c_birth_country#96, c_login#97, c_email_address#98, d_year#107, sum#108, isEmpty#109] -Arguments: hashpartitioning(c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust_flag#95, c_birth_country#96, c_login#97, c_email_address#98, d_year#107, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust_flag#95, c_birth_country#96, c_login#97, c_email_address#98, d_year#107, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (79) CometHashAggregate Input [10]: [c_customer_id#92, c_first_name#93, c_last_name#94, c_preferred_cust_flag#95, c_birth_country#96, c_login#97, c_email_address#98, d_year#107, sum#108, isEmpty#109] @@ -549,9 +549,9 @@ Input [12]: [c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_c Keys [8]: [c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_cust_flag#116, c_birth_country#117, c_login#118, c_email_address#119, d_year#128] Functions [1]: [partial_sum(((((ws_ext_list_price#124 - ws_ext_wholesale_cost#123) - ws_ext_discount_amt#121) + ws_ext_sales_price#122) / 2))] -(95) CometColumnarExchange +(95) CometExchange Input [10]: [c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_cust_flag#116, c_birth_country#117, c_login#118, c_email_address#119, d_year#128, sum#129, isEmpty#130] -Arguments: hashpartitioning(c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_cust_flag#116, c_birth_country#117, c_login#118, c_email_address#119, d_year#128, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_cust_flag#116, c_birth_country#117, c_login#118, c_email_address#119, d_year#128, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (96) CometHashAggregate Input [10]: [c_customer_id#113, c_first_name#114, c_last_name#115, c_preferred_cust_flag#116, c_birth_country#117, c_login#118, c_email_address#119, d_year#128, sum#129, isEmpty#130] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt index 7e6b12575..d6651fadc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q4/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #1 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -36,7 +36,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_preferred_cust_flag,customer_birth_country,customer_login,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ss_ext_list_price - ss_ext_wholesale_cost) - ss_ext_discount_amt) + ss_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #6 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ss_ext_list_price,ss_ext_wholesale_cost,ss_ext_discount_amt,ss_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_sales_price,ss_ext_wholesale_cost,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -60,7 +60,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] @@ -75,7 +75,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((cs_ext_list_price - cs_ext_wholesale_cost) - cs_ext_discount_amt) + cs_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,cs_ext_list_price,cs_ext_wholesale_cost,cs_ext_discount_amt,cs_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,cs_ext_discount_amt,cs_ext_sales_price,cs_ext_wholesale_cost,cs_ext_list_price,cs_sold_date_sk,d_date_sk,d_year] @@ -91,7 +91,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #16 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #17 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #17 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -106,7 +106,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #19 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,sum(((((ws_ext_list_price - ws_ext_wholesale_cost) - ws_ext_discount_amt) + ws_ext_sales_price) / 2))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #20 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #20 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,isEmpty,ws_ext_list_price,ws_ext_wholesale_cost,ws_ext_discount_amt,ws_ext_sales_price] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_sales_price,ws_ext_wholesale_cost,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt index 00967faa4..224e100ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (32) +- CometTakeOrderedAndProject (31) +- CometHashAggregate (30) - +- CometColumnarExchange (29) + +- CometExchange (29) +- CometHashAggregate (28) +- CometProject (27) +- CometBroadcastHashJoin (26) @@ -13,11 +13,11 @@ : : :- CometProject (11) : : : +- CometSortMergeJoin (10) : : : :- CometSort (4) - : : : : +- CometColumnarExchange (3) + : : : : +- CometExchange (3) : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) : : : +- CometSort (9) - : : : +- CometColumnarExchange (8) + : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) : : : +- CometScan parquet spark_catalog.default.catalog_returns (5) @@ -45,9 +45,9 @@ ReadSchema: struct= 2000-03-11) THEN (cs_sales_price#4 - coalesce(cast(cr_refunded_cash#9 as decimal(12,2)), 0.00)) ELSE 0.00 END)] -(29) CometColumnarExchange +(29) CometExchange Input [6]: [w_state#12, i_item_id#14, sum#18, isEmpty#19, sum#20, isEmpty#21] -Arguments: hashpartitioning(w_state#12, i_item_id#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(w_state#12, i_item_id#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (30) CometHashAggregate Input [6]: [w_state#12, i_item_id#14, sum#18, isEmpty#19, sum#20, isEmpty#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt index 6e47fa59f..eac613819 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q40/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [w_state,i_item_id,sales_before,sales_after] CometHashAggregate [w_state,i_item_id,sales_before,sales_after,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_date < 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END),sum(CASE WHEN (d_date >= 2000-03-11) THEN (cs_sales_price - coalesce(cast(cr_refunded_cash as decimal(12,2)), 0.00)) ELSE 0.00 END)] - CometColumnarExchange [w_state,i_item_id] #1 + CometExchange [w_state,i_item_id] #1 CometHashAggregate [w_state,i_item_id,sum,isEmpty,sum,isEmpty,d_date,cs_sales_price,cr_refunded_cash] CometProject [cs_sales_price,cr_refunded_cash,w_state,i_item_id,d_date] CometBroadcastHashJoin [cs_sales_price,cs_sold_date_sk,cr_refunded_cash,w_state,i_item_id,d_date_sk,d_date] @@ -14,7 +14,7 @@ WholeStageCodegen (1) CometProject [cs_warehouse_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk,cr_refunded_cash] CometSortMergeJoin [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_refunded_cash] CometSort [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] - CometColumnarExchange [cs_order_number,cs_item_sk] #2 + CometExchange [cs_order_number,cs_item_sk] #2 CometFilter [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_warehouse_sk,cs_item_sk,cs_order_number,cs_sales_price,cs_sold_date_sk] SubqueryBroadcast [d_date_sk] #1 @@ -25,7 +25,7 @@ WholeStageCodegen (1) CometFilter [d_date_sk,d_date] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash] - CometColumnarExchange [cr_order_number,cr_item_sk] #4 + CometExchange [cr_order_number,cr_item_sk] #4 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt index 5c2b28b9d..c4059e5f1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -13,7 +13,7 @@ +- CometProject (11) +- CometFilter (10) +- CometHashAggregate (9) - +- CometColumnarExchange (8) + +- CometExchange (8) +- CometHashAggregate (7) +- CometProject (6) +- CometFilter (5) @@ -55,9 +55,9 @@ Input [1]: [i_manufact#5] Keys [1]: [i_manufact#5] Functions [1]: [partial_count(1)] -(8) CometColumnarExchange +(8) CometExchange Input [2]: [i_manufact#5, count#9] -Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_manufact#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (9) CometHashAggregate Input [2]: [i_manufact#5, count#9] @@ -90,9 +90,9 @@ Input [1]: [i_product_name#3] Keys [1]: [i_product_name#3] Functions: [] -(16) CometColumnarExchange +(16) CometExchange Input [1]: [i_product_name#3] -Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_product_name#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (17) CometHashAggregate Input [1]: [i_product_name#3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt index 732e8679b..007103499 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q41/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_product_name] CometHashAggregate [i_product_name] - CometColumnarExchange [i_product_name] #1 + CometExchange [i_product_name] #1 CometHashAggregate [i_product_name] CometProject [i_product_name] CometBroadcastHashJoin [i_manufact,i_product_name,i_manufact] @@ -14,7 +14,7 @@ WholeStageCodegen (1) CometProject [i_manufact] CometFilter [item_cnt,i_manufact] CometHashAggregate [item_cnt,i_manufact,count,count(1)] - CometColumnarExchange [i_manufact] #3 + CometExchange [i_manufact] #3 CometHashAggregate [i_manufact,count] CometProject [i_manufact] CometFilter [i_category,i_manufact,i_size,i_color,i_units] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt index ae0d6f333..c2c2c8bf3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -93,9 +93,9 @@ Input [4]: [d_year#2, ss_ext_sales_price#5, i_category_id#8, i_category#9] Keys [3]: [d_year#2, i_category_id#8, i_category#9] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#11] -Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(d_year#2, i_category_id#8, i_category#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [d_year#2, i_category_id#8, i_category#9, sum#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt index bdc71f6da..40528f52d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q42/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [d_year,i_category_id,i_category,sum(ss_ext_sales_price)] CometHashAggregate [d_year,i_category_id,i_category,sum(ss_ext_sales_price),sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [d_year,i_category_id,i_category] #1 + CometExchange [d_year,i_category_id,i_category] #1 CometHashAggregate [d_year,i_category_id,i_category,sum,ss_ext_sales_price] CometProject [d_year,ss_ext_sales_price,i_category_id,i_category] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_category_id,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt index 41a564299..d6cb50a4f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -93,9 +93,9 @@ Input [4]: [d_day_name#3, ss_sales_price#5, s_store_id#8, s_store_name#9] Keys [2]: [s_store_name#9, s_store_id#8] Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Sunday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Monday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Tuesday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Wednesday) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Thursday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Friday ) THEN ss_sales_price#5 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#3 = Saturday ) THEN ss_sales_price#5 END))] -(16) CometColumnarExchange +(16) CometExchange Input [9]: [s_store_name#9, s_store_id#8, sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] -Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_name#9, s_store_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [9]: [s_store_name#9, s_store_id#8, sum#11, sum#12, sum#13, sum#14, sum#15, sum#16, sum#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt index 6e1b8d6bf..447131b77 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q43/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales] CometHashAggregate [s_store_name,s_store_id,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] - CometColumnarExchange [s_store_name,s_store_id] #1 + CometExchange [s_store_name,s_store_id] #1 CometHashAggregate [s_store_name,s_store_id,sum,sum,sum,sum,sum,sum,sum,d_day_name,ss_sales_price] CometProject [d_day_name,ss_sales_price,s_store_id,s_store_name] CometBroadcastHashJoin [d_day_name,ss_store_sk,ss_sales_price,s_store_sk,s_store_id,s_store_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt index af6264478..da94ff319 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/explain.txt @@ -12,10 +12,10 @@ TakeOrderedAndProject (33) : : : +- Window (11) : : : +- * ColumnarToRow (10) : : : +- CometSort (9) - : : : +- CometColumnarExchange (8) + : : : +- CometExchange (8) : : : +- CometFilter (7) : : : +- CometHashAggregate (6) - : : : +- CometColumnarExchange (5) + : : : +- CometExchange (5) : : : +- CometHashAggregate (4) : : : +- CometProject (3) : : : +- CometFilter (2) @@ -54,9 +54,9 @@ Input [2]: [ss_item_sk#1, ss_net_profit#3] Keys [1]: [ss_item_sk#1] Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#3))] -(5) CometColumnarExchange +(5) CometExchange Input [3]: [ss_item_sk#1, sum#5, count#6] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (6) CometHashAggregate Input [3]: [ss_item_sk#1, sum#5, count#6] @@ -67,9 +67,9 @@ Functions [1]: [avg(UnscaledValue(ss_net_profit#3))] Input [2]: [item_sk#7, rank_col#8] Condition : (isnotnull(rank_col#8) AND (cast(rank_col#8 as decimal(13,7)) > (0.9 * Subquery scalar-subquery#9, [id=#10]))) -(8) CometColumnarExchange +(8) CometExchange Input [2]: [item_sk#7, rank_col#8] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (9) CometSort Input [2]: [item_sk#7, rank_col#8] @@ -180,7 +180,7 @@ Arguments: 100, [rnk#11 ASC NULLS FIRST], [rnk#11, best_performing#19, worst_per Subquery:1 Hosting operator id = 7 Hosting Expression = Subquery scalar-subquery#9, [id=#10] * ColumnarToRow (40) +- CometHashAggregate (39) - +- CometColumnarExchange (38) + +- CometExchange (38) +- CometHashAggregate (37) +- CometProject (36) +- CometFilter (35) @@ -207,9 +207,9 @@ Input [2]: [ss_store_sk#22, ss_net_profit#23] Keys [1]: [ss_store_sk#22] Functions [1]: [partial_avg(UnscaledValue(ss_net_profit#23))] -(38) CometColumnarExchange +(38) CometExchange Input [3]: [ss_store_sk#22, sum#25, count#26] -Arguments: hashpartitioning(ss_store_sk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ss_store_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (39) CometHashAggregate Input [3]: [ss_store_sk#22, sum#25, count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt index acb8001a1..6ae18d879 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q44/simplified.txt @@ -17,20 +17,20 @@ TakeOrderedAndProject [rnk,best_performing,worst_performing] ColumnarToRow InputAdapter CometSort [item_sk,rank_col] - CometColumnarExchange #1 + CometExchange #1 CometFilter [item_sk,rank_col] Subquery #1 WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [rank_col,ss_store_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - CometColumnarExchange [ss_store_sk] #3 + CometExchange [ss_store_sk] #3 CometHashAggregate [ss_store_sk,sum,count,ss_net_profit] CometProject [ss_store_sk,ss_net_profit] CometFilter [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_addr_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] CometHashAggregate [item_sk,rank_col,ss_item_sk,sum,count,avg(UnscaledValue(ss_net_profit))] - CometColumnarExchange [ss_item_sk] #2 + CometExchange [ss_item_sk] #2 CometHashAggregate [ss_item_sk,sum,count,ss_net_profit] CometProject [ss_item_sk,ss_net_profit] CometFilter [ss_item_sk,ss_store_sk,ss_net_profit,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt index 044247540..f128499e3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/explain.txt @@ -1,42 +1,40 @@ == Physical Plan == -TakeOrderedAndProject (38) -+- * HashAggregate (37) - +- * ColumnarToRow (36) - +- CometColumnarExchange (35) - +- RowToColumnar (34) - +- * HashAggregate (33) - +- * Project (32) - +- * Filter (31) - +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) - :- * ColumnarToRow (24) - : +- CometProject (23) - : +- CometBroadcastHashJoin (22) - : :- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.customer (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.customer_address (8) - : : +- CometBroadcastExchange (16) - : : +- CometProject (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.date_dim (13) - : +- CometBroadcastExchange (21) - : +- CometFilter (20) - : +- CometScan parquet spark_catalog.default.item (19) - +- BroadcastExchange (29) - +- * ColumnarToRow (28) - +- CometProject (27) - +- CometFilter (26) - +- CometScan parquet spark_catalog.default.item (25) +TakeOrderedAndProject (36) ++- * HashAggregate (35) + +- Exchange (34) + +- * HashAggregate (33) + +- * Project (32) + +- * Filter (31) + +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (30) + :- * ColumnarToRow (24) + : +- CometProject (23) + : +- CometBroadcastHashJoin (22) + : :- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.customer (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.customer_address (8) + : : +- CometBroadcastExchange (16) + : : +- CometProject (15) + : : +- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.date_dim (13) + : +- CometBroadcastExchange (21) + : +- CometFilter (20) + : +- CometScan parquet spark_catalog.default.item (19) + +- BroadcastExchange (29) + +- * ColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.item (25) (1) Scan parquet spark_catalog.default.web_sales @@ -197,56 +195,50 @@ Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#4))] Aggregate Attributes [1]: [sum#19] Results [3]: [ca_zip#11, ca_city#10, sum#20] -(34) RowToColumnar +(34) Exchange Input [3]: [ca_zip#11, ca_city#10, sum#20] +Arguments: hashpartitioning(ca_zip#11, ca_city#10, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(35) CometColumnarExchange -Input [3]: [ca_zip#11, ca_city#10, sum#20] -Arguments: hashpartitioning(ca_zip#11, ca_city#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(36) ColumnarToRow [codegen id : 3] -Input [3]: [ca_zip#11, ca_city#10, sum#20] - -(37) HashAggregate [codegen id : 3] +(35) HashAggregate [codegen id : 3] Input [3]: [ca_zip#11, ca_city#10, sum#20] Keys [2]: [ca_zip#11, ca_city#10] Functions [1]: [sum(UnscaledValue(ws_sales_price#4))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#4))#21] Results [3]: [ca_zip#11, ca_city#10, MakeDecimal(sum(UnscaledValue(ws_sales_price#4))#21,17,2) AS sum(ws_sales_price)#22] -(38) TakeOrderedAndProject +(36) TakeOrderedAndProject Input [3]: [ca_zip#11, ca_city#10, sum(ws_sales_price)#22] Arguments: 100, [ca_zip#11 ASC NULLS FIRST, ca_city#10 ASC NULLS FIRST], [ca_zip#11, ca_city#10, sum(ws_sales_price)#22] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (43) -+- * ColumnarToRow (42) - +- CometProject (41) - +- CometFilter (40) - +- CometScan parquet spark_catalog.default.date_dim (39) +BroadcastExchange (41) ++- * ColumnarToRow (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan parquet spark_catalog.default.date_dim (37) -(39) Scan parquet spark_catalog.default.date_dim +(37) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#12, d_year#13, d_qoy#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_qoy), IsNotNull(d_year), EqualTo(d_qoy,2), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(40) CometFilter +(38) CometFilter Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] Condition : ((((isnotnull(d_qoy#14) AND isnotnull(d_year#13)) AND (d_qoy#14 = 2)) AND (d_year#13 = 2001)) AND isnotnull(d_date_sk#12)) -(41) CometProject +(39) CometProject Input [3]: [d_date_sk#12, d_year#13, d_qoy#14] Arguments: [d_date_sk#12], [d_date_sk#12] -(42) ColumnarToRow [codegen id : 1] +(40) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#12] -(43) BroadcastExchange +(41) BroadcastExchange Input [1]: [d_date_sk#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt index 8550b91c6..f60fdb18a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q45/simplified.txt @@ -1,53 +1,51 @@ TakeOrderedAndProject [ca_zip,ca_city,sum(ws_sales_price)] WholeStageCodegen (3) HashAggregate [ca_zip,ca_city,sum] [sum(UnscaledValue(ws_sales_price)),sum(ws_sales_price),sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_zip,ca_city] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] - Project [ws_sales_price,ca_city,ca_zip] - Filter [ca_zip,exists] - BroadcastHashJoin [i_item_id,i_item_id] - ColumnarToRow - InputAdapter - CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] - CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] - CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 - CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #4 - CometFilter [ca_address_sk,ca_city,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [i_item_sk,i_item_id] #6 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [i_item_id] - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + Exchange [ca_zip,ca_city] #1 + WholeStageCodegen (2) + HashAggregate [ca_zip,ca_city,ws_sales_price] [sum,sum] + Project [ws_sales_price,ca_city,ca_zip] + Filter [ca_zip,exists] + BroadcastHashJoin [i_item_id,i_item_id] + ColumnarToRow + InputAdapter + CometProject [ws_sales_price,ca_city,ca_zip,i_item_id] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ca_city,ca_zip,i_item_sk,i_item_id] + CometProject [ws_item_sk,ws_sales_price,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip,d_date_sk] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,ca_city,ca_zip] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk,ca_address_sk,ca_city,ca_zip] + CometProject [ws_item_sk,ws_sales_price,ws_sold_date_sk,c_current_addr_sk] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk,c_customer_sk,c_current_addr_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_sales_price,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #3 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ca_address_sk,ca_city,ca_zip] #4 + CometFilter [ca_address_sk,ca_city,ca_zip] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_city,ca_zip] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [i_item_id] + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt index b81931528..75334c745 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/explain.txt @@ -6,7 +6,7 @@ :- CometProject (33) : +- CometBroadcastHashJoin (32) : :- CometHashAggregate (28) - : : +- CometColumnarExchange (27) + : : +- CometExchange (27) : : +- CometHashAggregate (26) : : +- CometProject (25) : : +- CometBroadcastHashJoin (24) @@ -164,9 +164,9 @@ Input [6]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_coupon_amt#6, Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19] Functions [2]: [partial_sum(UnscaledValue(ss_coupon_amt#6)), partial_sum(UnscaledValue(ss_net_profit#7))] -(27) CometColumnarExchange +(27) CometExchange Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate Input [6]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#19, sum#20, sum#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt index 1d28a8ede..e1b53bf4d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q46/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometProject [ss_ticket_number,bought_city,amt,profit,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometHashAggregate [ss_ticket_number,ss_customer_sk,bought_city,amt,profit,ss_addr_sk,ca_city,sum,sum,sum(UnscaledValue(ss_coupon_amt)),sum(UnscaledValue(ss_net_profit))] - CometColumnarExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,ss_coupon_amt,ss_net_profit] CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_city] CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_coupon_amt,ss_net_profit,ca_address_sk,ca_city] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt index 3fe3003ce..8221d32a3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/explain.txt @@ -11,9 +11,9 @@ TakeOrderedAndProject (45) : : +- Window (24) : : +- * ColumnarToRow (23) : : +- CometSort (22) - : : +- CometColumnarExchange (21) + : : +- CometExchange (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -135,18 +135,18 @@ Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#10, d_moy#11, s_st Keys [6]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -(19) CometColumnarExchange +(19) CometExchange Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (20) CometHashAggregate Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#15] Keys [6]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -(21) CometColumnarExchange +(21) CometExchange Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (22) CometSort Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#16, _w0#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt index 91c2cd65d..2c8192f43 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q47/simplified.txt @@ -16,9 +16,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_bra ColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum,ss_sales_price] CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt index 3393afedf..40181264b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (28) +- CometHashAggregate (27) - +- CometColumnarExchange (26) + +- CometExchange (26) +- CometHashAggregate (25) +- CometProject (24) +- CometBroadcastHashJoin (23) @@ -150,9 +150,9 @@ Input [1]: [ss_quantity#4] Keys: [] Functions [1]: [partial_sum(ss_quantity#4)] -(26) CometColumnarExchange +(26) CometExchange Input [1]: [sum#18] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [1]: [sum#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt index 8bb0cede6..a46d8d603 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q48/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [sum(ss_quantity),sum,sum(ss_quantity)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,ss_quantity] CometProject [ss_quantity] CometBroadcastHashJoin [ss_quantity,ss_sold_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt index 4b0d5aed0..f7e750d47 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/explain.txt @@ -1,83 +1,81 @@ == Physical Plan == -* ColumnarToRow (79) -+- CometTakeOrderedAndProject (78) - +- CometHashAggregate (77) - +- CometColumnarExchange (76) - +- RowToColumnar (75) - +- * HashAggregate (74) - +- Union (73) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * ColumnarToRow (21) - : +- CometSort (20) - : +- CometColumnarExchange (19) - : +- CometHashAggregate (18) - : +- CometColumnarExchange (17) - : +- CometHashAggregate (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometScan parquet spark_catalog.default.web_returns (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.date_dim (10) - :- * Project (49) - : +- * Filter (48) - : +- Window (47) - : +- * Sort (46) - : +- Window (45) - : +- * ColumnarToRow (44) - : +- CometSort (43) - : +- CometColumnarExchange (42) - : +- CometHashAggregate (41) - : +- CometColumnarExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometBroadcastExchange (30) - : : : +- CometProject (29) - : : : +- CometFilter (28) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) - : : +- CometProject (33) - : : +- CometFilter (32) - : : +- CometScan parquet spark_catalog.default.catalog_returns (31) - : +- ReusedExchange (36) - +- * Project (72) - +- * Filter (71) - +- Window (70) - +- * Sort (69) - +- Window (68) - +- * ColumnarToRow (67) - +- CometSort (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometColumnarExchange (63) - +- CometHashAggregate (62) - +- CometProject (61) - +- CometBroadcastHashJoin (60) - :- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometBroadcastExchange (53) - : : +- CometProject (52) - : : +- CometFilter (51) - : : +- CometScan parquet spark_catalog.default.store_sales (50) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometScan parquet spark_catalog.default.store_returns (54) - +- ReusedExchange (59) +TakeOrderedAndProject (77) ++- * HashAggregate (76) + +- Exchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * ColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * ColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * ColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) (1) Scan parquet spark_catalog.default.web_sales @@ -157,18 +155,18 @@ Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, w Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -(19) CometColumnarExchange +(19) CometExchange Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort Input [3]: [item#22, return_ratio#23, currency_ratio#24] @@ -258,18 +256,18 @@ Input [5]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37 Keys [1]: [cs_item_sk#28] Functions [4]: [partial_sum(coalesce(cr_return_quantity#37, 0)), partial_sum(coalesce(cs_quantity#30, 0)), partial_sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] -(40) CometColumnarExchange +(40) CometExchange Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] Keys [1]: [cs_item_sk#28] Functions [4]: [sum(coalesce(cr_return_quantity#37, 0)), sum(coalesce(cs_quantity#30, 0)), sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] -(42) CometColumnarExchange +(42) CometExchange Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort Input [3]: [item#47, return_ratio#48, currency_ratio#49] @@ -359,18 +357,18 @@ Input [5]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62 Keys [1]: [ss_item_sk#53] Functions [4]: [partial_sum(coalesce(sr_return_quantity#62, 0)), partial_sum(coalesce(ss_quantity#55, 0)), partial_sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] -(63) CometColumnarExchange +(63) CometExchange Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (64) CometHashAggregate Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] Keys [1]: [ss_item_sk#53] Functions [4]: [sum(coalesce(sr_return_quantity#62, 0)), sum(coalesce(ss_quantity#55, 0)), sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] -(65) CometColumnarExchange +(65) CometExchange Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (66) CometSort Input [3]: [item#72, return_ratio#73, currency_ratio#74] @@ -408,54 +406,50 @@ Functions: [] Aggregate Attributes: [] Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -(75) RowToColumnar +(75) Exchange Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(76) CometColumnarExchange -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(77) CometHashAggregate +(76) HashAggregate [codegen id : 11] Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -(78) CometTakeOrderedAndProject -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#27 ASC NULLS FIRST,return_rank#25 ASC NULLS FIRST,currency_rank#26 ASC NULLS FIRST], output=[channel#27,item#22,return_ratio#23,return_rank#25,currency_rank#26]), [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26], 100, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] - -(79) ColumnarToRow [codegen id : 11] +(77) TakeOrderedAndProject Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Arguments: 100, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (84) -+- * ColumnarToRow (83) - +- CometProject (82) - +- CometFilter (81) - +- CometScan parquet spark_catalog.default.date_dim (80) +BroadcastExchange (82) ++- * ColumnarToRow (81) + +- CometProject (80) + +- CometFilter (79) + +- CometScan parquet spark_catalog.default.date_dim (78) -(80) Scan parquet spark_catalog.default.date_dim +(78) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(81) CometFilter +(79) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) -(82) CometProject +(80) CometProject Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Arguments: [d_date_sk#13], [d_date_sk#13] -(83) ColumnarToRow [codegen id : 1] +(81) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#13] -(84) BroadcastExchange +(82) BroadcastExchange Input [1]: [d_date_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt index 0d02a9c39..d98b2b0a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q49/simplified.txt @@ -1,111 +1,109 @@ -WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] - CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] - CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (3) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (2) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #2 - CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [ws_item_sk] #3 - CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (6) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (5) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #7 - CometHashAggregate [item,return_ratio,currency_ratio,cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [cs_item_sk] #8 - CometHashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - WholeStageCodegen (9) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (8) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #10 - CometHashAggregate [item,return_ratio,currency_ratio,ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [ss_item_sk] #11 - CometHashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 +TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] + WholeStageCodegen (11) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #7 + CometHashAggregate [item,return_ratio,currency_ratio,cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #8 + CometHashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #10 + CometHashAggregate [item,return_ratio,currency_ratio,ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #11 + CometHashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt index ed1f8feda..82b5c6690 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/explain.txt @@ -1,76 +1,74 @@ == Physical Plan == -TakeOrderedAndProject (72) -+- * HashAggregate (71) - +- * ColumnarToRow (70) - +- CometColumnarExchange (69) - +- RowToColumnar (68) - +- * HashAggregate (67) - +- * Expand (66) - +- Union (65) - :- * HashAggregate (22) - : +- * ColumnarToRow (21) - : +- CometColumnarExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometUnion (7) - : : : :- CometProject (3) - : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometScan parquet spark_catalog.default.store_returns (4) - : : +- CometBroadcastExchange (11) - : : +- CometProject (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.date_dim (8) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.store (14) - :- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometColumnarExchange (39) - : +- CometHashAggregate (38) - : +- CometProject (37) - : +- CometBroadcastHashJoin (36) - : :- CometProject (32) - : : +- CometBroadcastHashJoin (31) - : : :- CometUnion (29) - : : : :- CometProject (25) - : : : : +- CometFilter (24) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) - : : : +- CometProject (28) - : : : +- CometFilter (27) - : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) - : : +- ReusedExchange (30) - : +- CometBroadcastExchange (35) - : +- CometFilter (34) - : +- CometScan parquet spark_catalog.default.catalog_page (33) - +- * HashAggregate (64) - +- * ColumnarToRow (63) - +- CometColumnarExchange (62) - +- CometHashAggregate (61) - +- CometProject (60) - +- CometBroadcastHashJoin (59) - :- CometProject (55) - : +- CometBroadcastHashJoin (54) - : :- CometUnion (52) - : : :- CometProject (44) - : : : +- CometFilter (43) - : : : +- CometScan parquet spark_catalog.default.web_sales (42) - : : +- CometProject (51) - : : +- CometBroadcastHashJoin (50) - : : :- CometBroadcastExchange (46) - : : : +- CometScan parquet spark_catalog.default.web_returns (45) - : : +- CometProject (49) - : : +- CometFilter (48) - : : +- CometScan parquet spark_catalog.default.web_sales (47) - : +- ReusedExchange (53) - +- CometBroadcastExchange (58) - +- CometFilter (57) - +- CometScan parquet spark_catalog.default.web_site (56) +TakeOrderedAndProject (70) ++- * HashAggregate (69) + +- Exchange (68) + +- * HashAggregate (67) + +- * Expand (66) + +- Union (65) + :- * HashAggregate (22) + : +- * ColumnarToRow (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometUnion (7) + : : : :- CometProject (3) + : : : : +- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : +- CometBroadcastExchange (11) + : : +- CometProject (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.date_dim (8) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan parquet spark_catalog.default.store (14) + :- * HashAggregate (41) + : +- * ColumnarToRow (40) + : +- CometExchange (39) + : +- CometHashAggregate (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (32) + : : +- CometBroadcastHashJoin (31) + : : :- CometUnion (29) + : : : :- CometProject (25) + : : : : +- CometFilter (24) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) + : : : +- CometProject (28) + : : : +- CometFilter (27) + : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) + : : +- ReusedExchange (30) + : +- CometBroadcastExchange (35) + : +- CometFilter (34) + : +- CometScan parquet spark_catalog.default.catalog_page (33) + +- * HashAggregate (64) + +- * ColumnarToRow (63) + +- CometExchange (62) + +- CometHashAggregate (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (55) + : +- CometBroadcastHashJoin (54) + : :- CometUnion (52) + : : :- CometProject (44) + : : : +- CometFilter (43) + : : : +- CometScan parquet spark_catalog.default.web_sales (42) + : : +- CometProject (51) + : : +- CometBroadcastHashJoin (50) + : : :- CometBroadcastExchange (46) + : : : +- CometScan parquet spark_catalog.default.web_returns (45) + : : +- CometProject (49) + : : +- CometFilter (48) + : : +- CometScan parquet spark_catalog.default.web_sales (47) + : +- ReusedExchange (53) + +- CometBroadcastExchange (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.web_site (56) (1) Scan parquet spark_catalog.default.store_sales @@ -166,9 +164,9 @@ Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#25] Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] -(20) CometColumnarExchange +(20) CometExchange Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] -Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) ColumnarToRow [codegen id : 1] Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] @@ -257,9 +255,9 @@ Input [5]: [sales_price#46, profit#47, return_amt#48, net_loss#49, cp_catalog_pa Keys [1]: [cp_catalog_page_id#62] Functions [4]: [partial_sum(UnscaledValue(sales_price#46)), partial_sum(UnscaledValue(return_amt#48)), partial_sum(UnscaledValue(profit#47)), partial_sum(UnscaledValue(net_loss#49))] -(39) CometColumnarExchange +(39) CometExchange Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] -Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) ColumnarToRow [codegen id : 2] Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] @@ -367,9 +365,9 @@ Input [5]: [sales_price#83, profit#84, return_amt#85, net_loss#86, web_site_id#1 Keys [1]: [web_site_id#104] Functions [4]: [partial_sum(UnscaledValue(sales_price#83)), partial_sum(UnscaledValue(return_amt#85)), partial_sum(UnscaledValue(profit#84)), partial_sum(UnscaledValue(net_loss#86))] -(62) CometColumnarExchange +(62) CometExchange Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] -Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (63) ColumnarToRow [codegen id : 3] Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] @@ -394,56 +392,50 @@ Functions [3]: [partial_sum(sales#34), partial_sum(returns#35), partial_sum(prof Aggregate Attributes [6]: [sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126] Results [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -(68) RowToColumnar +(68) Exchange Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] +Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(69) CometColumnarExchange -Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] -Arguments: hashpartitioning(channel#118, id#119, spark_grouping_id#120, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(70) ColumnarToRow [codegen id : 5] -Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] - -(71) HashAggregate [codegen id : 5] +(69) HashAggregate [codegen id : 5] Input [9]: [channel#118, id#119, spark_grouping_id#120, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132] Keys [3]: [channel#118, id#119, spark_grouping_id#120] Functions [3]: [sum(sales#34), sum(returns#35), sum(profit#36)] Aggregate Attributes [3]: [sum(sales#34)#133, sum(returns#35)#134, sum(profit#36)#135] Results [5]: [channel#118, id#119, sum(sales#34)#133 AS sales#136, sum(returns#35)#134 AS returns#137, sum(profit#36)#135 AS profit#138] -(72) TakeOrderedAndProject +(70) TakeOrderedAndProject Input [5]: [channel#118, id#119, sales#136, returns#137, profit#138] Arguments: 100, [channel#118 ASC NULLS FIRST, id#119 ASC NULLS FIRST], [channel#118, id#119, sales#136, returns#137, profit#138] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (77) -+- * ColumnarToRow (76) - +- CometProject (75) - +- CometFilter (74) - +- CometScan parquet spark_catalog.default.date_dim (73) +BroadcastExchange (75) ++- * ColumnarToRow (74) + +- CometProject (73) + +- CometFilter (72) + +- CometScan parquet spark_catalog.default.date_dim (71) -(73) Scan parquet spark_catalog.default.date_dim +(71) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-23), LessThanOrEqual(d_date,2000-09-06), IsNotNull(d_date_sk)] ReadSchema: struct -(74) CometFilter +(72) CometFilter Input [2]: [d_date_sk#22, d_date#23] Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 2000-08-23)) AND (d_date#23 <= 2000-09-06)) AND isnotnull(d_date_sk#22)) -(75) CometProject +(73) CometProject Input [2]: [d_date_sk#22, d_date#23] Arguments: [d_date_sk#22], [d_date_sk#22] -(76) ColumnarToRow [codegen id : 1] +(74) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#22] -(77) BroadcastExchange +(75) BroadcastExchange Input [1]: [d_date_sk#22] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt index 7b4b411a4..c6f9f60af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q5/simplified.txt @@ -1,95 +1,93 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (5) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_id] #2 - CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #5 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [cp_catalog_page_id] #6 - CometHashAggregate [cp_catalog_page_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [page_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [web_site_id] #8 - CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [web_site_sk,web_site_id] #10 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + InputAdapter + Exchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #6 + CometHashAggregate [cp_catalog_page_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #7 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),sales,returns,profit,channel,id,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [web_site_id] #8 + CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #9 + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [web_site_sk,web_site_id] #10 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt index 1e9d7743c..84eef7b55 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (28) +- CometTakeOrderedAndProject (27) +- CometHashAggregate (26) - +- CometColumnarExchange (25) + +- CometExchange (25) +- CometHashAggregate (24) +- CometProject (23) +- CometBroadcastHashJoin (22) @@ -147,9 +147,9 @@ Input [12]: [ss_sold_date_sk#5, sr_returned_date_sk#9, s_store_name#12, s_compan Keys [10]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21] Functions [5]: [partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 30) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 60) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 90) AND ((sr_returned_date_sk#9 - ss_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((sr_returned_date_sk#9 - ss_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(25) CometColumnarExchange +(25) CometExchange Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, sum#26, sum#27, sum#28, sum#29, sum#30] -Arguments: hashpartitioning(s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometHashAggregate Input [15]: [s_store_name#12, s_company_id#13, s_street_number#14, s_street_name#15, s_street_type#16, s_suite_number#17, s_city#18, s_county#19, s_state#20, s_zip#21, sum#26, sum#27, sum#28, sum#29, sum#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt index 075786f9b..533334464 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q50/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] CometHashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,sum,sum,sum,sum,sum,sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 30) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 60) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((sr_returned_date_sk - ss_sold_date_sk) > 90) AND ((sr_returned_date_sk - ss_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((sr_returned_date_sk - ss_sold_date_sk) > 120) THEN 1 ELSE 0 END)] - CometColumnarExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 + CometExchange [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] #1 CometHashAggregate [s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,sum,sum,sum,sum,sum,sr_returned_date_sk,ss_sold_date_sk] CometProject [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip] CometBroadcastHashJoin [ss_sold_date_sk,sr_returned_date_sk,s_store_name,s_company_id,s_street_number,s_street_name,s_street_type,s_suite_number,s_city,s_county,s_state,s_zip,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt index 3bd86214d..fcd43fe3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/explain.txt @@ -1,47 +1,44 @@ == Physical Plan == -* ColumnarToRow (43) -+- CometTakeOrderedAndProject (42) - +- CometFilter (41) - +- CometWindowExec (40) - +- CometSort (39) - +- CometColumnarExchange (38) - +- CometProject (37) - +- CometSortMergeJoin (36) - :- CometSort (19) - : +- CometColumnarExchange (18) - : +- RowToColumnar (17) - : +- * Project (16) - : +- Window (15) - : +- * ColumnarToRow (14) - : +- CometSort (13) - : +- CometColumnarExchange (12) - : +- CometHashAggregate (11) - : +- CometColumnarExchange (10) - : +- CometHashAggregate (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.web_sales (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.date_dim (3) - +- CometSort (35) - +- CometColumnarExchange (34) - +- RowToColumnar (33) - +- * Project (32) - +- Window (31) - +- * ColumnarToRow (30) - +- CometSort (29) - +- CometColumnarExchange (28) - +- CometHashAggregate (27) - +- CometColumnarExchange (26) - +- CometHashAggregate (25) - +- CometProject (24) - +- CometBroadcastHashJoin (23) - :- CometFilter (21) - : +- CometScan parquet spark_catalog.default.store_sales (20) - +- ReusedExchange (22) +TakeOrderedAndProject (40) ++- * Filter (39) + +- Window (38) + +- * Sort (37) + +- Exchange (36) + +- * Project (35) + +- * SortMergeJoin FullOuter (34) + :- * Sort (18) + : +- Exchange (17) + : +- * Project (16) + : +- Window (15) + : +- * ColumnarToRow (14) + : +- CometSort (13) + : +- CometExchange (12) + : +- CometHashAggregate (11) + : +- CometExchange (10) + : +- CometHashAggregate (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.date_dim (3) + +- * Sort (33) + +- Exchange (32) + +- * Project (31) + +- Window (30) + +- * ColumnarToRow (29) + +- CometSort (28) + +- CometExchange (27) + +- CometHashAggregate (26) + +- CometExchange (25) + +- CometHashAggregate (24) + +- CometProject (23) + +- CometBroadcastHashJoin (22) + :- CometFilter (20) + : +- CometScan parquet spark_catalog.default.store_sales (19) + +- ReusedExchange (21) (1) Scan parquet spark_catalog.default.web_sales @@ -89,18 +86,18 @@ Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6] Keys [2]: [ws_item_sk#1, d_date#6] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -(10) CometColumnarExchange +(10) CometExchange Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate Input [3]: [ws_item_sk#1, d_date#6, sum#8] Keys [2]: [ws_item_sk#1, d_date#6] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -(12) CometColumnarExchange +(12) CometExchange Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (13) CometSort Input [4]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1] @@ -117,18 +114,15 @@ Arguments: [sum(_w0#10) windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FI Output [3]: [item_sk#9, d_date#6, cume_sales#11] Input [5]: [item_sk#9, d_date#6, _w0#10, ws_item_sk#1, cume_sales#11] -(17) RowToColumnar +(17) Exchange Input [3]: [item_sk#9, d_date#6, cume_sales#11] +Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(18) CometColumnarExchange +(18) Sort [codegen id : 3] Input [3]: [item_sk#9, d_date#6, cume_sales#11] -Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0 -(19) CometSort -Input [3]: [item_sk#9, d_date#6, cume_sales#11] -Arguments: [item_sk#9, d_date#6, cume_sales#11], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] - -(20) Scan parquet spark_catalog.default.store_sales +(19) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] Batched: true Location: InMemoryFileIndex [] @@ -136,130 +130,125 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#14), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(21) CometFilter +(20) CometFilter Input [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] Condition : isnotnull(ss_item_sk#12) -(22) ReusedExchange [Reuses operator id: 6] +(21) ReusedExchange [Reuses operator id: 6] Output [2]: [d_date_sk#16, d_date#17] -(23) CometBroadcastHashJoin +(22) CometBroadcastHashJoin Left output [3]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14] Right output [2]: [d_date_sk#16, d_date#17] Arguments: [ss_sold_date_sk#14], [d_date_sk#16], Inner, BuildRight -(24) CometProject +(23) CometProject Input [5]: [ss_item_sk#12, ss_sales_price#13, ss_sold_date_sk#14, d_date_sk#16, d_date#17] Arguments: [ss_item_sk#12, ss_sales_price#13, d_date#17], [ss_item_sk#12, ss_sales_price#13, d_date#17] -(25) CometHashAggregate +(24) CometHashAggregate Input [3]: [ss_item_sk#12, ss_sales_price#13, d_date#17] Keys [2]: [ss_item_sk#12, d_date#17] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#13))] -(26) CometColumnarExchange +(25) CometExchange Input [3]: [ss_item_sk#12, d_date#17, sum#18] -Arguments: hashpartitioning(ss_item_sk#12, d_date#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ss_item_sk#12, d_date#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(27) CometHashAggregate +(26) CometHashAggregate Input [3]: [ss_item_sk#12, d_date#17, sum#18] Keys [2]: [ss_item_sk#12, d_date#17] Functions [1]: [sum(UnscaledValue(ss_sales_price#13))] -(28) CometColumnarExchange +(27) CometExchange Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] -Arguments: hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ss_item_sk#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(29) CometSort +(28) CometSort Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] Arguments: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12], [ss_item_sk#12 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST] -(30) ColumnarToRow [codegen id : 3] +(29) ColumnarToRow [codegen id : 4] Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] -(31) Window +(30) Window Input [4]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12] Arguments: [sum(_w0#20) windowspecdefinition(ss_item_sk#12, d_date#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS cume_sales#21], [ss_item_sk#12], [d_date#17 ASC NULLS FIRST] -(32) Project [codegen id : 4] +(31) Project [codegen id : 5] Output [3]: [item_sk#19, d_date#17, cume_sales#21] Input [5]: [item_sk#19, d_date#17, _w0#20, ss_item_sk#12, cume_sales#21] -(33) RowToColumnar -Input [3]: [item_sk#19, d_date#17, cume_sales#21] - -(34) CometColumnarExchange +(32) Exchange Input [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: hashpartitioning(item_sk#19, d_date#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(item_sk#19, d_date#17, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(35) CometSort +(33) Sort [codegen id : 6] Input [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#19, d_date#17, cume_sales#21], [item_sk#19 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST] +Arguments: [item_sk#19 ASC NULLS FIRST, d_date#17 ASC NULLS FIRST], false, 0 -(36) CometSortMergeJoin -Left output [3]: [item_sk#9, d_date#6, cume_sales#11] -Right output [3]: [item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#9, d_date#6], [item_sk#19, d_date#17], FullOuter +(34) SortMergeJoin [codegen id : 7] +Left keys [2]: [item_sk#9, d_date#6] +Right keys [2]: [item_sk#19, d_date#17] +Join type: FullOuter +Join condition: None -(37) CometProject +(35) Project [codegen id : 7] +Output [4]: [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#19 END AS item_sk#22, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#17 END AS d_date#23, cume_sales#11 AS web_sales#24, cume_sales#21 AS store_sales#25] Input [6]: [item_sk#9, d_date#6, cume_sales#11, item_sk#19, d_date#17, cume_sales#21] -Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#19 END AS item_sk#22, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#17 END AS d_date#23, cume_sales#11 AS web_sales#24, cume_sales#21 AS store_sales#25] -(38) CometColumnarExchange +(36) Exchange Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: hashpartitioning(item_sk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(item_sk#22, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(39) CometSort +(37) Sort [codegen id : 8] Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25], [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST] +Arguments: [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], false, 0 -(40) CometWindowExec +(38) Window Input [4]: [item_sk#22, d_date#23, web_sales#24, store_sales#25] -Arguments: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27], [max(web_sales#24) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#26, max(store_sales#25) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#27], [item_sk#22], [d_date#23 ASC NULLS FIRST] +Arguments: [max(web_sales#24) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS web_cumulative#26, max(store_sales#25) windowspecdefinition(item_sk#22, d_date#23 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS store_cumulative#27], [item_sk#22], [d_date#23 ASC NULLS FIRST] -(41) CometFilter +(39) Filter [codegen id : 9] Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] Condition : ((isnotnull(web_cumulative#26) AND isnotnull(store_cumulative#27)) AND (web_cumulative#26 > store_cumulative#27)) -(42) CometTakeOrderedAndProject -Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[item_sk#22 ASC NULLS FIRST,d_date#23 ASC NULLS FIRST], output=[item_sk#22,d_date#23,web_sales#24,store_sales#25,web_cumulative#26,store_cumulative#27]), [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27], 100, [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] - -(43) ColumnarToRow [codegen id : 5] +(40) TakeOrderedAndProject Input [6]: [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] +Arguments: 100, [item_sk#22 ASC NULLS FIRST, d_date#23 ASC NULLS FIRST], [item_sk#22, d_date#23, web_sales#24, store_sales#25, web_cumulative#26, store_cumulative#27] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (48) -+- * ColumnarToRow (47) - +- CometProject (46) - +- CometFilter (45) - +- CometScan parquet spark_catalog.default.date_dim (44) +BroadcastExchange (45) ++- * ColumnarToRow (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan parquet spark_catalog.default.date_dim (41) -(44) Scan parquet spark_catalog.default.date_dim +(41) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(45) CometFilter +(42) CometFilter Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1200)) AND (d_month_seq#7 <= 1211)) AND isnotnull(d_date_sk#5)) -(46) CometProject +(43) CometProject Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] -(47) ColumnarToRow [codegen id : 1] +(44) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#5, d_date#6] -(48) BroadcastExchange +(45) BroadcastExchange Input [2]: [d_date_sk#5, d_date#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#4 +Subquery:2 Hosting operator id = 19 Hosting Expression = ss_sold_date_sk#14 IN dynamicpruning#4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt index 0e5a8e3d7..f972c31c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q51/simplified.txt @@ -1,62 +1,68 @@ -WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - CometFilter [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - CometWindowExec [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - CometSort [item_sk,d_date,web_sales,store_sales] - CometColumnarExchange [item_sk] #1 - CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] - CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #2 - RowToColumnar - WholeStageCodegen (2) - Project [item_sk,d_date,cume_sales] +TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] + WholeStageCodegen (9) + Filter [web_cumulative,store_cumulative] + InputAdapter + Window [web_sales,item_sk,d_date,store_sales] + WholeStageCodegen (8) + Sort [item_sk,d_date] + InputAdapter + Exchange [item_sk] #1 + WholeStageCodegen (7) + Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] + SortMergeJoin [item_sk,d_date,item_sk,d_date] + InputAdapter + WholeStageCodegen (3) + Sort [item_sk,d_date] InputAdapter - Window [_w0,ws_item_sk,d_date] - WholeStageCodegen (1) - ColumnarToRow + Exchange [item_sk,d_date] #2 + WholeStageCodegen (2) + Project [item_sk,d_date,cume_sales] InputAdapter - CometSort [item_sk,d_date,_w0,ws_item_sk] - CometColumnarExchange [ws_item_sk] #3 - CometHashAggregate [item_sk,d_date,_w0,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] - CometColumnarExchange [ws_item_sk,d_date] #4 - CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #6 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #7 - RowToColumnar - WholeStageCodegen (4) - Project [item_sk,d_date,cume_sales] + Window [_w0,ws_item_sk,d_date] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ws_item_sk] + CometExchange [ws_item_sk] #3 + CometHashAggregate [item_sk,d_date,_w0,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] + CometExchange [ws_item_sk,d_date] #4 + CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [d_date_sk,d_date] #6 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + InputAdapter + WholeStageCodegen (6) + Sort [item_sk,d_date] InputAdapter - Window [_w0,ss_item_sk,d_date] - WholeStageCodegen (3) - ColumnarToRow + Exchange [item_sk,d_date] #7 + WholeStageCodegen (5) + Project [item_sk,d_date,cume_sales] InputAdapter - CometSort [item_sk,d_date,_w0,ss_item_sk] - CometColumnarExchange [ss_item_sk] #8 - CometHashAggregate [item_sk,d_date,_w0,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [ss_item_sk,d_date] #9 - CometHashAggregate [ss_item_sk,d_date,sum,ss_sales_price] - CometProject [ss_item_sk,ss_sales_price,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #6 + Window [_w0,ss_item_sk,d_date] + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [item_sk,d_date,_w0,ss_item_sk] + CometExchange [ss_item_sk] #8 + CometHashAggregate [item_sk,d_date,_w0,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] + CometExchange [ss_item_sk,d_date] #9 + CometHashAggregate [ss_item_sk,d_date,sum,ss_sales_price] + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_date] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt index f8beff744..eee36d229 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -93,9 +93,9 @@ Input [4]: [d_year#2, ss_ext_sales_price#5, i_brand_id#8, i_brand#9] Keys [3]: [d_year#2, i_brand#9, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(d_year#2, i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [d_year#2, i_brand#9, i_brand_id#8, sum#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt index 99f3baf91..af5223b69 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q52/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [d_year,brand_id,brand,ext_price] CometHashAggregate [d_year,brand_id,brand,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [d_year,i_brand,i_brand_id] #1 + CometExchange [d_year,i_brand,i_brand_id] #1 CometHashAggregate [d_year,i_brand,i_brand_id,sum,ss_ext_sales_price] CometProject [d_year,ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [d_year,ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt index 71c84c040..8156e275b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/explain.txt @@ -5,9 +5,9 @@ TakeOrderedAndProject (29) +- Window (26) +- * ColumnarToRow (25) +- CometSort (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) - +- CometColumnarExchange (21) + +- CometExchange (21) +- CometHashAggregate (20) +- CometProject (19) +- CometBroadcastHashJoin (18) @@ -127,18 +127,18 @@ Input [3]: [i_manufact_id#5, ss_sales_price#12, d_qoy#17] Keys [2]: [i_manufact_id#5, d_qoy#17] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -(21) CometColumnarExchange +(21) CometExchange Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate Input [3]: [i_manufact_id#5, d_qoy#17, sum#19] Keys [2]: [i_manufact_id#5, d_qoy#17] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] -Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_manufact_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt index a0d09de86..363b84b35 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q53/simplified.txt @@ -8,9 +8,9 @@ TakeOrderedAndProject [avg_quarterly_sales,sum_sales,i_manufact_id] ColumnarToRow InputAdapter CometSort [i_manufact_id,sum_sales,_w0] - CometColumnarExchange [i_manufact_id] #1 + CometExchange [i_manufact_id] #1 CometHashAggregate [i_manufact_id,sum_sales,_w0,d_qoy,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_manufact_id,d_qoy] #2 + CometExchange [i_manufact_id,d_qoy] #2 CometHashAggregate [i_manufact_id,d_qoy,sum,ss_sales_price] CometProject [i_manufact_id,ss_sales_price,d_qoy] CometBroadcastHashJoin [i_manufact_id,ss_store_sk,ss_sales_price,d_qoy,s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt index cf1531550..d5bd8e387 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/explain.txt @@ -2,10 +2,10 @@ * ColumnarToRow (56) +- CometTakeOrderedAndProject (55) +- CometHashAggregate (54) - +- CometColumnarExchange (53) + +- CometExchange (53) +- CometHashAggregate (52) +- CometHashAggregate (51) - +- CometColumnarExchange (50) + +- CometExchange (50) +- CometHashAggregate (49) +- CometProject (48) +- CometBroadcastHashJoin (47) @@ -16,7 +16,7 @@ : : :- CometProject (32) : : : +- CometBroadcastHashJoin (31) : : : :- CometHashAggregate (27) - : : : : +- CometColumnarExchange (26) + : : : : +- CometExchange (26) : : : : +- CometHashAggregate (25) : : : : +- CometProject (24) : : : : +- CometBroadcastHashJoin (23) @@ -178,9 +178,9 @@ Input [2]: [c_customer_sk#20, c_current_addr_sk#21] Keys [2]: [c_customer_sk#20, c_current_addr_sk#21] Functions: [] -(26) CometColumnarExchange +(26) CometExchange Input [2]: [c_customer_sk#20, c_current_addr_sk#21] -Arguments: hashpartitioning(c_customer_sk#20, c_current_addr_sk#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_sk#20, c_current_addr_sk#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [2]: [c_customer_sk#20, c_current_addr_sk#21] @@ -293,9 +293,9 @@ Input [2]: [c_customer_sk#20, ss_ext_sales_price#23] Keys [1]: [c_customer_sk#20] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#23))] -(50) CometColumnarExchange +(50) CometExchange Input [2]: [c_customer_sk#20, sum#37] -Arguments: hashpartitioning(c_customer_sk#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_sk#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (51) CometHashAggregate Input [2]: [c_customer_sk#20, sum#37] @@ -307,9 +307,9 @@ Input [1]: [segment#38] Keys [1]: [segment#38] Functions [1]: [partial_count(1)] -(53) CometColumnarExchange +(53) CometExchange Input [2]: [segment#38, count#39] -Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(segment#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (54) CometHashAggregate Input [2]: [segment#38, count#39] @@ -390,7 +390,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:4 Hosting operator id = 63 Hosting Expression = Subquery scalar-subquery#33, [id=#34] * ColumnarToRow (73) +- CometHashAggregate (72) - +- CometColumnarExchange (71) + +- CometExchange (71) +- CometHashAggregate (70) +- CometProject (69) +- CometFilter (68) @@ -417,9 +417,9 @@ Input [1]: [(d_month_seq + 1)#45] Keys [1]: [(d_month_seq + 1)#45] Functions: [] -(71) CometColumnarExchange +(71) CometExchange Input [1]: [(d_month_seq + 1)#45] -Arguments: hashpartitioning((d_month_seq + 1)#45, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning((d_month_seq + 1)#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (72) CometHashAggregate Input [1]: [(d_month_seq + 1)#45] @@ -432,7 +432,7 @@ Input [1]: [(d_month_seq + 1)#45] Subquery:5 Hosting operator id = 63 Hosting Expression = Subquery scalar-subquery#35, [id=#36] * ColumnarToRow (80) +- CometHashAggregate (79) - +- CometColumnarExchange (78) + +- CometExchange (78) +- CometHashAggregate (77) +- CometProject (76) +- CometFilter (75) @@ -459,9 +459,9 @@ Input [1]: [(d_month_seq + 3)#49] Keys [1]: [(d_month_seq + 3)#49] Functions: [] -(78) CometColumnarExchange +(78) CometExchange Input [1]: [(d_month_seq + 3)#49] -Arguments: hashpartitioning((d_month_seq + 3)#49, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning((d_month_seq + 3)#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (79) CometHashAggregate Input [1]: [(d_month_seq + 3)#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt index d3a89a6c6..129018668 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q54/simplified.txt @@ -3,10 +3,10 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [segment,num_customers,segment_base] CometHashAggregate [segment,num_customers,segment_base,count,count(1)] - CometColumnarExchange [segment] #1 + CometExchange [segment] #1 CometHashAggregate [segment,count] CometHashAggregate [segment,c_customer_sk,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [c_customer_sk] #2 + CometExchange [c_customer_sk] #2 CometHashAggregate [c_customer_sk,sum,ss_ext_sales_price] CometProject [c_customer_sk,ss_ext_sales_price] CometBroadcastHashJoin [c_customer_sk,ss_ext_sales_price,ss_sold_date_sk,d_date_sk] @@ -17,7 +17,7 @@ WholeStageCodegen (1) CometProject [c_customer_sk,c_current_addr_sk,ss_ext_sales_price,ss_sold_date_sk] CometBroadcastHashJoin [c_customer_sk,c_current_addr_sk,ss_customer_sk,ss_ext_sales_price,ss_sold_date_sk] CometHashAggregate [c_customer_sk,c_current_addr_sk] - CometColumnarExchange [c_customer_sk,c_current_addr_sk] #3 + CometExchange [c_customer_sk,c_current_addr_sk] #3 CometHashAggregate [c_customer_sk,c_current_addr_sk] CometProject [c_customer_sk,c_current_addr_sk] CometBroadcastHashJoin [customer_sk,c_customer_sk,c_current_addr_sk] @@ -67,7 +67,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [(d_month_seq + 1)] - CometColumnarExchange [(d_month_seq + 1)] #10 + CometExchange [(d_month_seq + 1)] #10 CometHashAggregate [(d_month_seq + 1)] CometProject [d_month_seq] [(d_month_seq + 1)] CometFilter [d_month_seq,d_year,d_moy] @@ -77,7 +77,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [(d_month_seq + 3)] - CometColumnarExchange [(d_month_seq + 3)] #11 + CometExchange [(d_month_seq + 3)] #11 CometHashAggregate [(d_month_seq + 3)] CometProject [d_month_seq] [(d_month_seq + 3)] CometFilter [d_month_seq,d_year,d_moy] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt index ee250ed9b..6eff9f52d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (19) +- CometTakeOrderedAndProject (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometProject (14) +- CometBroadcastHashJoin (13) @@ -93,9 +93,9 @@ Input [3]: [ss_ext_sales_price#5, i_brand_id#8, i_brand#9] Keys [2]: [i_brand#9, i_brand_id#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(16) CometColumnarExchange +(16) CometExchange Input [3]: [i_brand#9, i_brand_id#8, sum#11] -Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_brand#9, i_brand_id#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [3]: [i_brand#9, i_brand_id#8, sum#11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt index 45200509f..999c8a6c4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q55/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [brand_id,brand,ext_price] CometHashAggregate [brand_id,brand,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_brand,i_brand_id] #1 + CometExchange [i_brand,i_brand_id] #1 CometHashAggregate [i_brand,i_brand_id,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_brand_id,i_brand] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_brand_id,i_brand] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt index 8061fb91e..d9af63246 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/explain.txt @@ -2,11 +2,11 @@ * ColumnarToRow (61) +- CometTakeOrderedAndProject (60) +- CometHashAggregate (59) - +- CometColumnarExchange (58) + +- CometExchange (58) +- CometHashAggregate (57) +- CometUnion (56) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -33,7 +33,7 @@ : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.item (17) :- CometHashAggregate (41) - : +- CometColumnarExchange (40) + : +- CometExchange (40) : +- CometHashAggregate (39) : +- CometProject (38) : +- CometBroadcastHashJoin (37) @@ -47,7 +47,7 @@ : : +- ReusedExchange (33) : +- ReusedExchange (36) +- CometHashAggregate (55) - +- CometColumnarExchange (54) + +- CometExchange (54) +- CometHashAggregate (53) +- CometProject (52) +- CometBroadcastHashJoin (51) @@ -183,9 +183,9 @@ Input [2]: [ss_ext_sales_price#3, i_item_id#12] Keys [1]: [i_item_id#12] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(26) CometColumnarExchange +(26) CometExchange Input [2]: [i_item_id#12, sum#15] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [2]: [i_item_id#12, sum#15] @@ -245,9 +245,9 @@ Input [2]: [cs_ext_sales_price#18, i_item_id#24] Keys [1]: [i_item_id#24] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] -(40) CometColumnarExchange +(40) CometExchange Input [2]: [i_item_id#24, sum#25] -Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (41) CometHashAggregate Input [2]: [i_item_id#24, sum#25] @@ -307,9 +307,9 @@ Input [2]: [ws_ext_sales_price#28, i_item_id#34] Keys [1]: [i_item_id#34] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] -(54) CometColumnarExchange +(54) CometExchange Input [2]: [i_item_id#34, sum#35] -Arguments: hashpartitioning(i_item_id#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (55) CometHashAggregate Input [2]: [i_item_id#34, sum#35] @@ -326,9 +326,9 @@ Input [2]: [i_item_id#12, total_sales#36] Keys [1]: [i_item_id#12] Functions [1]: [partial_sum(total_sales#36)] -(58) CometColumnarExchange +(58) CometExchange Input [3]: [i_item_id#12, sum#39, isEmpty#40] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (59) CometHashAggregate Input [3]: [i_item_id#12, sum#39, isEmpty#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt index 4bb76eecc..2603a2781 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q56/simplified.txt @@ -3,11 +3,11 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,total_sales] CometHashAggregate [i_item_id,total_sales,sum,isEmpty,sum(total_sales)] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,isEmpty,total_sales] CometUnion [i_item_id,total_sales] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_item_id] #2 + CometExchange [i_item_id] #2 CometHashAggregate [i_item_id,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] @@ -42,7 +42,7 @@ WholeStageCodegen (1) CometFilter [i_item_id,i_color] CometScan parquet spark_catalog.default.item [i_item_id,i_color] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_item_id] #8 + CometExchange [i_item_id] #8 CometHashAggregate [i_item_id,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] @@ -57,7 +57,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk] #5 ReusedExchange [i_item_sk,i_item_id] #6 CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_item_id] #9 + CometExchange [i_item_id] #9 CometHashAggregate [i_item_id,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt index b5372cffc..e8a40a0f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/explain.txt @@ -11,9 +11,9 @@ TakeOrderedAndProject (45) : : +- Window (24) : : +- * ColumnarToRow (23) : : +- CometSort (22) - : : +- CometColumnarExchange (21) + : : +- CometExchange (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -135,18 +135,18 @@ Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#10, d_moy#11, cc_n Keys [5]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -(19) CometColumnarExchange +(19) CometExchange Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (20) CometHashAggregate Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#14] Keys [5]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -(21) CometColumnarExchange +(21) CometExchange Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#15, _w0#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (22) CometSort Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#15, _w0#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt index 454d12fae..96a3fdfa0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q57/simplified.txt @@ -16,9 +16,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,cc_name,i_category,i_brand,d_ ColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,cc_name] #1 + CometExchange [i_category,i_brand,cc_name] #1 CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(cs_sales_price))] - CometColumnarExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum,cs_sales_price] CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt index 06e3fb959..f05dbd720 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/explain.txt @@ -7,7 +7,7 @@ : +- CometBroadcastHashJoin (33) : :- CometFilter (19) : : +- CometHashAggregate (18) - : : +- CometColumnarExchange (17) + : : +- CometExchange (17) : : +- CometHashAggregate (16) : : +- CometProject (15) : : +- CometBroadcastHashJoin (14) @@ -27,7 +27,7 @@ : +- CometBroadcastExchange (32) : +- CometFilter (31) : +- CometHashAggregate (30) - : +- CometColumnarExchange (29) + : +- CometExchange (29) : +- CometHashAggregate (28) : +- CometProject (27) : +- CometBroadcastHashJoin (26) @@ -40,7 +40,7 @@ +- CometBroadcastExchange (47) +- CometFilter (46) +- CometHashAggregate (45) - +- CometColumnarExchange (44) + +- CometExchange (44) +- CometHashAggregate (43) +- CometProject (42) +- CometBroadcastHashJoin (41) @@ -129,9 +129,9 @@ Input [2]: [ss_ext_sales_price#2, i_item_id#6] Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [2]: [i_item_id#6, sum#10] -Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [2]: [i_item_id#6, sum#10] @@ -183,9 +183,9 @@ Input [2]: [cs_ext_sales_price#14, i_item_id#18] Keys [1]: [i_item_id#18] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#14))] -(29) CometColumnarExchange +(29) CometExchange Input [2]: [i_item_id#18, sum#20] -Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (30) CometHashAggregate Input [2]: [i_item_id#18, sum#20] @@ -250,9 +250,9 @@ Input [2]: [ws_ext_sales_price#24, i_item_id#28] Keys [1]: [i_item_id#28] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#24))] -(44) CometColumnarExchange +(44) CometExchange Input [2]: [i_item_id#28, sum#30] -Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (45) CometHashAggregate Input [2]: [i_item_id#28, sum#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt index 326fdcb78..36fc8ac57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q58/simplified.txt @@ -8,7 +8,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [item_id,ss_item_rev,item_id,cs_item_rev] CometFilter [item_id,ss_item_rev] CometHashAggregate [item_id,ss_item_rev,i_item_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,d_date_sk] @@ -48,7 +48,7 @@ WholeStageCodegen (1) CometBroadcastExchange [item_id,cs_item_rev] #6 CometFilter [item_id,cs_item_rev] CometHashAggregate [item_id,cs_item_rev,i_item_id,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_item_id] #7 + CometExchange [i_item_id] #7 CometHashAggregate [i_item_id,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,d_date_sk] @@ -62,7 +62,7 @@ WholeStageCodegen (1) CometBroadcastExchange [item_id,ws_item_rev] #8 CometFilter [item_id,ws_item_rev] CometHashAggregate [item_id,ws_item_rev,i_item_id,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_item_id] #9 + CometExchange [i_item_id] #9 CometHashAggregate [i_item_id,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt index d3621f8de..d194660f8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/explain.txt @@ -8,7 +8,7 @@ : :- CometProject (15) : : +- CometBroadcastHashJoin (14) : : :- CometHashAggregate (10) - : : : +- CometColumnarExchange (9) + : : : +- CometExchange (9) : : : +- CometHashAggregate (8) : : : +- CometProject (7) : : : +- CometBroadcastHashJoin (6) @@ -81,9 +81,9 @@ Input [4]: [ss_store_sk#1, ss_sales_price#2, d_week_seq#5, d_day_name#6] Keys [2]: [d_week_seq#5, ss_store_sk#1] Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Sunday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Monday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Tuesday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Wednesday) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Thursday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Friday ) THEN ss_sales_price#2 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#6 = Saturday ) THEN ss_sales_price#2 END))] -(9) CometColumnarExchange +(9) CometExchange Input [9]: [d_week_seq#5, ss_store_sk#1, sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] -Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(d_week_seq#5, ss_store_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometHashAggregate Input [9]: [d_week_seq#5, ss_store_sk#1, sum#7, sum#8, sum#9, sum#10, sum#11, sum#12, sum#13] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt index bb7a85817..140a76439 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q59/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometProject [d_week_seq,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_id,s_store_name] CometBroadcastHashJoin [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,s_store_sk,s_store_id,s_store_name] CometHashAggregate [d_week_seq,ss_store_sk,sun_sales,mon_sales,tue_sales,wed_sales,thu_sales,fri_sales,sat_sales,sum,sum,sum,sum,sum,sum,sum,sum(UnscaledValue(CASE WHEN (d_day_name = Sunday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Monday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Tuesday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Wednesday) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Thursday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Friday ) THEN ss_sales_price END)),sum(UnscaledValue(CASE WHEN (d_day_name = Saturday ) THEN ss_sales_price END))] - CometColumnarExchange [d_week_seq,ss_store_sk] #1 + CometExchange [d_week_seq,ss_store_sk] #1 CometHashAggregate [d_week_seq,ss_store_sk,sum,sum,sum,sum,sum,sum,sum,d_day_name,ss_sales_price] CometProject [ss_store_sk,ss_sales_price,d_week_seq,d_day_name] CometBroadcastHashJoin [ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_week_seq,d_day_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt index 0341c20d7..7a8044667 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (37) +- CometFilter (36) +- CometHashAggregate (35) - +- CometColumnarExchange (34) + +- CometExchange (34) +- CometHashAggregate (33) +- CometProject (32) +- CometBroadcastHashJoin (31) @@ -33,7 +33,7 @@ +- CometBroadcastExchange (27) +- CometFilter (26) +- CometHashAggregate (25) - +- CometColumnarExchange (24) + +- CometExchange (24) +- CometHashAggregate (23) +- CometFilter (22) +- CometScan parquet spark_catalog.default.item (21) @@ -154,9 +154,9 @@ Input [2]: [i_current_price#16, i_category#17] Keys [1]: [i_category#17] Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] -(24) CometColumnarExchange +(24) CometExchange Input [3]: [i_category#17, sum#18, count#19] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (25) CometHashAggregate Input [3]: [i_category#17, sum#18, count#19] @@ -198,9 +198,9 @@ Input [1]: [ca_state#2] Keys [1]: [ca_state#2] Functions [1]: [partial_count(1)] -(34) CometColumnarExchange +(34) CometExchange Input [2]: [ca_state#2, count#21] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (35) CometHashAggregate Input [2]: [ca_state#2, count#21] @@ -253,7 +253,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 40 Hosting Expression = Subquery scalar-subquery#11, [id=#12] * ColumnarToRow (50) +- CometHashAggregate (49) - +- CometColumnarExchange (48) + +- CometExchange (48) +- CometHashAggregate (47) +- CometProject (46) +- CometFilter (45) @@ -280,9 +280,9 @@ Input [1]: [d_month_seq#24] Keys [1]: [d_month_seq#24] Functions: [] -(48) CometColumnarExchange +(48) CometExchange Input [1]: [d_month_seq#24] -Arguments: hashpartitioning(d_month_seq#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(d_month_seq#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (49) CometHashAggregate Input [1]: [d_month_seq#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt index 0e983b96a..05566cbca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q6/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [state,cnt] CometFilter [state,cnt] CometHashAggregate [state,cnt,ca_state,count,count(1)] - CometColumnarExchange [ca_state] #1 + CometExchange [ca_state] #1 CometHashAggregate [ca_state,count] CometProject [ca_state] CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] @@ -34,7 +34,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [d_month_seq] - CometColumnarExchange [d_month_seq] #5 + CometExchange [d_month_seq] #5 CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] @@ -53,7 +53,7 @@ WholeStageCodegen (1) CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] - CometColumnarExchange [i_category] #9 + CometExchange [i_category] #9 CometHashAggregate [i_category,sum,count,i_current_price] CometFilter [i_current_price,i_category] CometScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt index 0aeb4e2b4..3b5101925 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/explain.txt @@ -2,11 +2,11 @@ * ColumnarToRow (61) +- CometTakeOrderedAndProject (60) +- CometHashAggregate (59) - +- CometColumnarExchange (58) + +- CometExchange (58) +- CometHashAggregate (57) +- CometUnion (56) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -33,7 +33,7 @@ : +- CometFilter (18) : +- CometScan parquet spark_catalog.default.item (17) :- CometHashAggregate (41) - : +- CometColumnarExchange (40) + : +- CometExchange (40) : +- CometHashAggregate (39) : +- CometProject (38) : +- CometBroadcastHashJoin (37) @@ -47,7 +47,7 @@ : : +- ReusedExchange (33) : +- ReusedExchange (36) +- CometHashAggregate (55) - +- CometColumnarExchange (54) + +- CometExchange (54) +- CometHashAggregate (53) +- CometProject (52) +- CometBroadcastHashJoin (51) @@ -183,9 +183,9 @@ Input [2]: [ss_ext_sales_price#3, i_item_id#12] Keys [1]: [i_item_id#12] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(26) CometColumnarExchange +(26) CometExchange Input [2]: [i_item_id#12, sum#15] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [2]: [i_item_id#12, sum#15] @@ -245,9 +245,9 @@ Input [2]: [cs_ext_sales_price#18, i_item_id#24] Keys [1]: [i_item_id#24] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#18))] -(40) CometColumnarExchange +(40) CometExchange Input [2]: [i_item_id#24, sum#25] -Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (41) CometHashAggregate Input [2]: [i_item_id#24, sum#25] @@ -307,9 +307,9 @@ Input [2]: [ws_ext_sales_price#28, i_item_id#34] Keys [1]: [i_item_id#34] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#28))] -(54) CometColumnarExchange +(54) CometExchange Input [2]: [i_item_id#34, sum#35] -Arguments: hashpartitioning(i_item_id#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#34, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (55) CometHashAggregate Input [2]: [i_item_id#34, sum#35] @@ -326,9 +326,9 @@ Input [2]: [i_item_id#12, total_sales#36] Keys [1]: [i_item_id#12] Functions [1]: [partial_sum(total_sales#36)] -(58) CometColumnarExchange +(58) CometExchange Input [3]: [i_item_id#12, sum#39, isEmpty#40] -Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_item_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (59) CometHashAggregate Input [3]: [i_item_id#12, sum#39, isEmpty#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt index 871f477b7..4dd8b3b5b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q60/simplified.txt @@ -3,11 +3,11 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,total_sales] CometHashAggregate [i_item_id,total_sales,sum,isEmpty,sum(total_sales)] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,isEmpty,total_sales] CometUnion [i_item_id,total_sales] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_item_id] #2 + CometExchange [i_item_id] #2 CometHashAggregate [i_item_id,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_item_id] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk,i_item_id] @@ -42,7 +42,7 @@ WholeStageCodegen (1) CometFilter [i_item_id,i_category] CometScan parquet spark_catalog.default.item [i_item_id,i_category] CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_item_id] #8 + CometExchange [i_item_id] #8 CometHashAggregate [i_item_id,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_item_id] CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,i_item_sk,i_item_id] @@ -57,7 +57,7 @@ WholeStageCodegen (1) ReusedExchange [ca_address_sk] #5 ReusedExchange [i_item_sk,i_item_id] #6 CometHashAggregate [i_item_id,total_sales,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_item_id] #9 + CometExchange [i_item_id] #9 CometHashAggregate [i_item_id,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_item_id] CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,i_item_sk,i_item_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt index 3bbacfbb9..8fcd94392 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/explain.txt @@ -3,7 +3,7 @@ +- * BroadcastNestedLoopJoin Inner BuildRight (64) :- * ColumnarToRow (41) : +- CometHashAggregate (40) - : +- CometColumnarExchange (39) + : +- CometExchange (39) : +- CometHashAggregate (38) : +- CometProject (37) : +- CometBroadcastHashJoin (36) @@ -45,7 +45,7 @@ +- BroadcastExchange (63) +- * ColumnarToRow (62) +- CometHashAggregate (61) - +- CometColumnarExchange (60) + +- CometExchange (60) +- CometHashAggregate (59) +- CometProject (58) +- CometBroadcastHashJoin (57) @@ -247,9 +247,9 @@ Input [1]: [ss_ext_sales_price#5] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#5))] -(39) CometColumnarExchange +(39) CometExchange Input [1]: [sum#23] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (40) CometHashAggregate Input [1]: [sum#23] @@ -336,9 +336,9 @@ Input [1]: [ss_ext_sales_price#28] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#28))] -(60) CometColumnarExchange +(60) CometExchange Input [1]: [sum#37] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (61) CometHashAggregate Input [1]: [sum#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt index b95b0d616..548614416 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q61/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [promotions,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,ss_ext_sales_price] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] @@ -57,7 +57,7 @@ WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [total,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange #10 + CometExchange #10 CometHashAggregate [sum,ss_ext_sales_price] CometProject [ss_ext_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/explain.txt index f58411101..82005f930 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (28) +- CometTakeOrderedAndProject (27) +- CometHashAggregate (26) - +- CometColumnarExchange (25) + +- CometExchange (25) +- CometHashAggregate (24) +- CometProject (23) +- CometBroadcastHashJoin (22) @@ -145,9 +145,9 @@ Input [5]: [ws_ship_date_sk#1, ws_sold_date_sk#5, sm_type#9, web_name#11, _group Keys [3]: [_groupingexpression#14, sm_type#9, web_name#11] Functions [5]: [partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 30) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 60) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 90) AND ((ws_ship_date_sk#1 - ws_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((ws_ship_date_sk#1 - ws_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(25) CometColumnarExchange +(25) CometExchange Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, web_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometHashAggregate Input [8]: [_groupingexpression#14, sm_type#9, web_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/simplified.txt index c7bcf72f0..6a9b1ced3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q62/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] CometHashAggregate [substr(w_warehouse_name, 1, 20),sm_type,web_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum,sum,sum,sum,sum,sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 30) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 60) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((ws_ship_date_sk - ws_sold_date_sk) > 90) AND ((ws_ship_date_sk - ws_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((ws_ship_date_sk - ws_sold_date_sk) > 120) THEN 1 ELSE 0 END)] - CometColumnarExchange [_groupingexpression,sm_type,web_name] #1 + CometExchange [_groupingexpression,sm_type,web_name] #1 CometHashAggregate [_groupingexpression,sm_type,web_name,sum,sum,sum,sum,sum,ws_ship_date_sk,ws_sold_date_sk] CometProject [w_warehouse_name] [ws_ship_date_sk,ws_sold_date_sk,sm_type,web_name,_groupingexpression] CometBroadcastHashJoin [ws_ship_date_sk,ws_sold_date_sk,w_warehouse_name,sm_type,web_name,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt index 5fd263660..4dfd50a34 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/explain.txt @@ -5,9 +5,9 @@ TakeOrderedAndProject (29) +- Window (26) +- * ColumnarToRow (25) +- CometSort (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) - +- CometColumnarExchange (21) + +- CometExchange (21) +- CometHashAggregate (20) +- CometProject (19) +- CometBroadcastHashJoin (18) @@ -127,18 +127,18 @@ Input [3]: [i_manager_id#5, ss_sales_price#12, d_moy#17] Keys [2]: [i_manager_id#5, d_moy#17] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#12))] -(21) CometColumnarExchange +(21) CometExchange Input [3]: [i_manager_id#5, d_moy#17, sum#19] -Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate Input [3]: [i_manager_id#5, d_moy#17, sum#19] Keys [2]: [i_manager_id#5, d_moy#17] Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] -Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_manager_id#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt index 1b538c9ab..8ea16ea74 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q63/simplified.txt @@ -8,9 +8,9 @@ TakeOrderedAndProject [i_manager_id,avg_monthly_sales,sum_sales] ColumnarToRow InputAdapter CometSort [i_manager_id,sum_sales,_w0] - CometColumnarExchange [i_manager_id] #1 + CometExchange [i_manager_id] #1 CometHashAggregate [i_manager_id,sum_sales,_w0,d_moy,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_manager_id,d_moy] #2 + CometExchange [i_manager_id,d_moy] #2 CometHashAggregate [i_manager_id,d_moy,sum,ss_sales_price] CometProject [i_manager_id,ss_sales_price,d_moy] CometBroadcastHashJoin [i_manager_id,ss_store_sk,ss_sales_price,d_moy,s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt index b4e05a864..35bc18855 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/explain.txt @@ -7,7 +7,7 @@ +- * SortMergeJoin Inner (171) :- * ColumnarToRow (101) : +- CometSort (100) - : +- CometColumnarExchange (99) + : +- CometExchange (99) : +- CometHashAggregate (98) : +- CometHashAggregate (97) : +- CometProject (96) @@ -43,7 +43,7 @@ : : : : : : : : : : : : : : : :- CometProject (30) : : : : : : : : : : : : : : : : +- CometSortMergeJoin (29) : : : : : : : : : : : : : : : : :- CometSort (10) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (9) + : : : : : : : : : : : : : : : : : +- CometExchange (9) : : : : : : : : : : : : : : : : : +- CometProject (8) : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) @@ -56,17 +56,17 @@ : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) : : : : : : : : : : : : : : : : +- CometHashAggregate (25) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (24) + : : : : : : : : : : : : : : : : +- CometExchange (24) : : : : : : : : : : : : : : : : +- CometHashAggregate (23) : : : : : : : : : : : : : : : : +- CometProject (22) : : : : : : : : : : : : : : : : +- CometSortMergeJoin (21) : : : : : : : : : : : : : : : : :- CometSort (15) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (14) + : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (19) + : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (16) @@ -108,7 +108,7 @@ : +- CometScan parquet spark_catalog.default.item (91) +- * ColumnarToRow (170) +- CometSort (169) - +- CometColumnarExchange (168) + +- CometExchange (168) +- CometHashAggregate (167) +- CometHashAggregate (166) +- CometProject (165) @@ -144,7 +144,7 @@ : : : : : : : : : : : : : : :- CometProject (118) : : : : : : : : : : : : : : : +- CometSortMergeJoin (117) : : : : : : : : : : : : : : : :- CometSort (111) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (110) + : : : : : : : : : : : : : : : : +- CometExchange (110) : : : : : : : : : : : : : : : : +- CometProject (109) : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (108) : : : : : : : : : : : : : : : : :- CometBroadcastExchange (104) @@ -217,9 +217,9 @@ Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number# Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(9) CometColumnarExchange +(9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -240,9 +240,9 @@ Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -(14) CometColumnarExchange +(14) CometExchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (15) CometSort Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] @@ -263,9 +263,9 @@ Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -(19) CometColumnarExchange +(19) CometExchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] @@ -285,9 +285,9 @@ Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reverse Keys [1]: [cs_item_sk#17] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] -(24) CometColumnarExchange +(24) CometExchange Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometHashAggregate Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] @@ -629,9 +629,9 @@ Input [19]: [i_product_name#71, i_item_sk#68, s_store_name#35, s_zip#36, ca_stre Keys [15]: [i_product_name#71, i_item_sk#68, s_store_name#35, s_zip#36, ca_street_number#57, ca_street_name#58, ca_city#59, ca_zip#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, d_year#33, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -(99) CometColumnarExchange +(99) CometExchange Input [17]: [product_name#76, item_sk#77, store_name#78, store_zip#79, b_street_number#80, b_streen_name#81, b_city#82, b_zip#83, c_street_number#84, c_street_name#85, c_city#86, c_zip#87, syear#88, cnt#89, s1#90, s2#91, s3#92] -Arguments: hashpartitioning(item_sk#77, store_name#78, store_zip#79, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(item_sk#77, store_name#78, store_zip#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (100) CometSort Input [17]: [product_name#76, item_sk#77, store_name#78, store_zip#79, b_street_number#80, b_streen_name#81, b_city#82, b_zip#83, c_street_number#84, c_street_name#85, c_city#86, c_zip#87, syear#88, cnt#89, s1#90, s2#91, s3#92] @@ -680,9 +680,9 @@ Arguments: [ss_item_sk#93, ss_ticket_number#100], [sr_item_sk#106, sr_ticket_num Input [14]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_ticket_number#100, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104, sr_item_sk#106, sr_ticket_number#107] Arguments: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104], [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] -(110) CometColumnarExchange +(110) CometExchange Input [11]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] -Arguments: hashpartitioning(ss_item_sk#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ss_item_sk#93, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (111) CometSort Input [11]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] @@ -919,9 +919,9 @@ Input [19]: [i_product_name#154, i_item_sk#153, s_store_name#120, s_zip#121, ca_ Keys [15]: [i_product_name#154, i_item_sk#153, s_store_name#120, s_zip#121, ca_street_number#142, ca_street_name#143, ca_city#144, ca_zip#145, ca_street_number#147, ca_street_name#148, ca_city#149, ca_zip#150, d_year#118, d_year#129, d_year#131] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#101)), sum(UnscaledValue(ss_list_price#102)), sum(UnscaledValue(ss_coupon_amt#103))] -(168) CometColumnarExchange +(168) CometExchange Input [8]: [item_sk#158, store_name#159, store_zip#160, syear#161, cnt#162, s1#163, s2#164, s3#165] -Arguments: hashpartitioning(item_sk#158, store_name#159, store_zip#160, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(item_sk#158, store_name#159, store_zip#160, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (169) CometSort Input [8]: [item_sk#158, store_name#159, store_zip#160, syear#161, cnt#162, s1#163, s2#164, s3#165] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt index 84be0e3c3..665cf4e99 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q64/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (4) ColumnarToRow InputAdapter CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #2 + CometExchange [item_sk,store_name,store_zip] #2 CometHashAggregate [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -48,7 +48,7 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #3 + CometExchange [ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 @@ -68,17 +68,17 @@ WholeStageCodegen (4) CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - CometColumnarExchange [cs_item_sk] #6 + CometExchange [cs_item_sk] #6 CometHashAggregate [cs_item_sk,sum,sum,isEmpty,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometColumnarExchange [cs_item_sk,cs_order_number] #7 + CometExchange [cs_item_sk,cs_order_number] #7 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometColumnarExchange [cr_item_sk,cr_order_number] #8 + CometExchange [cr_item_sk,cr_order_number] #8 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] @@ -123,7 +123,7 @@ WholeStageCodegen (4) ColumnarToRow InputAdapter CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #19 + CometExchange [item_sk,store_name,store_zip] #19 CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -159,7 +159,7 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #20 + CometExchange [ss_item_sk] #20 CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #21 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt index dd1796dd4..179c42f98 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/explain.txt @@ -12,7 +12,7 @@ : : +- CometBroadcastExchange (15) : : +- CometFilter (14) : : +- CometHashAggregate (13) - : : +- CometColumnarExchange (12) + : : +- CometExchange (12) : : +- CometHashAggregate (11) : : +- CometProject (10) : : +- CometBroadcastHashJoin (9) @@ -28,10 +28,10 @@ +- CometBroadcastExchange (35) +- CometFilter (34) +- CometHashAggregate (33) - +- CometColumnarExchange (32) + +- CometExchange (32) +- CometHashAggregate (31) +- CometHashAggregate (30) - +- CometColumnarExchange (29) + +- CometExchange (29) +- CometHashAggregate (28) +- CometProject (27) +- CometBroadcastHashJoin (26) @@ -96,9 +96,9 @@ Input [3]: [ss_item_sk#3, ss_store_sk#4, ss_sales_price#5] Keys [2]: [ss_store_sk#4, ss_item_sk#3] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#5))] -(12) CometColumnarExchange +(12) CometExchange Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] -Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_store_sk#4, ss_item_sk#3, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (13) CometHashAggregate Input [3]: [ss_store_sk#4, ss_item_sk#3, sum#10] @@ -175,9 +175,9 @@ Input [3]: [ss_item_sk#17, ss_store_sk#18, ss_sales_price#19] Keys [2]: [ss_store_sk#18, ss_item_sk#17] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#19))] -(29) CometColumnarExchange +(29) CometExchange Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#23] -Arguments: hashpartitioning(ss_store_sk#18, ss_item_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ss_store_sk#18, ss_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (30) CometHashAggregate Input [3]: [ss_store_sk#18, ss_item_sk#17, sum#23] @@ -189,9 +189,9 @@ Input [2]: [ss_store_sk#18, revenue#24] Keys [1]: [ss_store_sk#18] Functions [1]: [partial_avg(revenue#24)] -(32) CometColumnarExchange +(32) CometExchange Input [3]: [ss_store_sk#18, sum#25, count#26] -Arguments: hashpartitioning(ss_store_sk#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_store_sk#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate Input [3]: [ss_store_sk#18, sum#25, count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt index e41f9161d..be08ee23a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q65/simplified.txt @@ -13,7 +13,7 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_store_sk,ss_item_sk,revenue] #1 CometFilter [ss_store_sk,ss_item_sk,revenue] CometHashAggregate [ss_store_sk,ss_item_sk,revenue,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [ss_store_sk,ss_item_sk] #2 + CometExchange [ss_store_sk,ss_item_sk] #2 CometHashAggregate [ss_store_sk,ss_item_sk,sum,ss_sales_price] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] @@ -37,10 +37,10 @@ WholeStageCodegen (1) CometBroadcastExchange [ss_store_sk,ave] #6 CometFilter [ss_store_sk,ave] CometHashAggregate [ss_store_sk,ave,sum,count,avg(revenue)] - CometColumnarExchange [ss_store_sk] #7 + CometExchange [ss_store_sk] #7 CometHashAggregate [ss_store_sk,sum,count,revenue] CometHashAggregate [ss_store_sk,revenue,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [ss_store_sk,ss_item_sk] #8 + CometExchange [ss_store_sk,ss_item_sk] #8 CometHashAggregate [ss_store_sk,ss_item_sk,sum,ss_sales_price] CometProject [ss_item_sk,ss_store_sk,ss_sales_price] CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt index d0c1814d1..c8cdc8094 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/explain.txt @@ -2,11 +2,11 @@ * ColumnarToRow (50) +- CometTakeOrderedAndProject (49) +- CometHashAggregate (48) - +- CometColumnarExchange (47) + +- CometExchange (47) +- CometHashAggregate (46) +- CometUnion (45) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -33,7 +33,7 @@ : +- CometFilter (20) : +- CometScan parquet spark_catalog.default.ship_mode (19) +- CometHashAggregate (44) - +- CometColumnarExchange (43) + +- CometExchange (43) +- CometHashAggregate (42) +- CometProject (41) +- CometBroadcastHashJoin (40) @@ -172,9 +172,9 @@ Input [11]: [ws_quantity#4, ws_ext_sales_price#5, ws_net_paid#6, w_warehouse_nam Keys [7]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17] Functions [24]: [partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_ext_sales_price#5 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 1) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 2) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 3) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 4) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 5) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 6) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 7) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 8) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 9) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 10) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 11) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#18 = 12) THEN (ws_net_paid#6 * cast(ws_quantity#4 as decimal(10,0))) ELSE 0.00 END)] -(26) CometColumnarExchange +(26) CometExchange Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [55]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, d_year#17, sum#23, isEmpty#24, sum#25, isEmpty#26, sum#27, isEmpty#28, sum#29, isEmpty#30, sum#31, isEmpty#32, sum#33, isEmpty#34, sum#35, isEmpty#36, sum#37, isEmpty#38, sum#39, isEmpty#40, sum#41, isEmpty#42, sum#43, isEmpty#44, sum#45, isEmpty#46, sum#47, isEmpty#48, sum#49, isEmpty#50, sum#51, isEmpty#52, sum#53, isEmpty#54, sum#55, isEmpty#56, sum#57, isEmpty#58, sum#59, isEmpty#60, sum#61, isEmpty#62, sum#63, isEmpty#64, sum#65, isEmpty#66, sum#67, isEmpty#68, sum#69, isEmpty#70] @@ -246,9 +246,9 @@ Input [11]: [cs_quantity#74, cs_sales_price#75, cs_net_paid_inc_tax#76, w_wareho Keys [7]: [w_warehouse_name#80, w_warehouse_sq_ft#81, w_city#82, w_county#83, w_state#84, w_country#85, d_year#87] Functions [24]: [partial_sum(CASE WHEN (d_moy#88 = 1) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 2) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 3) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 4) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 5) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 6) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 7) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 8) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 9) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 10) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 11) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 12) THEN (cs_sales_price#75 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 1) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 2) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 3) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 4) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 5) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 6) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 7) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 8) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 9) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 10) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 11) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END), partial_sum(CASE WHEN (d_moy#88 = 12) THEN (cs_net_paid_inc_tax#76 * cast(cs_quantity#74 as decimal(10,0))) ELSE 0.00 END)] -(43) CometColumnarExchange +(43) CometExchange Input [55]: [w_warehouse_name#80, w_warehouse_sq_ft#81, w_city#82, w_county#83, w_state#84, w_country#85, d_year#87, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] -Arguments: hashpartitioning(w_warehouse_name#80, w_warehouse_sq_ft#81, w_city#82, w_county#83, w_state#84, w_country#85, d_year#87, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(w_warehouse_name#80, w_warehouse_sq_ft#81, w_city#82, w_county#83, w_state#84, w_country#85, d_year#87, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (44) CometHashAggregate Input [55]: [w_warehouse_name#80, w_warehouse_sq_ft#81, w_city#82, w_county#83, w_state#84, w_country#85, d_year#87, sum#91, isEmpty#92, sum#93, isEmpty#94, sum#95, isEmpty#96, sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108, sum#109, isEmpty#110, sum#111, isEmpty#112, sum#113, isEmpty#114, sum#115, isEmpty#116, sum#117, isEmpty#118, sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124, sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130, sum#131, isEmpty#132, sum#133, isEmpty#134, sum#135, isEmpty#136, sum#137, isEmpty#138] @@ -264,9 +264,9 @@ Input [32]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, Keys [8]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#139, year#140] Functions [36]: [partial_sum(jan_sales#141), partial_sum(feb_sales#142), partial_sum(mar_sales#143), partial_sum(apr_sales#144), partial_sum(may_sales#145), partial_sum(jun_sales#146), partial_sum(jul_sales#147), partial_sum(aug_sales#148), partial_sum(sep_sales#149), partial_sum(oct_sales#150), partial_sum(nov_sales#151), partial_sum(dec_sales#152), partial_sum((jan_sales#141 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((feb_sales#142 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((mar_sales#143 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((apr_sales#144 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((may_sales#145 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jun_sales#146 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((jul_sales#147 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((aug_sales#148 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((sep_sales#149 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((oct_sales#150 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((nov_sales#151 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum((dec_sales#152 / cast(w_warehouse_sq_ft#11 as decimal(10,0)))), partial_sum(jan_net#153), partial_sum(feb_net#154), partial_sum(mar_net#155), partial_sum(apr_net#156), partial_sum(may_net#157), partial_sum(jun_net#158), partial_sum(jul_net#159), partial_sum(aug_net#160), partial_sum(sep_net#161), partial_sum(oct_net#162), partial_sum(nov_net#163), partial_sum(dec_net#164)] -(47) CometColumnarExchange +(47) CometExchange Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#139, year#140, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262] -Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#139, year#140, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#139, year#140, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (48) CometHashAggregate Input [80]: [w_warehouse_name#10, w_warehouse_sq_ft#11, w_city#12, w_county#13, w_state#14, w_country#15, ship_carriers#139, year#140, sum#191, isEmpty#192, sum#193, isEmpty#194, sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200, sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206, sum#207, isEmpty#208, sum#209, isEmpty#210, sum#211, isEmpty#212, sum#213, isEmpty#214, sum#215, isEmpty#216, sum#217, isEmpty#218, sum#219, isEmpty#220, sum#221, isEmpty#222, sum#223, isEmpty#224, sum#225, isEmpty#226, sum#227, isEmpty#228, sum#229, isEmpty#230, sum#231, isEmpty#232, sum#233, isEmpty#234, sum#235, isEmpty#236, sum#237, isEmpty#238, sum#239, isEmpty#240, sum#241, isEmpty#242, sum#243, isEmpty#244, sum#245, isEmpty#246, sum#247, isEmpty#248, sum#249, isEmpty#250, sum#251, isEmpty#252, sum#253, isEmpty#254, sum#255, isEmpty#256, sum#257, isEmpty#258, sum#259, isEmpty#260, sum#261, isEmpty#262] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt index ce4354bc5..a2b5963e1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q66/simplified.txt @@ -3,11 +3,11 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_sales_per_sq_foot,feb_sales_per_sq_foot,mar_sales_per_sq_foot,apr_sales_per_sq_foot,may_sales_per_sq_foot,jun_sales_per_sq_foot,jul_sales_per_sq_foot,aug_sales_per_sq_foot,sep_sales_per_sq_foot,oct_sales_per_sq_foot,nov_sales_per_sq_foot,dec_sales_per_sq_foot,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(jan_sales),sum(feb_sales),sum(mar_sales),sum(apr_sales),sum(may_sales),sum(jun_sales),sum(jul_sales),sum(aug_sales),sum(sep_sales),sum(oct_sales),sum(nov_sales),sum(dec_sales),sum((jan_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((feb_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((mar_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((apr_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((may_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jun_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((jul_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((aug_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((sep_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((oct_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((nov_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum((dec_sales / cast(w_warehouse_sq_ft as decimal(10,0)))),sum(jan_net),sum(feb_net),sum(mar_net),sum(apr_net),sum(may_net),sum(jun_net),sum(jul_net),sum(aug_net),sum(sep_net),sum(oct_net),sum(nov_net),sum(dec_net)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year] #1 CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] CometUnion [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net] CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_moy = 1) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_ext_sales_price * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (ws_net_paid * cast(ws_quantity as decimal(10,0))) ELSE 0.00 END)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #2 CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,d_moy,ws_ext_sales_price,ws_quantity,ws_net_paid] CometProject [ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] CometBroadcastHashJoin [ws_ship_mode_sk,ws_quantity,ws_ext_sales_price,ws_net_paid,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] @@ -41,7 +41,7 @@ WholeStageCodegen (1) CometFilter [sm_ship_mode_sk,sm_carrier] CometScan parquet spark_catalog.default.ship_mode [sm_ship_mode_sk,sm_carrier] CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,ship_carriers,year,jan_sales,feb_sales,mar_sales,apr_sales,may_sales,jun_sales,jul_sales,aug_sales,sep_sales,oct_sales,nov_sales,dec_sales,jan_net,feb_net,mar_net,apr_net,may_net,jun_net,jul_net,aug_net,sep_net,oct_net,nov_net,dec_net,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum(CASE WHEN (d_moy = 1) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_sales_price * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 1) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 2) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 3) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 4) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 5) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 6) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 7) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 8) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 9) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 10) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 11) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END),sum(CASE WHEN (d_moy = 12) THEN (cs_net_paid_inc_tax * cast(cs_quantity as decimal(10,0))) ELSE 0.00 END)] - CometColumnarExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #8 + CometExchange [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year] #8 CometHashAggregate [w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,d_moy,cs_sales_price,cs_quantity,cs_net_paid_inc_tax] CometProject [cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy] CometBroadcastHashJoin [cs_ship_mode_sk,cs_quantity,cs_sales_price,cs_net_paid_inc_tax,w_warehouse_name,w_warehouse_sq_ft,w_city,w_county,w_state,w_country,d_year,d_moy,sm_ship_mode_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt index e121b160b..5645438fc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (28) +- Window (26) +- * ColumnarToRow (25) +- CometSort (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) - +- CometColumnarExchange (21) + +- CometExchange (21) +- CometHashAggregate (20) +- CometExpand (19) +- CometProject (18) @@ -126,18 +126,18 @@ Input [11]: [ss_quantity#3, ss_sales_price#4, i_category#19, i_class#20, i_brand Keys [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(21) CometColumnarExchange +(21) CometExchange Input [11]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate Input [11]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27, sum#28, isEmpty#29] Keys [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, spark_grouping_id#27] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(23) CometColumnarExchange +(23) CometExchange Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30] -Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (24) CometSort Input [9]: [i_category#19, i_class#20, i_brand#21, i_product_name#22, d_year#23, d_qoy#24, d_moy#25, s_store_id#26, sumsales#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt index d54aa1fcb..4825ab898 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q67/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ ColumnarToRow InputAdapter CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometColumnarExchange [i_category] #1 + CometExchange [i_category] #1 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,spark_grouping_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] #2 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id,sum,isEmpty,ss_sales_price,ss_quantity] CometExpand [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,spark_grouping_id] CometProject [ss_quantity,ss_sales_price,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt index c07b9bcb9..12a223631 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/explain.txt @@ -6,7 +6,7 @@ :- CometProject (33) : +- CometBroadcastHashJoin (32) : :- CometHashAggregate (28) - : : +- CometColumnarExchange (27) + : : +- CometExchange (27) : : +- CometHashAggregate (26) : : +- CometProject (25) : : +- CometBroadcastHashJoin (24) @@ -164,9 +164,9 @@ Input [7]: [ss_customer_sk#1, ss_addr_sk#3, ss_ticket_number#5, ss_ext_sales_pri Keys [4]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20] Functions [3]: [partial_sum(UnscaledValue(ss_ext_sales_price#6)), partial_sum(UnscaledValue(ss_ext_list_price#7)), partial_sum(UnscaledValue(ss_ext_tax#8))] -(27) CometColumnarExchange +(27) CometExchange Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] -Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate Input [7]: [ss_ticket_number#5, ss_customer_sk#1, ss_addr_sk#3, ca_city#20, sum#21, sum#22, sum#23] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt index 426f7c489..568fd734b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q68/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometProject [ss_ticket_number,bought_city,extended_price,list_price,extended_tax,c_current_addr_sk,c_first_name,c_last_name] CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name] CometHashAggregate [ss_ticket_number,ss_customer_sk,bought_city,extended_price,list_price,extended_tax,ss_addr_sk,ca_city,sum,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_ext_list_price)),sum(UnscaledValue(ss_ext_tax))] - CometColumnarExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 + CometExchange [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city] #1 CometHashAggregate [ss_ticket_number,ss_customer_sk,ss_addr_sk,ca_city,sum,sum,sum,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax] CometProject [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_city] CometBroadcastHashJoin [ss_customer_sk,ss_addr_sk,ss_ticket_number,ss_ext_sales_price,ss_ext_list_price,ss_ext_tax,ca_address_sk,ca_city] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt index 4991d569f..b931de292 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/explain.txt @@ -1,50 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (27) - : : +- * BroadcastHashJoin LeftAnti BuildRight (26) - : : :- * BroadcastHashJoin LeftAnti BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometProject (30) - : +- CometFilter (29) - : +- CometScan parquet spark_catalog.default.customer_address (28) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- Exchange (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (27) + : : +- * BroadcastHashJoin LeftAnti BuildRight (26) + : : :- * BroadcastHashJoin LeftAnti BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * ColumnarToRow (31) + : +- CometProject (30) + : +- CometFilter (29) + : +- CometScan parquet spark_catalog.default.customer_address (28) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometFilter (36) + +- CometScan parquet spark_catalog.default.customer_demographics (35) (1) Scan parquet spark_catalog.default.customer @@ -240,56 +238,50 @@ Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#26] Results [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] -(42) RowToColumnar +(42) Exchange Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] +Arguments: hashpartitioning(cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) CometColumnarExchange -Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] -Arguments: hashpartitioning(cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 6] -Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] - -(45) HashAggregate [codegen id : 6] +(43) HashAggregate [codegen id : 6] Input [6]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25, count#27] Keys [5]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cd_purchase_estimate#24, cd_credit_rating#25] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#28] Results [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, count(1)#28 AS cnt1#29, cd_purchase_estimate#24, count(1)#28 AS cnt2#30, cd_credit_rating#25, count(1)#28 AS cnt3#31] -(46) TakeOrderedAndProject +(44) TakeOrderedAndProject Input [8]: [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#29, cd_purchase_estimate#24, cnt2#30, cd_credit_rating#25, cnt3#31] Arguments: 100, [cd_gender#21 ASC NULLS FIRST, cd_marital_status#22 ASC NULLS FIRST, cd_education_status#23 ASC NULLS FIRST, cd_purchase_estimate#24 ASC NULLS FIRST, cd_credit_rating#25 ASC NULLS FIRST], [cd_gender#21, cd_marital_status#22, cd_education_status#23, cnt1#29, cd_purchase_estimate#24, cnt2#30, cd_credit_rating#25, cnt3#31] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (51) -+- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.date_dim (47) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan parquet spark_catalog.default.date_dim (45) -(47) Scan parquet spark_catalog.default.date_dim +(45) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_moy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), GreaterThanOrEqual(d_moy,4), LessThanOrEqual(d_moy,6), IsNotNull(d_date_sk)] ReadSchema: struct -(48) CometFilter +(46) CometFilter Input [3]: [d_date_sk#7, d_year#8, d_moy#9] Condition : (((((isnotnull(d_year#8) AND isnotnull(d_moy#9)) AND (d_year#8 = 2001)) AND (d_moy#9 >= 4)) AND (d_moy#9 <= 6)) AND isnotnull(d_date_sk#7)) -(49) CometProject +(47) CometProject Input [3]: [d_date_sk#7, d_year#8, d_moy#9] Arguments: [d_date_sk#7], [d_date_sk#7] -(50) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#7] -(51) BroadcastExchange +(49) BroadcastExchange Input [1]: [d_date_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt index 9e1e3edb6..c9cc4959b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q69/simplified.txt @@ -1,72 +1,70 @@ TakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cnt1,cnt2,cnt3] WholeStageCodegen (6) HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,count] [count(1),cnt1,cnt2,cnt3,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] - Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] - ColumnarToRow - InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + InputAdapter + Exchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] #1 + WholeStageCodegen (5) + HashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] [count,count] + Project [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt index c8d74de3d..1f5e1338e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (30) +- CometTakeOrderedAndProject (29) +- CometHashAggregate (28) - +- CometColumnarExchange (27) + +- CometExchange (27) +- CometHashAggregate (26) +- CometProject (25) +- CometBroadcastHashJoin (24) @@ -156,9 +156,9 @@ Input [5]: [ss_quantity#4, ss_list_price#5, ss_sales_price#6, ss_coupon_amt#7, i Keys [1]: [i_item_id#17] Functions [4]: [partial_avg(ss_quantity#4), partial_avg(UnscaledValue(ss_list_price#5)), partial_avg(UnscaledValue(ss_coupon_amt#7)), partial_avg(UnscaledValue(ss_sales_price#6))] -(27) CometColumnarExchange +(27) CometExchange Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] -Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (28) CometHashAggregate Input [9]: [i_item_id#17, sum#21, count#22, sum#23, count#24, sum#25, count#26, sum#27, count#28] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt index 0725862ca..044813ad4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q7/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,agg1,agg2,agg3,agg4] CometHashAggregate [i_item_id,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(ss_quantity),avg(UnscaledValue(ss_list_price)),avg(UnscaledValue(ss_coupon_amt)),avg(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] CometProject [ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id] CometBroadcastHashJoin [ss_promo_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_id,p_promo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt index 1f56b1034..36bc9403c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/explain.txt @@ -1,52 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (48) -+- * Project (47) - +- Window (46) - +- * ColumnarToRow (45) - +- CometSort (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * ColumnarToRow (40) - +- CometColumnarExchange (39) - +- RowToColumnar (38) - +- * HashAggregate (37) - +- * Expand (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (9) - : +- CometProject (8) - : +- CometBroadcastHashJoin (7) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.store_sales (1) - : +- CometBroadcastExchange (6) - : +- CometProject (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.date_dim (3) - +- BroadcastExchange (33) - +- * BroadcastHashJoin LeftSemi BuildRight (32) - :- * ColumnarToRow (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.store (10) - +- BroadcastExchange (31) - +- * Project (30) - +- * Filter (29) - +- Window (28) - +- * ColumnarToRow (27) - +- CometSort (26) - +- CometHashAggregate (25) - +- CometColumnarExchange (24) - +- CometHashAggregate (23) - +- CometProject (22) - +- CometBroadcastHashJoin (21) - :- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.store_sales (13) - : +- CometBroadcastExchange (17) - : +- CometFilter (16) - : +- CometScan parquet spark_catalog.default.store (15) - +- ReusedExchange (20) +TakeOrderedAndProject (44) ++- * Project (43) + +- Window (42) + +- * Sort (41) + +- Exchange (40) + +- * HashAggregate (39) + +- Exchange (38) + +- * HashAggregate (37) + +- * Expand (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (9) + : +- CometProject (8) + : +- CometBroadcastHashJoin (7) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (6) + : +- CometProject (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.date_dim (3) + +- BroadcastExchange (33) + +- * BroadcastHashJoin LeftSemi BuildRight (32) + :- * ColumnarToRow (12) + : +- CometFilter (11) + : +- CometScan parquet spark_catalog.default.store (10) + +- BroadcastExchange (31) + +- * Project (30) + +- * Filter (29) + +- Window (28) + +- * ColumnarToRow (27) + +- CometSort (26) + +- CometHashAggregate (25) + +- CometExchange (24) + +- CometHashAggregate (23) + +- CometProject (22) + +- CometBroadcastHashJoin (21) + :- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.store_sales (13) + : +- CometBroadcastExchange (17) + : +- CometFilter (16) + : +- CometScan parquet spark_catalog.default.store (15) + +- ReusedExchange (20) (1) Scan parquet spark_catalog.default.store_sales @@ -159,9 +155,9 @@ Input [2]: [ss_net_profit#11, s_state#15] Keys [1]: [s_state#15] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] -(24) CometColumnarExchange +(24) CometExchange Input [2]: [s_state#15, sum#17] -Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (25) CometHashAggregate Input [2]: [s_state#15, sum#17] @@ -222,78 +218,66 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum#23] Results [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -(38) RowToColumnar +(38) Exchange Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] +Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(39) CometColumnarExchange -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] -Arguments: hashpartitioning(s_state#20, s_county#21, spark_grouping_id#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(40) ColumnarToRow [codegen id : 5] -Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] - -(41) HashAggregate [codegen id : 5] +(39) HashAggregate [codegen id : 5] Input [4]: [s_state#20, s_county#21, spark_grouping_id#22, sum#24] Keys [3]: [s_state#20, s_county#21, spark_grouping_id#22] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#25] Results [7]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS total_sum#26, s_state#20, s_county#21, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS lochierarchy#27, MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#25,17,2) AS _w0#28, (cast((shiftright(spark_grouping_id#22, 1) & 1) as tinyint) + cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint)) AS _w1#29, CASE WHEN (cast((shiftright(spark_grouping_id#22, 0) & 1) as tinyint) = 0) THEN s_state#20 END AS _w2#30] -(42) RowToColumnar -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] - -(43) CometColumnarExchange -Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) CometSort +(40) Exchange Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] -Arguments: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30], [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST] +Arguments: hashpartitioning(_w1#29, _w2#30, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(45) ColumnarToRow [codegen id : 6] +(41) Sort [codegen id : 6] Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] +Arguments: [_w1#29 ASC NULLS FIRST, _w2#30 ASC NULLS FIRST, _w0#28 DESC NULLS LAST], false, 0 -(46) Window +(42) Window Input [7]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30] Arguments: [rank(_w0#28) windowspecdefinition(_w1#29, _w2#30, _w0#28 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#31], [_w1#29, _w2#30], [_w0#28 DESC NULLS LAST] -(47) Project [codegen id : 7] +(43) Project [codegen id : 7] Output [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] Input [8]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, _w0#28, _w1#29, _w2#30, rank_within_parent#31] -(48) TakeOrderedAndProject +(44) TakeOrderedAndProject Input [5]: [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] Arguments: 100, [lochierarchy#27 DESC NULLS LAST, CASE WHEN (lochierarchy#27 = 0) THEN s_state#20 END ASC NULLS FIRST, rank_within_parent#31 ASC NULLS FIRST], [total_sum#26, s_state#20, s_county#21, lochierarchy#27, rank_within_parent#31] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (53) -+- * ColumnarToRow (52) - +- CometProject (51) - +- CometFilter (50) - +- CometScan parquet spark_catalog.default.date_dim (49) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan parquet spark_catalog.default.date_dim (45) -(49) Scan parquet spark_catalog.default.date_dim +(45) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(50) CometFilter +(46) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) -(51) CometProject +(47) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(52) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(53) BroadcastExchange +(49) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt index 81e584f75..704213148 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q70/simplified.txt @@ -4,70 +4,66 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_count InputAdapter Window [_w0,_w1,_w2] WholeStageCodegen (6) - ColumnarToRow + Sort [_w1,_w2,_w0] InputAdapter - CometSort [total_sum,s_state,s_county,lochierarchy,_w0,_w1,_w2] - CometColumnarExchange [_w1,_w2] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_state,s_county,spark_grouping_id] #2 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] - Expand [ss_net_profit,s_state,s_county] - Project [ss_net_profit,s_state,s_county] - BroadcastHashJoin [ss_store_sk,s_store_sk] + Exchange [_w1,_w2] #1 + WholeStageCodegen (5) + HashAggregate [s_state,s_county,spark_grouping_id,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,lochierarchy,_w0,_w1,_w2,sum] + InputAdapter + Exchange [s_state,s_county,spark_grouping_id] #2 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,spark_grouping_id,ss_net_profit] [sum,sum] + Expand [ss_net_profit,s_state,s_county] + Project [ss_net_profit,s_state,s_county] + BroadcastHashJoin [ss_store_sk,s_store_sk] + ColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (3) + BroadcastHashJoin [s_state,s_state] ColumnarToRow InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometFilter [s_store_sk,s_county,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow + BroadcastExchange #6 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - Project [s_state] - Filter [ranking] + Window [_w0,s_state] + WholeStageCodegen (1) + ColumnarToRow InputAdapter - Window [_w0,s_state] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [s_state,_w0] - CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] - CometColumnarExchange [s_state] #7 - CometHashAggregate [s_state,sum,ss_net_profit] - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #8 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #4 + CometSort [s_state,_w0] + CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #7 + CometHashAggregate [s_state,sum,ss_net_profit] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [s_store_sk,s_state] #8 + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt index 53691edc5..65ffab7e8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/explain.txt @@ -3,7 +3,7 @@ +- CometSort (36) +- CometColumnarExchange (35) +- CometHashAggregate (34) - +- CometColumnarExchange (33) + +- CometExchange (33) +- CometHashAggregate (32) +- CometProject (31) +- CometBroadcastHashJoin (30) @@ -192,9 +192,9 @@ Input [5]: [i_brand_id#2, i_brand#3, ext_price#13, t_hour#35, t_minute#36] Keys [4]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36] Functions [1]: [partial_sum(UnscaledValue(ext_price#13))] -(33) CometColumnarExchange +(33) CometExchange Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#38] -Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (34) CometHashAggregate Input [5]: [i_brand#3, i_brand_id#2, t_hour#35, t_minute#36, sum#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt index cd0700c28..88ca8b67d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q71/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometSort [brand_id,brand,t_hour,t_minute,ext_price] CometColumnarExchange [ext_price,brand_id] #1 CometHashAggregate [brand_id,brand,t_hour,t_minute,ext_price,i_brand,i_brand_id,sum,sum(UnscaledValue(ext_price))] - CometColumnarExchange [i_brand,i_brand_id,t_hour,t_minute] #2 + CometExchange [i_brand,i_brand_id,t_hour,t_minute] #2 CometHashAggregate [i_brand,i_brand_id,t_hour,t_minute,sum,ext_price] CometProject [i_brand_id,i_brand,ext_price,t_hour,t_minute] CometBroadcastHashJoin [i_brand_id,i_brand,ext_price,time_sk,t_time_sk,t_hour,t_minute] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt index 2f6da7d32..f53b2f84e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/explain.txt @@ -1,72 +1,71 @@ == Physical Plan == -* ColumnarToRow (68) -+- CometTakeOrderedAndProject (67) - +- CometHashAggregate (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometProject (63) - +- CometSortMergeJoin (62) - :- CometSort (56) - : +- CometColumnarExchange (55) - : +- RowToColumnar (54) - : +- * Project (53) - : +- * BroadcastHashJoin LeftOuter BuildRight (52) - : :- * Project (47) - : : +- * BroadcastHashJoin Inner BuildRight (46) - : : :- * ColumnarToRow (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometProject (35) - : : : : +- CometBroadcastHashJoin (34) - : : : : :- CometProject (29) - : : : : : +- CometBroadcastHashJoin (28) - : : : : : :- CometProject (23) - : : : : : : +- CometBroadcastHashJoin (22) - : : : : : : :- CometProject (17) - : : : : : : : +- CometBroadcastHashJoin (16) - : : : : : : : :- CometProject (12) - : : : : : : : : +- CometBroadcastHashJoin (11) - : : : : : : : : :- CometProject (7) - : : : : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- CometBroadcastExchange (5) - : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) - : : : : : : : : +- CometBroadcastExchange (10) - : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) - : : : : : : : +- CometBroadcastExchange (15) - : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan parquet spark_catalog.default.item (13) - : : : : : : +- CometBroadcastExchange (21) - : : : : : : +- CometProject (20) - : : : : : : +- CometFilter (19) - : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) - : : : : : +- CometBroadcastExchange (27) - : : : : : +- CometProject (26) - : : : : : +- CometFilter (25) - : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) - : : : : +- CometBroadcastExchange (33) - : : : : +- CometProject (32) - : : : : +- CometFilter (31) - : : : : +- CometScan parquet spark_catalog.default.date_dim (30) - : : : +- CometBroadcastExchange (38) - : : : +- CometFilter (37) - : : : +- CometScan parquet spark_catalog.default.date_dim (36) - : : +- BroadcastExchange (45) - : : +- * ColumnarToRow (44) - : : +- CometFilter (43) - : : +- CometScan parquet spark_catalog.default.date_dim (42) - : +- BroadcastExchange (51) - : +- * ColumnarToRow (50) - : +- CometFilter (49) - : +- CometScan parquet spark_catalog.default.promotion (48) - +- CometSort (61) - +- CometColumnarExchange (60) - +- CometProject (59) - +- CometFilter (58) - +- CometScan parquet spark_catalog.default.catalog_returns (57) +TakeOrderedAndProject (67) ++- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * SortMergeJoin LeftOuter (62) + :- * Sort (55) + : +- Exchange (54) + : +- * Project (53) + : +- * BroadcastHashJoin LeftOuter BuildRight (52) + : :- * Project (47) + : : +- * BroadcastHashJoin Inner BuildRight (46) + : : :- * ColumnarToRow (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometScan parquet spark_catalog.default.date_dim (36) + : : +- BroadcastExchange (45) + : : +- * ColumnarToRow (44) + : : +- CometFilter (43) + : : +- CometScan parquet spark_catalog.default.date_dim (42) + : +- BroadcastExchange (51) + : +- * ColumnarToRow (50) + : +- CometFilter (49) + : +- CometScan parquet spark_catalog.default.promotion (48) + +- * ColumnarToRow (61) + +- CometSort (60) + +- CometExchange (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.catalog_returns (56) (1) Scan parquet spark_catalog.default.catalog_sales @@ -321,99 +320,101 @@ Join condition: None Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -(54) RowToColumnar +(54) Exchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(55) CometColumnarExchange +(55) Sort [codegen id : 4] Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 -(56) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] - -(57) Scan parquet spark_catalog.default.catalog_returns +(56) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(58) CometFilter +(57) CometFilter Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) -(59) CometProject +(58) CometProject Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] -(60) CometColumnarExchange +(59) CometExchange Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(61) CometSort +(60) CometSort Input [2]: [cr_item_sk#31, cr_order_number#32] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] -(62) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +(61) ColumnarToRow [codegen id : 5] +Input [2]: [cr_item_sk#31, cr_order_number#32] -(63) CometProject +(62) SortMergeJoin [codegen id : 6] +Left keys [2]: [cs_item_sk#4, cs_order_number#6] +Right keys [2]: [cr_item_sk#31, cr_order_number#32] +Join type: LeftOuter +Join condition: None + +(63) Project [codegen id : 6] +Output [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(64) CometHashAggregate +(64) HashAggregate [codegen id : 6] Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#34] +Results [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] -(65) CometColumnarExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +(65) Exchange +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] +Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(66) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] +(66) HashAggregate [codegen id : 7] +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#36] +Results [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count(1)#36 AS no_promo#37, count(1)#36 AS promo#38, count(1)#36 AS total_cnt#39] -(67) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -(68) ColumnarToRow [codegen id : 4] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +(67) TakeOrderedAndProject +Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] +Arguments: 100, [total_cnt#39 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (73) -+- * ColumnarToRow (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan parquet spark_catalog.default.date_dim (69) +BroadcastExchange (72) ++- * ColumnarToRow (71) + +- CometProject (70) + +- CometFilter (69) + +- CometScan parquet spark_catalog.default.date_dim (68) -(69) Scan parquet spark_catalog.default.date_dim +(68) Scan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(70) CometFilter +(69) CometFilter Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 1999)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) -(71) CometProject +(70) CometProject Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] -(72) ColumnarToRow [codegen id : 1] +(71) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -(73) BroadcastExchange +(72) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt index 1022e1a90..1efc2e5d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q72/simplified.txt @@ -1,86 +1,92 @@ -WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] - CometColumnarExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] - CometProject [w_warehouse_name,i_item_desc,d_week_seq] - CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometColumnarExchange [cs_item_sk,cs_order_number] #2 - RowToColumnar - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - ColumnarToRow - InputAdapter - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 - CometFilter [d_date_sk,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] +TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] + WholeStageCodegen (7) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] + InputAdapter + Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + WholeStageCodegen (6) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] + Project [w_warehouse_name,i_item_desc,d_week_seq] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter + WholeStageCodegen (4) + Sort [cs_item_sk,cs_order_number] + InputAdapter + Exchange [cs_item_sk,cs_order_number] #2 + WholeStageCodegen (3) + Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_promo_sk,p_promo_sk] + Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] + ColumnarToRow + InputAdapter + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #7 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #8 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometFilter [d_date_sk,d_week_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #11 - WholeStageCodegen (1) + BroadcastExchange #12 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] - CometSort [cr_item_sk,cr_order_number] - CometColumnarExchange [cr_item_sk,cr_order_number] #13 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometFilter [p_promo_sk] + CometScan parquet spark_catalog.default.promotion [p_promo_sk] + InputAdapter + WholeStageCodegen (5) + ColumnarToRow + InputAdapter + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #13 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt index 6ea426978..8b4e36c96 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/explain.txt @@ -6,7 +6,7 @@ +- CometBroadcastHashJoin (28) :- CometFilter (24) : +- CometHashAggregate (23) - : +- CometColumnarExchange (22) + : +- CometExchange (22) : +- CometHashAggregate (21) : +- CometProject (20) : +- CometBroadcastHashJoin (19) @@ -134,9 +134,9 @@ Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt index 5fb8a197a..441736532 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q73/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometColumnarExchange [ss_ticket_number,ss_customer_sk] #2 + CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt index ecc0fd1ca..b23b0b48c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/explain.txt @@ -8,7 +8,7 @@ : :- CometBroadcastHashJoin (33) : : :- CometFilter (16) : : : +- CometHashAggregate (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometHashAggregate (13) : : : +- CometProject (12) : : : +- CometBroadcastHashJoin (11) @@ -24,7 +24,7 @@ : : : +- CometScan parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometProject (28) : : +- CometBroadcastHashJoin (27) @@ -41,7 +41,7 @@ : +- CometBroadcastExchange (48) : +- CometFilter (47) : +- CometHashAggregate (46) - : +- CometColumnarExchange (45) + : +- CometExchange (45) : +- CometHashAggregate (44) : +- CometProject (43) : +- CometBroadcastHashJoin (42) @@ -55,7 +55,7 @@ : +- ReusedExchange (41) +- CometBroadcastExchange (64) +- CometHashAggregate (63) - +- CometColumnarExchange (62) + +- CometExchange (62) +- CometHashAggregate (61) +- CometProject (60) +- CometBroadcastHashJoin (59) @@ -134,9 +134,9 @@ Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_yea Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -(14) CometColumnarExchange +(14) CometExchange Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#11] @@ -212,9 +212,9 @@ Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, ss_net_paid#19, d Keys [4]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#19))] -(30) CometColumnarExchange +(30) CometExchange Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, sum#24] -Arguments: hashpartitioning(c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, sum#24] @@ -283,9 +283,9 @@ Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ws_net_paid#34, d Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#34))] -(45) CometColumnarExchange +(45) CometExchange Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, sum#39] -Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometHashAggregate Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, sum#39] @@ -362,9 +362,9 @@ Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, ws_net_paid#47, d Keys [4]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#47))] -(62) CometColumnarExchange +(62) CometExchange Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, sum#52] -Arguments: hashpartitioning(c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (63) CometHashAggregate Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, sum#52] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt index 0b288df94..631a82f1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q74/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -32,7 +32,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -56,7 +56,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -71,7 +71,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt index ca9a5cd59..76b1cac0c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/explain.txt @@ -4,19 +4,19 @@ TakeOrderedAndProject (125) +- * SortMergeJoin Inner (123) :- * ColumnarToRow (67) : +- CometSort (66) - : +- CometColumnarExchange (65) + : +- CometExchange (65) : +- CometFilter (64) : +- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometHashAggregate (60) - : +- CometColumnarExchange (59) + : +- CometExchange (59) : +- CometHashAggregate (58) : +- CometUnion (57) : :- CometProject (22) : : +- CometSortMergeJoin (21) : : :- CometSort (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometProject (13) : : : +- CometBroadcastHashJoin (12) : : : :- CometProject (8) @@ -31,14 +31,14 @@ TakeOrderedAndProject (125) : : : +- CometFilter (10) : : : +- CometScan parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) : : +- CometScan parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) - : : : +- CometColumnarExchange (31) + : : : +- CometExchange (31) : : : +- CometProject (30) : : : +- CometBroadcastHashJoin (29) : : : :- CometProject (27) @@ -48,14 +48,14 @@ TakeOrderedAndProject (125) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) - : : +- CometColumnarExchange (36) + : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) : : +- CometScan parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) - : : +- CometColumnarExchange (48) + : : +- CometExchange (48) : : +- CometProject (47) : : +- CometBroadcastHashJoin (46) : : :- CometProject (44) @@ -65,25 +65,25 @@ TakeOrderedAndProject (125) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) - : +- CometColumnarExchange (53) + : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) : +- CometScan parquet spark_catalog.default.web_returns (50) +- * ColumnarToRow (122) +- CometSort (121) - +- CometColumnarExchange (120) + +- CometExchange (120) +- CometFilter (119) +- CometHashAggregate (118) - +- CometColumnarExchange (117) + +- CometExchange (117) +- CometHashAggregate (116) +- CometHashAggregate (115) - +- CometColumnarExchange (114) + +- CometExchange (114) +- CometHashAggregate (113) +- CometUnion (112) :- CometProject (83) : +- CometSortMergeJoin (82) : :- CometSort (79) - : : +- CometColumnarExchange (78) + : : +- CometExchange (78) : : +- CometProject (77) : : +- CometBroadcastHashJoin (76) : : :- CometProject (72) @@ -99,7 +99,7 @@ TakeOrderedAndProject (125) :- CometProject (97) : +- CometSortMergeJoin (96) : :- CometSort (93) - : : +- CometColumnarExchange (92) + : : +- CometExchange (92) : : +- CometProject (91) : : +- CometBroadcastHashJoin (90) : : :- CometProject (88) @@ -113,7 +113,7 @@ TakeOrderedAndProject (125) +- CometProject (111) +- CometSortMergeJoin (110) :- CometSort (107) - : +- CometColumnarExchange (106) + : +- CometExchange (106) : +- CometProject (105) : +- CometBroadcastHashJoin (104) : :- CometProject (102) @@ -190,9 +190,9 @@ Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -(14) CometColumnarExchange +(14) CometExchange Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] @@ -213,9 +213,9 @@ Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -(19) CometColumnarExchange +(19) CometExchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -266,9 +266,9 @@ Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -(31) CometColumnarExchange +(31) CometExchange Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] @@ -289,9 +289,9 @@ Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -(36) CometColumnarExchange +(36) CometExchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -342,9 +342,9 @@ Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -(48) CometColumnarExchange +(48) CometExchange Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] @@ -365,9 +365,9 @@ Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -(53) CometColumnarExchange +(53) CometExchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -392,9 +392,9 @@ Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_ Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] Functions: [] -(59) CometColumnarExchange +(59) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] @@ -406,9 +406,9 @@ Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_ Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -(62) CometColumnarExchange +(62) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (63) CometHashAggregate Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] @@ -419,9 +419,9 @@ Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] Condition : isnotnull(sales_cnt#64) -(65) CometColumnarExchange +(65) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (66) CometSort Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] @@ -478,9 +478,9 @@ Arguments: [cs_sold_date_sk#70], [d_date_sk#77], Inner, BuildRight Input [11]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -(78) CometColumnarExchange +(78) CometExchange Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (79) CometSort Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] @@ -538,9 +538,9 @@ Arguments: [ss_sold_date_sk#87], [d_date_sk#94], Inner, BuildRight Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_date_sk#94, d_year#95] Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -(92) CometColumnarExchange +(92) CometExchange Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (93) CometSort Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] @@ -598,9 +598,9 @@ Arguments: [ws_sold_date_sk#104], [d_date_sk#111], Inner, BuildRight Input [11]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -(106) CometColumnarExchange +(106) CometExchange Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (107) CometSort Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] @@ -632,9 +632,9 @@ Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufac Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] Functions: [] -(114) CometColumnarExchange +(114) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (115) CometHashAggregate Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] @@ -646,9 +646,9 @@ Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufac Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -(117) CometColumnarExchange +(117) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] +Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (118) CometHashAggregate Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] @@ -659,9 +659,9 @@ Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] Condition : isnotnull(sales_cnt#118) -(120) CometColumnarExchange +(120) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] +Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (121) CometSort Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt index a942e8071..4ae82c2cf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q75/simplified.txt @@ -7,19 +7,19 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #4 + CometExchange [cs_order_number,cs_item_sk] #4 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -41,14 +41,14 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometColumnarExchange [cr_order_number,cr_item_sk] #8 + CometExchange [cr_order_number,cr_item_sk] #8 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #9 + CometExchange [ss_ticket_number,ss_item_sk] #9 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -59,14 +59,14 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 ReusedExchange [d_date_sk,d_year] #7 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #10 + CometExchange [sr_ticket_number,sr_item_sk] #10 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #11 + CometExchange [ws_order_number,ws_item_sk] #11 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -77,7 +77,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 ReusedExchange [d_date_sk,d_year] #7 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometColumnarExchange [wr_order_number,wr_item_sk] #12 + CometExchange [wr_order_number,wr_item_sk] #12 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] @@ -86,19 +86,19 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat ColumnarToRow InputAdapter CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #16 + CometExchange [cs_order_number,cs_item_sk] #16 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -121,7 +121,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #19 + CometExchange [ss_ticket_number,ss_item_sk] #19 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -136,7 +136,7 @@ TakeOrderedAndProject [sales_cnt_diff,prev_year,year,i_brand_id,i_class_id,i_cat CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #20 + CometExchange [ws_order_number,ws_item_sk] #20 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/explain.txt index d1e7938f7..b60b7b3a2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (34) +- CometTakeOrderedAndProject (33) +- CometHashAggregate (32) - +- CometColumnarExchange (31) + +- CometExchange (31) +- CometHashAggregate (30) +- CometUnion (29) :- CometProject (12) @@ -177,9 +177,9 @@ Input [6]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, ext_sales_ Keys [5]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6] Functions [2]: [partial_count(1), partial_sum(UnscaledValue(ext_sales_price#12))] -(31) CometColumnarExchange +(31) CometExchange Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#37, sum#38] -Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [7]: [channel#10, col_name#11, d_year#8, d_qoy#9, i_category#6, count#37, sum#38] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt index a3a571fa1..eb3e14d5e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q76/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt] CometHashAggregate [channel,col_name,d_year,d_qoy,i_category,sales_cnt,sales_amt,count,sum,count(1),sum(UnscaledValue(ext_sales_price))] - CometColumnarExchange [channel,col_name,d_year,d_qoy,i_category] #1 + CometExchange [channel,col_name,d_year,d_qoy,i_category] #1 CometHashAggregate [channel,col_name,d_year,d_qoy,i_category,count,sum,ext_sales_price] CometUnion [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] CometProject [ss_store_sk,ss_ext_sales_price] [channel,col_name,d_year,d_qoy,i_category,ext_sales_price] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt index 2c2016542..b160070d9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/explain.txt @@ -1,90 +1,88 @@ == Physical Plan == -TakeOrderedAndProject (86) -+- * HashAggregate (85) - +- * ColumnarToRow (84) - +- CometColumnarExchange (83) - +- RowToColumnar (82) - +- * HashAggregate (81) - +- * Expand (80) - +- Union (79) - :- * ColumnarToRow (31) - : +- CometProject (30) - : +- CometBroadcastHashJoin (29) - : :- CometHashAggregate (16) - : : +- CometColumnarExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) - : +- CometBroadcastExchange (28) - : +- CometHashAggregate (27) - : +- CometColumnarExchange (26) - : +- CometHashAggregate (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (21) - : : +- CometBroadcastHashJoin (20) - : : :- CometFilter (18) - : : : +- CometScan parquet spark_catalog.default.store_returns (17) - : : +- ReusedExchange (19) - : +- ReusedExchange (22) - :- * Project (50) - : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) - : :- BroadcastExchange (40) - : : +- * ColumnarToRow (39) - : : +- CometHashAggregate (38) - : : +- CometColumnarExchange (37) - : : +- CometHashAggregate (36) - : : +- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometScan parquet spark_catalog.default.catalog_sales (32) - : : +- ReusedExchange (33) - : +- * ColumnarToRow (48) - : +- CometHashAggregate (47) - : +- CometColumnarExchange (46) - : +- CometHashAggregate (45) - : +- CometProject (44) - : +- CometBroadcastHashJoin (43) - : :- CometScan parquet spark_catalog.default.catalog_returns (41) - : +- ReusedExchange (42) - +- * ColumnarToRow (78) - +- CometProject (77) - +- CometBroadcastHashJoin (76) - :- CometHashAggregate (63) - : +- CometColumnarExchange (62) - : +- CometHashAggregate (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometFilter (52) - : : : +- CometScan parquet spark_catalog.default.web_sales (51) - : : +- ReusedExchange (53) - : +- CometBroadcastExchange (58) - : +- CometFilter (57) - : +- CometScan parquet spark_catalog.default.web_page (56) - +- CometBroadcastExchange (75) - +- CometHashAggregate (74) - +- CometColumnarExchange (73) - +- CometHashAggregate (72) - +- CometProject (71) - +- CometBroadcastHashJoin (70) - :- CometProject (68) - : +- CometBroadcastHashJoin (67) - : :- CometFilter (65) - : : +- CometScan parquet spark_catalog.default.web_returns (64) - : +- ReusedExchange (66) - +- ReusedExchange (69) +TakeOrderedAndProject (84) ++- * HashAggregate (83) + +- Exchange (82) + +- * HashAggregate (81) + +- * Expand (80) + +- Union (79) + :- * ColumnarToRow (31) + : +- CometProject (30) + : +- CometBroadcastHashJoin (29) + : :- CometHashAggregate (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (28) + : +- CometHashAggregate (27) + : +- CometExchange (26) + : +- CometHashAggregate (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (21) + : : +- CometBroadcastHashJoin (20) + : : :- CometFilter (18) + : : : +- CometScan parquet spark_catalog.default.store_returns (17) + : : +- ReusedExchange (19) + : +- ReusedExchange (22) + :- * Project (50) + : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : :- BroadcastExchange (40) + : : +- * ColumnarToRow (39) + : : +- CometHashAggregate (38) + : : +- CometExchange (37) + : : +- CometHashAggregate (36) + : : +- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometScan parquet spark_catalog.default.catalog_sales (32) + : : +- ReusedExchange (33) + : +- * ColumnarToRow (48) + : +- CometHashAggregate (47) + : +- CometExchange (46) + : +- CometHashAggregate (45) + : +- CometProject (44) + : +- CometBroadcastHashJoin (43) + : :- CometScan parquet spark_catalog.default.catalog_returns (41) + : +- ReusedExchange (42) + +- * ColumnarToRow (78) + +- CometProject (77) + +- CometBroadcastHashJoin (76) + :- CometHashAggregate (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometFilter (52) + : : : +- CometScan parquet spark_catalog.default.web_sales (51) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometScan parquet spark_catalog.default.web_page (56) + +- CometBroadcastExchange (75) + +- CometHashAggregate (74) + +- CometExchange (73) + +- CometHashAggregate (72) + +- CometProject (71) + +- CometBroadcastHashJoin (70) + :- CometProject (68) + : +- CometBroadcastHashJoin (67) + : :- CometFilter (65) + : : +- CometScan parquet spark_catalog.default.web_returns (64) + : +- ReusedExchange (66) + +- ReusedExchange (69) (1) Scan parquet spark_catalog.default.store_sales @@ -156,9 +154,9 @@ Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] Keys [1]: [s_store_sk#8] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [s_store_sk#8, sum#9, sum#10] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [s_store_sk#8, sum#9, sum#10] @@ -206,9 +204,9 @@ Input [3]: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] Keys [1]: [s_store_sk#17] Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#12)), partial_sum(UnscaledValue(sr_net_loss#13))] -(26) CometColumnarExchange +(26) CometExchange Input [3]: [s_store_sk#17, sum#18, sum#19] -Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate Input [3]: [s_store_sk#17, sum#18, sum#19] @@ -255,9 +253,9 @@ Input [3]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] Keys [1]: [cs_call_center_sk#28] Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#29)), partial_sum(UnscaledValue(cs_net_profit#30))] -(37) CometColumnarExchange +(37) CometExchange Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometHashAggregate Input [3]: [cs_call_center_sk#28, sum#34, sum#35] @@ -295,9 +293,9 @@ Input [2]: [cr_return_amount#38, cr_net_loss#39] Keys: [] Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#38)), partial_sum(UnscaledValue(cr_net_loss#39))] -(46) CometColumnarExchange +(46) CometExchange Input [2]: [sum#43, sum#44] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (47) CometHashAggregate Input [2]: [sum#43, sum#44] @@ -368,9 +366,9 @@ Input [3]: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] Keys [1]: [wp_web_page_sk#56] Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#51)), partial_sum(UnscaledValue(ws_net_profit#52))] -(62) CometColumnarExchange +(62) CometExchange Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (63) CometHashAggregate Input [3]: [wp_web_page_sk#56, sum#57, sum#58] @@ -418,9 +416,9 @@ Input [3]: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] Keys [1]: [wp_web_page_sk#65] Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#60)), partial_sum(UnscaledValue(wr_net_loss#61))] -(73) CometColumnarExchange +(73) CometExchange Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (74) CometHashAggregate Input [3]: [wp_web_page_sk#65, sum#66, sum#67] @@ -456,56 +454,50 @@ Functions [3]: [partial_sum(sales#22), partial_sum(returns#24), partial_sum(prof Aggregate Attributes [6]: [sum#79, isEmpty#80, sum#81, isEmpty#82, sum#83, isEmpty#84] Results [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] -(82) RowToColumnar +(82) Exchange Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] +Arguments: hashpartitioning(channel#76, id#77, spark_grouping_id#78, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(83) CometColumnarExchange -Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] -Arguments: hashpartitioning(channel#76, id#77, spark_grouping_id#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(84) ColumnarToRow [codegen id : 6] -Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] - -(85) HashAggregate [codegen id : 6] +(83) HashAggregate [codegen id : 6] Input [9]: [channel#76, id#77, spark_grouping_id#78, sum#85, isEmpty#86, sum#87, isEmpty#88, sum#89, isEmpty#90] Keys [3]: [channel#76, id#77, spark_grouping_id#78] Functions [3]: [sum(sales#22), sum(returns#24), sum(profit#25)] Aggregate Attributes [3]: [sum(sales#22)#91, sum(returns#24)#92, sum(profit#25)#93] Results [5]: [channel#76, id#77, sum(sales#22)#91 AS sales#94, sum(returns#24)#92 AS returns#95, sum(profit#25)#93 AS profit#96] -(86) TakeOrderedAndProject +(84) TakeOrderedAndProject Input [5]: [channel#76, id#77, sales#94, returns#95, profit#96] Arguments: 100, [channel#76 ASC NULLS FIRST, id#77 ASC NULLS FIRST], [channel#76, id#77, sales#94, returns#95, profit#96] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (91) -+- * ColumnarToRow (90) - +- CometProject (89) - +- CometFilter (88) - +- CometScan parquet spark_catalog.default.date_dim (87) +BroadcastExchange (89) ++- * ColumnarToRow (88) + +- CometProject (87) + +- CometFilter (86) + +- CometScan parquet spark_catalog.default.date_dim (85) -(87) Scan parquet spark_catalog.default.date_dim +(85) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_date#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,2000-08-03), LessThanOrEqual(d_date,2000-09-02), IsNotNull(d_date_sk)] ReadSchema: struct -(88) CometFilter +(86) CometFilter Input [2]: [d_date_sk#6, d_date#7] Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 2000-08-03)) AND (d_date#7 <= 2000-09-02)) AND isnotnull(d_date_sk#6)) -(89) CometProject +(87) CometProject Input [2]: [d_date_sk#6, d_date#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(90) ColumnarToRow [codegen id : 1] +(88) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(91) BroadcastExchange +(89) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt index 605180016..e8226ab1f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q77/simplified.txt @@ -1,112 +1,110 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (6) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) + InputAdapter + Exchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (5) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [returns,profit,profit_loss,s_store_sk] [sales,returns,profit,channel,id] + CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] + CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_sk] #2 + CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #5 + CometFilter [s_store_sk] + CometScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 + CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] + CometExchange [s_store_sk] #7 + CometHashAggregate [s_store_sk,sum,sum,sr_return_amt,sr_net_loss] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #5 + WholeStageCodegen (3) + Project [sales,returns,profit,profit_loss,cs_call_center_sk] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] + CometExchange [cs_call_center_sk] #9 + CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 ColumnarToRow InputAdapter - CometProject [returns,profit,profit_loss,s_store_sk] [sales,returns,profit,channel,id] - CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] - CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] - CometColumnarExchange [s_store_sk] #2 - CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #5 - CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #6 - CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - CometColumnarExchange [s_store_sk] #7 - CometHashAggregate [s_store_sk,sum,sum,sr_return_amt,sr_net_loss] - CometProject [sr_return_amt,sr_net_loss,s_store_sk] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] - CometProject [sr_store_sk,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [s_store_sk] #5 - WholeStageCodegen (3) - Project [sales,returns,profit,profit_loss,cs_call_center_sk] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometColumnarExchange [cs_call_center_sk] #9 - CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ColumnarToRow - InputAdapter - CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometColumnarExchange #10 - CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 + CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] + CometExchange #10 + CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] + CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] + CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] + CometExchange [wp_web_page_sk] #11 + CometHashAggregate [wp_web_page_sk,sum,sum,ws_ext_sales_price,ws_net_profit] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 ReusedExchange [d_date_sk] #4 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [returns,profit,profit_loss,wp_web_page_sk] [sales,returns,profit,channel,id] - CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] - CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - CometColumnarExchange [wp_web_page_sk] #11 - CometHashAggregate [wp_web_page_sk,sum,sum,ws_ext_sales_price,ws_net_profit] - CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] - CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [wp_web_page_sk] #12 - CometFilter [wp_web_page_sk] - CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 - CometHashAggregate [wp_web_page_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] - CometColumnarExchange [wp_web_page_sk] #14 - CometHashAggregate [wp_web_page_sk,sum,sum,wr_return_amt,wr_net_loss] - CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] - CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] - CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - ReusedExchange [wp_web_page_sk] #12 + CometBroadcastExchange [wp_web_page_sk] #12 + CometFilter [wp_web_page_sk] + CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #13 + CometHashAggregate [wp_web_page_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] + CometExchange [wp_web_page_sk] #14 + CometHashAggregate [wp_web_page_sk,sum,sum,wr_return_amt,wr_net_loss] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + ReusedExchange [wp_web_page_sk] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt index 2ffeb0737..acd754b08 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q78/explain.txt @@ -7,7 +7,7 @@ TakeOrderedAndProject (67) : +- CometSortMergeJoin (42) : :- CometSort (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -15,11 +15,11 @@ TakeOrderedAndProject (67) : : : +- CometFilter (11) : : : +- CometSortMergeJoin (10) : : : :- CometSort (4) - : : : : +- CometColumnarExchange (3) + : : : : +- CometExchange (3) : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- CometSort (9) - : : : +- CometColumnarExchange (8) + : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) : : : +- CometScan parquet spark_catalog.default.store_returns (5) @@ -29,7 +29,7 @@ TakeOrderedAndProject (67) : +- CometSort (41) : +- CometFilter (40) : +- CometHashAggregate (39) - : +- CometColumnarExchange (38) + : +- CometExchange (38) : +- CometHashAggregate (37) : +- CometProject (36) : +- CometBroadcastHashJoin (35) @@ -37,11 +37,11 @@ TakeOrderedAndProject (67) : : +- CometFilter (32) : : +- CometSortMergeJoin (31) : : :- CometSort (25) - : : : +- CometColumnarExchange (24) + : : : +- CometExchange (24) : : : +- CometFilter (23) : : : +- CometScan parquet spark_catalog.default.web_sales (22) : : +- CometSort (30) - : : +- CometColumnarExchange (29) + : : +- CometExchange (29) : : +- CometProject (28) : : +- CometFilter (27) : : +- CometScan parquet spark_catalog.default.web_returns (26) @@ -49,7 +49,7 @@ TakeOrderedAndProject (67) +- CometSort (63) +- CometFilter (62) +- CometHashAggregate (61) - +- CometColumnarExchange (60) + +- CometExchange (60) +- CometHashAggregate (59) +- CometProject (58) +- CometBroadcastHashJoin (57) @@ -57,11 +57,11 @@ TakeOrderedAndProject (67) : +- CometFilter (54) : +- CometSortMergeJoin (53) : :- CometSort (47) - : : +- CometColumnarExchange (46) + : : +- CometExchange (46) : : +- CometFilter (45) : : +- CometScan parquet spark_catalog.default.catalog_sales (44) : +- CometSort (52) - : +- CometColumnarExchange (51) + : +- CometExchange (51) : +- CometProject (50) : +- CometFilter (49) : +- CometScan parquet spark_catalog.default.catalog_returns (48) @@ -80,9 +80,9 @@ ReadSchema: struct -(106) CometFilter +(104) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 2000-08-23)) AND (d_date#15 <= 2000-09-22)) AND isnotnull(d_date_sk#14)) -(107) CometProject +(105) CometProject Input [2]: [d_date_sk#14, d_date#15] Arguments: [d_date_sk#14], [d_date_sk#14] -(108) ColumnarToRow [codegen id : 1] +(106) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(109) BroadcastExchange +(107) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt index dfcc35f6e..012c711bf 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q80/simplified.txt @@ -1,124 +1,122 @@ TakeOrderedAndProject [channel,id,sales,returns,profit] WholeStageCodegen (5) HashAggregate [channel,id,spark_grouping_id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - Expand [sales,returns,profit,channel,id] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_id] #2 - CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk,ss_ticket_number] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometColumnarExchange [sr_item_sk,sr_ticket_number] #5 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #7 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #8 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #9 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [cp_catalog_page_id] #10 - CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometColumnarExchange [cs_item_sk,cs_order_number] #11 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometColumnarExchange [cr_item_sk,cr_order_number] #12 - CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [web_site_id] #14 - CometHashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] - CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] - CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] - CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometColumnarExchange [ws_item_sk,ws_order_number] #15 - CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometColumnarExchange [wr_item_sk,wr_order_number] #16 - CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - CometBroadcastExchange [web_site_sk,web_site_id] #17 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #8 - ReusedExchange [p_promo_sk] #9 + InputAdapter + Exchange [channel,id,spark_grouping_id] #1 + WholeStageCodegen (4) + HashAggregate [channel,id,spark_grouping_id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + Expand [sales,returns,profit,channel,id] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [s_store_id] #2 + CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #5 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #7 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #8 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #9 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #10 + CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #11 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #12 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #13 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),sales,returns,profit,channel,id,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [web_site_id] #14 + CometHashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #15 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #16 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + CometBroadcastExchange [web_site_sk,web_site_id] #17 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #8 + ReusedExchange [p_promo_sk] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt index 97abada79..8be7d0b3e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/explain.txt @@ -9,7 +9,7 @@ : : +- CometBroadcastHashJoin (34) : : :- CometFilter (17) : : : +- CometHashAggregate (16) - : : : +- CometColumnarExchange (15) + : : : +- CometExchange (15) : : : +- CometHashAggregate (14) : : : +- CometProject (13) : : : +- CometBroadcastHashJoin (12) @@ -27,10 +27,10 @@ : : +- CometBroadcastExchange (33) : : +- CometFilter (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometHashAggregate (28) - : : +- CometColumnarExchange (27) + : : +- CometExchange (27) : : +- CometHashAggregate (26) : : +- CometProject (25) : : +- CometBroadcastHashJoin (24) @@ -117,9 +117,9 @@ Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#10] @@ -171,9 +171,9 @@ Input [3]: [cr_returning_customer_sk#1, cr_return_amt_inc_tax#3, ca_state#9] Keys [2]: [cr_returning_customer_sk#1, ca_state#9] Functions [1]: [partial_sum(UnscaledValue(cr_return_amt_inc_tax#3))] -(27) CometColumnarExchange +(27) CometExchange Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#15] -Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cr_returning_customer_sk#1, ca_state#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [3]: [cr_returning_customer_sk#1, ca_state#9, sum#15] @@ -185,9 +185,9 @@ Input [2]: [ctr_state#12, ctr_total_return#13] Keys [1]: [ctr_state#12] Functions [1]: [partial_avg(ctr_total_return#13)] -(30) CometColumnarExchange +(30) CometExchange Input [3]: [ctr_state#12, sum#16, count#17] -Arguments: hashpartitioning(ctr_state#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ctr_state#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (31) CometHashAggregate Input [3]: [ctr_state#12, sum#16, count#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt index 62f7d5462..b5d2c8321 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q81/simplified.txt @@ -10,7 +10,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ctr_customer_sk,ctr_state,ctr_total_return,(avg(ctr_total_return) * 1.2),ctr_state] CometFilter [ctr_customer_sk,ctr_state,ctr_total_return] CometHashAggregate [ctr_customer_sk,ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(cr_return_amt_inc_tax))] - CometColumnarExchange [cr_returning_customer_sk,ca_state] #1 + CometExchange [cr_returning_customer_sk,ca_state] #1 CometHashAggregate [cr_returning_customer_sk,ca_state,sum,cr_return_amt_inc_tax] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] @@ -36,10 +36,10 @@ WholeStageCodegen (1) CometBroadcastExchange [(avg(ctr_total_return) * 1.2),ctr_state] #5 CometFilter [(avg(ctr_total_return) * 1.2),ctr_state] CometHashAggregate [(avg(ctr_total_return) * 1.2),ctr_state,ctr_state,sum,count,avg(ctr_total_return)] - CometColumnarExchange [ctr_state] #6 + CometExchange [ctr_state] #6 CometHashAggregate [ctr_state,sum,count,ctr_total_return] CometHashAggregate [ctr_state,ctr_total_return,cr_returning_customer_sk,ca_state,sum,sum(UnscaledValue(cr_return_amt_inc_tax))] - CometColumnarExchange [cr_returning_customer_sk,ca_state] #7 + CometExchange [cr_returning_customer_sk,ca_state] #7 CometHashAggregate [cr_returning_customer_sk,ca_state,sum,cr_return_amt_inc_tax] CometProject [cr_returning_customer_sk,cr_return_amt_inc_tax,ca_state] CometBroadcastHashJoin [cr_returning_customer_sk,cr_returning_addr_sk,cr_return_amt_inc_tax,ca_address_sk,ca_state] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt index 2519fbc97..774b6b112 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (26) +- CometTakeOrderedAndProject (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometBroadcastHashJoin (20) @@ -132,9 +132,9 @@ Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Keys [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] Functions: [] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] -Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#2, i_item_desc#3, i_current_price#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [3]: [i_item_id#2, i_item_desc#3, i_current_price#4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt index 869073de1..42c3f40e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q82/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [i_item_id,i_item_desc,i_current_price] CometHashAggregate [i_item_id,i_item_desc,i_current_price] - CometColumnarExchange [i_item_id,i_item_desc,i_current_price] #1 + CometExchange [i_item_id,i_item_desc,i_current_price] #1 CometHashAggregate [i_item_id,i_item_desc,i_current_price] CometProject [i_item_id,i_item_desc,i_current_price] CometBroadcastHashJoin [i_item_sk,i_item_id,i_item_desc,i_current_price,ss_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt index 972a58976..ffd8a5eb1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/explain.txt @@ -6,7 +6,7 @@ :- CometProject (32) : +- CometBroadcastHashJoin (31) : :- CometHashAggregate (18) - : : +- CometColumnarExchange (17) + : : +- CometExchange (17) : : +- CometHashAggregate (16) : : +- CometProject (15) : : +- CometBroadcastHashJoin (14) @@ -25,7 +25,7 @@ : : +- ReusedExchange (10) : +- CometBroadcastExchange (30) : +- CometHashAggregate (29) - : +- CometColumnarExchange (28) + : +- CometExchange (28) : +- CometHashAggregate (27) : +- CometProject (26) : +- CometBroadcastHashJoin (25) @@ -37,7 +37,7 @@ : +- ReusedExchange (24) +- CometBroadcastExchange (44) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometProject (40) +- CometBroadcastHashJoin (39) @@ -126,9 +126,9 @@ Input [2]: [sr_return_quantity#2, i_item_id#6] Keys [1]: [i_item_id#6] Functions [1]: [partial_sum(sr_return_quantity#2)] -(17) CometColumnarExchange +(17) CometExchange Input [2]: [i_item_id#6, sum#10] -Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [2]: [i_item_id#6, sum#10] @@ -176,9 +176,9 @@ Input [2]: [cr_return_quantity#12, i_item_id#16] Keys [1]: [i_item_id#16] Functions [1]: [partial_sum(cr_return_quantity#12)] -(28) CometColumnarExchange +(28) CometExchange Input [2]: [i_item_id#16, sum#18] -Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate Input [2]: [i_item_id#16, sum#18] @@ -239,9 +239,9 @@ Input [2]: [wr_return_quantity#24, i_item_id#28] Keys [1]: [i_item_id#28] Functions [1]: [partial_sum(wr_return_quantity#24)] -(42) CometColumnarExchange +(42) CometExchange Input [2]: [i_item_id#28, sum#30] -Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (43) CometHashAggregate Input [2]: [i_item_id#28, sum#30] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt index 8365f11a4..77345d28a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q83/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometProject [item_id,sr_item_qty,cr_item_qty] CometBroadcastHashJoin [item_id,sr_item_qty,item_id,cr_item_qty] CometHashAggregate [item_id,sr_item_qty,i_item_id,sum,sum(sr_return_quantity)] - CometColumnarExchange [i_item_id] #1 + CometExchange [i_item_id] #1 CometHashAggregate [i_item_id,sum,sr_return_quantity] CometProject [sr_return_quantity,i_item_id] CometBroadcastHashJoin [sr_return_quantity,sr_returned_date_sk,i_item_id,d_date_sk] @@ -43,7 +43,7 @@ WholeStageCodegen (1) ReusedExchange [d_date] #3 CometBroadcastExchange [item_id,cr_item_qty] #7 CometHashAggregate [item_id,cr_item_qty,i_item_id,sum,sum(cr_return_quantity)] - CometColumnarExchange [i_item_id] #8 + CometExchange [i_item_id] #8 CometHashAggregate [i_item_id,sum,cr_return_quantity] CometProject [cr_return_quantity,i_item_id] CometBroadcastHashJoin [cr_return_quantity,cr_returned_date_sk,i_item_id,d_date_sk] @@ -56,7 +56,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk] #6 CometBroadcastExchange [item_id,wr_item_qty] #9 CometHashAggregate [item_id,wr_item_qty,i_item_id,sum,sum(wr_return_quantity)] - CometColumnarExchange [i_item_id] #10 + CometExchange [i_item_id] #10 CometHashAggregate [i_item_id,sum,wr_return_quantity] CometProject [wr_return_quantity,i_item_id] CometBroadcastHashJoin [wr_return_quantity,wr_returned_date_sk,i_item_id,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt index 968c7402e..07aa05d94 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (45) +- CometTakeOrderedAndProject (44) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometProject (40) +- CometBroadcastHashJoin (39) @@ -243,9 +243,9 @@ Input [4]: [ws_quantity#4, wr_fee#15, wr_refunded_cash#16, r_reason_desc#31] Keys [1]: [r_reason_desc#31] Functions [3]: [partial_avg(ws_quantity#4), partial_avg(UnscaledValue(wr_refunded_cash#16)), partial_avg(UnscaledValue(wr_fee#15))] -(42) CometColumnarExchange +(42) CometExchange Input [7]: [r_reason_desc#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] -Arguments: hashpartitioning(r_reason_desc#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(r_reason_desc#31, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (43) CometHashAggregate Input [7]: [r_reason_desc#31, sum#32, count#33, sum#34, count#35, sum#36, count#37] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt index 9467b5c77..4e4ece1bd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q85/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee)] CometHashAggregate [substr(r_reason_desc, 1, 20),avg(ws_quantity),avg(wr_refunded_cash),avg(wr_fee),r_reason_desc,sum,count,sum,count,sum,count,avg(ws_quantity),avg(UnscaledValue(wr_refunded_cash)),avg(UnscaledValue(wr_fee))] - CometColumnarExchange [r_reason_desc] #1 + CometExchange [r_reason_desc] #1 CometHashAggregate [r_reason_desc,sum,count,sum,count,sum,count,ws_quantity,wr_refunded_cash,wr_fee] CometProject [ws_quantity,wr_fee,wr_refunded_cash,r_reason_desc] CometBroadcastHashJoin [ws_quantity,wr_reason_sk,wr_fee,wr_refunded_cash,r_reason_sk,r_reason_desc] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt index 3d0fe32c7..675e6e931 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (23) +- Window (21) +- * ColumnarToRow (20) +- CometSort (19) - +- CometColumnarExchange (18) + +- CometExchange (18) +- CometHashAggregate (17) - +- CometColumnarExchange (16) + +- CometExchange (16) +- CometHashAggregate (15) +- CometExpand (14) +- CometProject (13) @@ -97,18 +97,18 @@ Input [4]: [ws_net_paid#2, i_category#10, i_class#11, spark_grouping_id#12] Keys [3]: [i_category#10, i_class#11, spark_grouping_id#12] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -(16) CometColumnarExchange +(16) CometExchange Input [4]: [i_category#10, i_class#11, spark_grouping_id#12, sum#13] -Arguments: hashpartitioning(i_category#10, i_class#11, spark_grouping_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#10, i_class#11, spark_grouping_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (17) CometHashAggregate Input [4]: [i_category#10, i_class#11, spark_grouping_id#12, sum#13] Keys [3]: [i_category#10, i_class#11, spark_grouping_id#12] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] -(18) CometColumnarExchange +(18) CometExchange Input [7]: [total_sum#14, i_category#10, i_class#11, lochierarchy#15, _w0#16, _w1#17, _w2#18] -Arguments: hashpartitioning(_w1#17, _w2#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(_w1#17, _w2#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (19) CometSort Input [7]: [total_sum#14, i_category#10, i_class#11, lochierarchy#15, _w0#16, _w1#17, _w2#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt index bbe4a2d2f..86480d58c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q86/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl ColumnarToRow InputAdapter CometSort [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2] - CometColumnarExchange [_w1,_w2] #1 + CometExchange [_w1,_w2] #1 CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,_w1,_w2,spark_grouping_id,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [i_category,i_class,spark_grouping_id] #2 + CometExchange [i_category,i_class,spark_grouping_id] #2 CometHashAggregate [i_category,i_class,spark_grouping_id,sum,ws_net_paid] CometExpand [i_category,i_class] [ws_net_paid,i_category,i_class,spark_grouping_id] CometProject [ws_net_paid,i_category,i_class] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt index d7ea7098e..d023b5b31 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/explain.txt @@ -1,55 +1,53 @@ == Physical Plan == -* HashAggregate (51) -+- * ColumnarToRow (50) - +- CometColumnarExchange (49) - +- RowToColumnar (48) - +- * HashAggregate (47) - +- * Project (46) - +- * BroadcastHashJoin LeftAnti BuildRight (45) - :- * BroadcastHashJoin LeftAnti BuildRight (31) - : :- * ColumnarToRow (17) - : : +- CometHashAggregate (16) - : : +- CometColumnarExchange (15) - : : +- CometHashAggregate (14) - : : +- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.customer (9) - : +- BroadcastExchange (30) - : +- * ColumnarToRow (29) - : +- CometHashAggregate (28) - : +- CometColumnarExchange (27) - : +- CometHashAggregate (26) - : +- CometProject (25) - : +- CometBroadcastHashJoin (24) - : :- CometProject (22) - : : +- CometBroadcastHashJoin (21) - : : :- CometFilter (19) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) - : : +- ReusedExchange (20) - : +- ReusedExchange (23) - +- BroadcastExchange (44) - +- * ColumnarToRow (43) - +- CometHashAggregate (42) - +- CometColumnarExchange (41) - +- CometHashAggregate (40) - +- CometProject (39) - +- CometBroadcastHashJoin (38) - :- CometProject (36) - : +- CometBroadcastHashJoin (35) - : :- CometFilter (33) - : : +- CometScan parquet spark_catalog.default.web_sales (32) - : +- ReusedExchange (34) - +- ReusedExchange (37) +* HashAggregate (49) ++- Exchange (48) + +- * HashAggregate (47) + +- * Project (46) + +- * BroadcastHashJoin LeftAnti BuildRight (45) + :- * BroadcastHashJoin LeftAnti BuildRight (31) + : :- * ColumnarToRow (17) + : : +- CometHashAggregate (16) + : : +- CometExchange (15) + : : +- CometHashAggregate (14) + : : +- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.customer (9) + : +- BroadcastExchange (30) + : +- * ColumnarToRow (29) + : +- CometHashAggregate (28) + : +- CometExchange (27) + : +- CometHashAggregate (26) + : +- CometProject (25) + : +- CometBroadcastHashJoin (24) + : :- CometProject (22) + : : +- CometBroadcastHashJoin (21) + : : :- CometFilter (19) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (18) + : : +- ReusedExchange (20) + : +- ReusedExchange (23) + +- BroadcastExchange (44) + +- * ColumnarToRow (43) + +- CometHashAggregate (42) + +- CometExchange (41) + +- CometHashAggregate (40) + +- CometProject (39) + +- CometBroadcastHashJoin (38) + :- CometProject (36) + : +- CometBroadcastHashJoin (35) + : :- CometFilter (33) + : : +- CometScan parquet spark_catalog.default.web_sales (32) + : +- ReusedExchange (34) + +- ReusedExchange (37) (1) Scan parquet spark_catalog.default.store_sales @@ -121,9 +119,9 @@ Input [3]: [c_last_name#9, c_first_name#8, d_date#5] Keys [3]: [c_last_name#9, c_first_name#8, d_date#5] Functions: [] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [c_last_name#9, c_first_name#8, d_date#5] -Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_last_name#9, c_first_name#8, d_date#5, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [c_last_name#9, c_first_name#8, d_date#5] @@ -174,9 +172,9 @@ Input [3]: [c_last_name#17, c_first_name#16, d_date#14] Keys [3]: [c_last_name#17, c_first_name#16, d_date#14] Functions: [] -(27) CometColumnarExchange +(27) CometExchange Input [3]: [c_last_name#17, c_first_name#16, d_date#14] -Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_last_name#17, c_first_name#16, d_date#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [3]: [c_last_name#17, c_first_name#16, d_date#14] @@ -237,9 +235,9 @@ Input [3]: [c_last_name#25, c_first_name#24, d_date#22] Keys [3]: [c_last_name#25, c_first_name#24, d_date#22] Functions: [] -(41) CometColumnarExchange +(41) CometExchange Input [3]: [c_last_name#25, c_first_name#24, d_date#22] -Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_last_name#25, c_first_name#24, d_date#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (42) CometHashAggregate Input [3]: [c_last_name#25, c_first_name#24, d_date#22] @@ -270,17 +268,11 @@ Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#26] Results [1]: [count#27] -(48) RowToColumnar +(48) Exchange Input [1]: [count#27] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(49) CometColumnarExchange -Input [1]: [count#27] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(50) ColumnarToRow [codegen id : 4] -Input [1]: [count#27] - -(51) HashAggregate [codegen id : 4] +(49) HashAggregate [codegen id : 4] Input [1]: [count#27] Keys: [] Functions [1]: [count(1)] @@ -290,32 +282,32 @@ Results [1]: [count(1)#28 AS count(1)#29] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#2 IN dynamicpruning#3 -BroadcastExchange (56) -+- * ColumnarToRow (55) - +- CometProject (54) - +- CometFilter (53) - +- CometScan parquet spark_catalog.default.date_dim (52) +BroadcastExchange (54) ++- * ColumnarToRow (53) + +- CometProject (52) + +- CometFilter (51) + +- CometScan parquet spark_catalog.default.date_dim (50) -(52) Scan parquet spark_catalog.default.date_dim +(50) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(53) CometFilter +(51) CometFilter Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#4)) -(54) CometProject +(52) CometProject Input [3]: [d_date_sk#4, d_date#5, d_month_seq#6] Arguments: [d_date_sk#4, d_date#5], [d_date_sk#4, d_date#5] -(55) ColumnarToRow [codegen id : 1] +(53) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#4, d_date#5] -(56) BroadcastExchange +(54) BroadcastExchange Input [2]: [d_date_sk#4, d_date#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt index 69f33a57a..6e3328ae1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q87/simplified.txt @@ -1,71 +1,69 @@ WholeStageCodegen (4) HashAggregate [count] [count(1),count(1),count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [count,count] - Project - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] - BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (3) + HashAggregate [count,count] + Project + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + BroadcastHashJoin [c_last_name,c_first_name,d_date,c_last_name,c_first_name,d_date] + ColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #2 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ss_customer_sk,d_date] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_customer_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [d_date_sk,d_date] #4 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometFilter [c_customer_sk,c_first_name,c_last_name] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometHashAggregate [c_last_name,c_first_name,d_date] + CometExchange [c_last_name,c_first_name,d_date] #7 + CometHashAggregate [c_last_name,c_first_name,d_date] + CometProject [c_last_name,c_first_name,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [cs_bill_customer_sk,d_date] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] + CometFilter [cs_bill_customer_sk,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_date] #4 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #2 + CometExchange [c_last_name,c_first_name,d_date] #9 CometHashAggregate [c_last_name,c_first_name,d_date] CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ss_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ss_customer_sk,d_date] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_customer_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #4 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [c_customer_sk,c_first_name,c_last_name] #5 - CometFilter [c_customer_sk,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_first_name,c_last_name] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #7 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [cs_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [cs_bill_customer_sk,d_date] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_sold_date_sk,d_date_sk,d_date] - CometFilter [cs_bill_customer_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [c_last_name,c_first_name,d_date] - CometColumnarExchange [c_last_name,c_first_name,d_date] #9 - CometHashAggregate [c_last_name,c_first_name,d_date] - CometProject [c_last_name,c_first_name,d_date] - CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] - CometProject [ws_bill_customer_sk,d_date] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_bill_customer_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #4 - ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 + CometBroadcastHashJoin [ws_bill_customer_sk,d_date,c_customer_sk,c_first_name,c_last_name] + CometProject [ws_bill_customer_sk,d_date] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_bill_customer_sk,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_date] #4 + ReusedExchange [c_customer_sk,c_first_name,c_last_name] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt index cd9e4817c..1dface7ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/explain.txt @@ -8,7 +8,7 @@ : : : : : :- * BroadcastNestedLoopJoin Inner BuildRight (46) : : : : : : :- * ColumnarToRow (25) : : : : : : : +- CometHashAggregate (24) -: : : : : : : +- CometColumnarExchange (23) +: : : : : : : +- CometExchange (23) : : : : : : : +- CometHashAggregate (22) : : : : : : : +- CometProject (21) : : : : : : : +- CometBroadcastHashJoin (20) @@ -34,7 +34,7 @@ : : : : : : +- BroadcastExchange (45) : : : : : : +- * ColumnarToRow (44) : : : : : : +- CometHashAggregate (43) -: : : : : : +- CometColumnarExchange (42) +: : : : : : +- CometExchange (42) : : : : : : +- CometHashAggregate (41) : : : : : : +- CometProject (40) : : : : : : +- CometBroadcastHashJoin (39) @@ -54,7 +54,7 @@ : : : : : +- BroadcastExchange (66) : : : : : +- * ColumnarToRow (65) : : : : : +- CometHashAggregate (64) -: : : : : +- CometColumnarExchange (63) +: : : : : +- CometExchange (63) : : : : : +- CometHashAggregate (62) : : : : : +- CometProject (61) : : : : : +- CometBroadcastHashJoin (60) @@ -74,7 +74,7 @@ : : : : +- BroadcastExchange (87) : : : : +- * ColumnarToRow (86) : : : : +- CometHashAggregate (85) -: : : : +- CometColumnarExchange (84) +: : : : +- CometExchange (84) : : : : +- CometHashAggregate (83) : : : : +- CometProject (82) : : : : +- CometBroadcastHashJoin (81) @@ -94,7 +94,7 @@ : : : +- BroadcastExchange (108) : : : +- * ColumnarToRow (107) : : : +- CometHashAggregate (106) -: : : +- CometColumnarExchange (105) +: : : +- CometExchange (105) : : : +- CometHashAggregate (104) : : : +- CometProject (103) : : : +- CometBroadcastHashJoin (102) @@ -114,7 +114,7 @@ : : +- BroadcastExchange (129) : : +- * ColumnarToRow (128) : : +- CometHashAggregate (127) -: : +- CometColumnarExchange (126) +: : +- CometExchange (126) : : +- CometHashAggregate (125) : : +- CometProject (124) : : +- CometBroadcastHashJoin (123) @@ -134,7 +134,7 @@ : +- BroadcastExchange (150) : +- * ColumnarToRow (149) : +- CometHashAggregate (148) -: +- CometColumnarExchange (147) +: +- CometExchange (147) : +- CometHashAggregate (146) : +- CometProject (145) : +- CometBroadcastHashJoin (144) @@ -154,7 +154,7 @@ +- BroadcastExchange (171) +- * ColumnarToRow (170) +- CometHashAggregate (169) - +- CometColumnarExchange (168) + +- CometExchange (168) +- CometHashAggregate (167) +- CometProject (166) +- CometBroadcastHashJoin (165) @@ -276,9 +276,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(23) CometColumnarExchange +(23) CometExchange Input [1]: [count#13] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [1]: [count#13] @@ -359,9 +359,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(42) CometColumnarExchange +(42) CometExchange Input [1]: [count#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (43) CometHashAggregate Input [1]: [count#24] @@ -450,9 +450,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(63) CometColumnarExchange +(63) CometExchange Input [1]: [count#35] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (64) CometHashAggregate Input [1]: [count#35] @@ -541,9 +541,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(84) CometColumnarExchange +(84) CometExchange Input [1]: [count#46] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (85) CometHashAggregate Input [1]: [count#46] @@ -632,9 +632,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(105) CometColumnarExchange +(105) CometExchange Input [1]: [count#57] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (106) CometHashAggregate Input [1]: [count#57] @@ -723,9 +723,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(126) CometColumnarExchange +(126) CometExchange Input [1]: [count#68] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (127) CometHashAggregate Input [1]: [count#68] @@ -814,9 +814,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(147) CometColumnarExchange +(147) CometExchange Input [1]: [count#79] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (148) CometHashAggregate Input [1]: [count#79] @@ -905,9 +905,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(168) CometColumnarExchange +(168) CometExchange Input [1]: [count#90] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (169) CometHashAggregate Input [1]: [count#90] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt index b01685ed0..1bb61b6c9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q88/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h8_30_to_9,count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -38,7 +38,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h9_to_9_30,count,count(1)] - CometColumnarExchange #6 + CometExchange #6 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -61,7 +61,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h9_30_to_10,count,count(1)] - CometColumnarExchange #9 + CometExchange #9 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -84,7 +84,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h10_to_10_30,count,count(1)] - CometColumnarExchange #12 + CometExchange #12 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -107,7 +107,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h10_30_to_11,count,count(1)] - CometColumnarExchange #15 + CometExchange #15 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -130,7 +130,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h11_to_11_30,count,count(1)] - CometColumnarExchange #18 + CometExchange #18 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -153,7 +153,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h11_30_to_12,count,count(1)] - CometColumnarExchange #21 + CometExchange #21 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] @@ -176,7 +176,7 @@ WholeStageCodegen (8) ColumnarToRow InputAdapter CometHashAggregate [h12_to_12_30,count,count(1)] - CometColumnarExchange #24 + CometExchange #24 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt index 6f973cee6..6789e5b18 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/explain.txt @@ -5,9 +5,9 @@ TakeOrderedAndProject (28) +- Window (25) +- * ColumnarToRow (24) +- CometSort (23) - +- CometColumnarExchange (22) + +- CometExchange (22) +- CometHashAggregate (21) - +- CometColumnarExchange (20) + +- CometExchange (20) +- CometHashAggregate (19) +- CometProject (18) +- CometBroadcastHashJoin (17) @@ -122,18 +122,18 @@ Input [7]: [i_brand#2, i_class#3, i_category#4, ss_sales_price#7, d_moy#12, s_st Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#7))] -(20) CometColumnarExchange +(20) CometExchange Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum#16] -Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [7]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum#16] Keys [6]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12] Functions [1]: [sum(UnscaledValue(ss_sales_price#7))] -(22) CometColumnarExchange +(22) CometExchange Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#4, i_brand#2, s_store_name#14, s_company_name#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (23) CometSort Input [8]: [i_category#4, i_class#3, i_brand#2, s_store_name#14, s_company_name#15, d_moy#12, sum_sales#17, _w0#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt index 26c510f5a..97b9563be 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q89/simplified.txt @@ -8,9 +8,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,s_store_name,i_category,i_cla ColumnarToRow InputAdapter CometSort [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 CometHashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 + CometExchange [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy] #2 CometHashAggregate [i_category,i_class,i_brand,s_store_name,s_company_name,d_moy,sum,ss_sales_price] CometProject [i_brand,i_class,i_category,ss_sales_price,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_class,i_category,ss_store_sk,ss_sales_price,d_moy,s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt index 375b036fc..30b6c1e44 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/explain.txt @@ -29,7 +29,7 @@ Subquery:1 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery * ColumnarToRow (12) +- CometProject (11) +- CometHashAggregate (10) - +- CometColumnarExchange (9) + +- CometExchange (9) +- CometHashAggregate (8) +- CometProject (7) +- CometFilter (6) @@ -56,9 +56,9 @@ Input [2]: [ss_ext_discount_amt#18, ss_net_paid#19] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#18)), partial_avg(UnscaledValue(ss_net_paid#19))] -(9) CometColumnarExchange +(9) CometExchange Input [5]: [count#21, sum#22, count#23, sum#24, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometHashAggregate Input [5]: [count#21, sum#22, count#23, sum#24, count#25] @@ -80,7 +80,7 @@ Subquery:4 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery * ColumnarToRow (20) +- CometProject (19) +- CometHashAggregate (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometFilter (14) @@ -107,9 +107,9 @@ Input [2]: [ss_ext_discount_amt#31, ss_net_paid#32] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#31)), partial_avg(UnscaledValue(ss_net_paid#32))] -(17) CometColumnarExchange +(17) CometExchange Input [5]: [count#34, sum#35, count#36, sum#37, count#38] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometHashAggregate Input [5]: [count#34, sum#35, count#36, sum#37, count#38] @@ -131,7 +131,7 @@ Subquery:7 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquery * ColumnarToRow (28) +- CometProject (27) +- CometHashAggregate (26) - +- CometColumnarExchange (25) + +- CometExchange (25) +- CometHashAggregate (24) +- CometProject (23) +- CometFilter (22) @@ -158,9 +158,9 @@ Input [2]: [ss_ext_discount_amt#44, ss_net_paid#45] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#44)), partial_avg(UnscaledValue(ss_net_paid#45))] -(25) CometColumnarExchange +(25) CometExchange Input [5]: [count#47, sum#48, count#49, sum#50, count#51] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (26) CometHashAggregate Input [5]: [count#47, sum#48, count#49, sum#50, count#51] @@ -182,7 +182,7 @@ Subquery:10 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer * ColumnarToRow (36) +- CometProject (35) +- CometHashAggregate (34) - +- CometColumnarExchange (33) + +- CometExchange (33) +- CometHashAggregate (32) +- CometProject (31) +- CometFilter (30) @@ -209,9 +209,9 @@ Input [2]: [ss_ext_discount_amt#57, ss_net_paid#58] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#57)), partial_avg(UnscaledValue(ss_net_paid#58))] -(33) CometColumnarExchange +(33) CometExchange Input [5]: [count#60, sum#61, count#62, sum#63, count#64] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (34) CometHashAggregate Input [5]: [count#60, sum#61, count#62, sum#63, count#64] @@ -233,7 +233,7 @@ Subquery:13 Hosting operator id = 4 Hosting Expression = Subquery scalar-subquer * ColumnarToRow (44) +- CometProject (43) +- CometHashAggregate (42) - +- CometColumnarExchange (41) + +- CometExchange (41) +- CometHashAggregate (40) +- CometProject (39) +- CometFilter (38) @@ -260,9 +260,9 @@ Input [2]: [ss_ext_discount_amt#70, ss_net_paid#71] Keys: [] Functions [3]: [partial_count(1), partial_avg(UnscaledValue(ss_ext_discount_amt#70)), partial_avg(UnscaledValue(ss_net_paid#71))] -(41) CometColumnarExchange +(41) CometExchange Input [5]: [count#73, sum#74, count#75, sum#76, count#77] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (42) CometHashAggregate Input [5]: [count#73, sum#74, count#75, sum#76, count#77] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt index e0d03e2ef..3d69c60d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q9/simplified.txt @@ -6,7 +6,7 @@ WholeStageCodegen (1) InputAdapter CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] @@ -19,7 +19,7 @@ WholeStageCodegen (1) InputAdapter CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometColumnarExchange #2 + CometExchange #2 CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] @@ -32,7 +32,7 @@ WholeStageCodegen (1) InputAdapter CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometColumnarExchange #3 + CometExchange #3 CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] @@ -45,7 +45,7 @@ WholeStageCodegen (1) InputAdapter CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometColumnarExchange #4 + CometExchange #4 CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] @@ -58,7 +58,7 @@ WholeStageCodegen (1) InputAdapter CometProject [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid)] [mergedValue] CometHashAggregate [count(1),avg(ss_ext_discount_amt),avg(ss_net_paid),count,sum,count,sum,count,count(1),avg(UnscaledValue(ss_ext_discount_amt)),avg(UnscaledValue(ss_net_paid))] - CometColumnarExchange #5 + CometExchange #5 CometHashAggregate [count,sum,count,sum,count,ss_ext_discount_amt,ss_net_paid] CometProject [ss_ext_discount_amt,ss_net_paid] CometFilter [ss_quantity,ss_ext_discount_amt,ss_net_paid,ss_sold_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt index ab17ec1d0..b55193e7e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/explain.txt @@ -3,7 +3,7 @@ +- * BroadcastNestedLoopJoin Inner BuildRight (46) :- * ColumnarToRow (25) : +- CometHashAggregate (24) - : +- CometColumnarExchange (23) + : +- CometExchange (23) : +- CometHashAggregate (22) : +- CometProject (21) : +- CometBroadcastHashJoin (20) @@ -29,7 +29,7 @@ +- BroadcastExchange (45) +- * ColumnarToRow (44) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometProject (40) +- CometBroadcastHashJoin (39) @@ -151,9 +151,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(23) CometColumnarExchange +(23) CometExchange Input [1]: [count#11] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [1]: [count#11] @@ -234,9 +234,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(42) CometColumnarExchange +(42) CometExchange Input [1]: [count#21] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (43) CometHashAggregate Input [1]: [count#21] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt index 65860a550..95fd73d86 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q90/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [amc,count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] @@ -33,7 +33,7 @@ WholeStageCodegen (2) ColumnarToRow InputAdapter CometHashAggregate [pmc,count,count(1)] - CometColumnarExchange #6 + CometExchange #6 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ws_web_page_sk,wp_web_page_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt index 836e03f4a..aecb46518 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/explain.txt @@ -3,7 +3,7 @@ +- CometSort (40) +- CometColumnarExchange (39) +- CometHashAggregate (38) - +- CometColumnarExchange (37) + +- CometExchange (37) +- CometHashAggregate (36) +- CometProject (35) +- CometBroadcastHashJoin (34) @@ -215,9 +215,9 @@ Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cr_net_loss#7, cd_mari Keys [5]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21] Functions [1]: [partial_sum(UnscaledValue(cr_net_loss#7))] -(37) CometColumnarExchange +(37) CometExchange Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, sum#24] -Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (38) CometHashAggregate Input [6]: [cc_call_center_id#2, cc_name#3, cc_manager#4, cd_marital_status#20, cd_education_status#21, sum#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt index ff17b986a..0dc197b1b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q91/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometSort [Call_Center,Call_Center_Name,Manager,Returns_Loss] CometColumnarExchange [Returns_Loss] #1 CometHashAggregate [Call_Center,Call_Center_Name,Manager,Returns_Loss,cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum,sum(UnscaledValue(cr_net_loss))] - CometColumnarExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 + CometExchange [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status] #2 CometHashAggregate [cc_call_center_id,cc_name,cc_manager,cd_marital_status,cd_education_status,sum,cr_net_loss] CometProject [cc_call_center_id,cc_name,cc_manager,cr_net_loss,cd_marital_status,cd_education_status] CometBroadcastHashJoin [cc_call_center_id,cc_name,cc_manager,cr_net_loss,c_current_hdemo_sk,cd_marital_status,cd_education_status,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt index 7d3423272..f3c063f6b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (30) +- CometHashAggregate (29) - +- CometColumnarExchange (28) + +- CometExchange (28) +- CometHashAggregate (27) +- CometProject (26) +- CometBroadcastHashJoin (25) @@ -18,7 +18,7 @@ : +- CometBroadcastExchange (21) : +- CometFilter (20) : +- CometHashAggregate (19) - : +- CometColumnarExchange (18) + : +- CometExchange (18) : +- CometHashAggregate (17) : +- CometProject (16) : +- CometBroadcastHashJoin (15) @@ -116,9 +116,9 @@ Input [2]: [ws_item_sk#7, ws_ext_discount_amt#8] Keys [1]: [ws_item_sk#7] Functions [1]: [partial_avg(UnscaledValue(ws_ext_discount_amt#8))] -(18) CometColumnarExchange +(18) CometExchange Input [3]: [ws_item_sk#7, sum#13, count#14] -Arguments: hashpartitioning(ws_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (19) CometHashAggregate Input [3]: [ws_item_sk#7, sum#13, count#14] @@ -159,9 +159,9 @@ Input [1]: [ws_ext_discount_amt#2] Keys: [] Functions [1]: [partial_sum(UnscaledValue(ws_ext_discount_amt#2))] -(28) CometColumnarExchange +(28) CometExchange Input [1]: [sum#17] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (29) CometHashAggregate Input [1]: [sum#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt index 065e38bb1..f49dd1ea3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q92/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [Excess Discount Amount ,sum,sum(UnscaledValue(ws_ext_discount_amt))] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,ws_ext_discount_amt] CometProject [ws_ext_discount_amt] CometBroadcastHashJoin [ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] @@ -27,7 +27,7 @@ WholeStageCodegen (1) CometBroadcastExchange [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] #4 CometFilter [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk] CometHashAggregate [(1.3 * avg(ws_ext_discount_amt)),ws_item_sk,sum,count,avg(UnscaledValue(ws_ext_discount_amt))] - CometColumnarExchange [ws_item_sk] #5 + CometExchange [ws_item_sk] #5 CometHashAggregate [ws_item_sk,sum,count,ws_ext_discount_amt] CometProject [ws_item_sk,ws_ext_discount_amt] CometBroadcastHashJoin [ws_item_sk,ws_ext_discount_amt,ws_sold_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt index ee3fb07aa..4c672e397 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q93/explain.txt @@ -2,18 +2,18 @@ * ColumnarToRow (22) +- CometTakeOrderedAndProject (21) +- CometHashAggregate (20) - +- CometColumnarExchange (19) + +- CometExchange (19) +- CometHashAggregate (18) +- CometProject (17) +- CometBroadcastHashJoin (16) :- CometProject (11) : +- CometSortMergeJoin (10) : :- CometSort (4) - : : +- CometColumnarExchange (3) + : : +- CometExchange (3) : : +- CometProject (2) : : +- CometScan parquet spark_catalog.default.store_sales (1) : +- CometSort (9) - : +- CometColumnarExchange (8) + : +- CometExchange (8) : +- CometProject (7) : +- CometFilter (6) : +- CometScan parquet spark_catalog.default.store_returns (5) @@ -33,9 +33,9 @@ ReadSchema: struct Input [3]: [ws_warehouse_sk#9, ws_order_number#10, ws_sold_date_sk#11] Arguments: [ws_warehouse_sk#9, ws_order_number#10], [ws_warehouse_sk#9, ws_order_number#10] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [ws_warehouse_sk#9, ws_order_number#10] -Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [ws_warehouse_sk#9, ws_order_number#10] @@ -115,9 +113,9 @@ ReadSchema: struct Input [2]: [wr_order_number#12, wr_returned_date_sk#13] Arguments: [wr_order_number#12], [wr_order_number#12] -(16) CometColumnarExchange +(16) CometExchange Input [1]: [wr_order_number#12] -Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (17) CometSort Input [1]: [wr_order_number#12] @@ -249,17 +247,11 @@ Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#6)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#6))#20, sum(UnscaledValue(ws_net_profit#7))#21, count(ws_order_number#5)#24] Results [3]: [sum#22, sum#23, count#25] -(44) RowToColumnar +(44) Exchange Input [3]: [sum#22, sum#23, count#25] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(45) CometColumnarExchange -Input [3]: [sum#22, sum#23, count#25] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(46) ColumnarToRow [codegen id : 9] -Input [3]: [sum#22, sum#23, count#25] - -(47) HashAggregate [codegen id : 9] +(45) HashAggregate [codegen id : 9] Input [3]: [sum#22, sum#23, count#25] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#6)), sum(UnscaledValue(ws_net_profit#7)), count(distinct ws_order_number#5)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt index 3f324f9f2..16295d42c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q94/simplified.txt @@ -1,70 +1,68 @@ WholeStageCodegen (9) HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (8) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (3) - Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometColumnarExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - CometColumnarExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [wr_order_number] - CometColumnarExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + InputAdapter + Exchange #1 + WholeStageCodegen (8) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_web_site_sk,web_site_sk] + Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] + Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_date_sk,d_date_sk] + SortMergeJoin [ws_order_number,wr_order_number] + InputAdapter + WholeStageCodegen (3) + Project [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_warehouse_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] InputAdapter - BroadcastExchange #6 - WholeStageCodegen (6) + BroadcastExchange #5 + WholeStageCodegen (5) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (7) + BroadcastExchange #6 + WholeStageCodegen (6) ColumnarToRow InputAdapter - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt index e1d16718d..8eb03fe45 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/explain.txt @@ -1,67 +1,65 @@ == Physical Plan == -* HashAggregate (63) -+- * ColumnarToRow (62) - +- CometColumnarExchange (61) - +- RowToColumnar (60) - +- * HashAggregate (59) - +- * HashAggregate (58) - +- * HashAggregate (57) - +- * Project (56) - +- * BroadcastHashJoin Inner BuildRight (55) - :- * Project (49) - : +- * BroadcastHashJoin Inner BuildRight (48) - : :- * Project (42) - : : +- * BroadcastHashJoin Inner BuildRight (41) - : : :- * SortMergeJoin LeftSemi (35) - : : : :- * SortMergeJoin LeftSemi (18) - : : : : :- * ColumnarToRow (6) - : : : : : +- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : : +- * Project (17) - : : : : +- * SortMergeJoin Inner (16) - : : : : :- * ColumnarToRow (12) - : : : : : +- CometSort (11) - : : : : : +- CometColumnarExchange (10) - : : : : : +- CometProject (9) - : : : : : +- CometFilter (8) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (7) - : : : : +- * ColumnarToRow (15) - : : : : +- CometSort (14) - : : : : +- ReusedExchange (13) - : : : +- * Project (34) - : : : +- * SortMergeJoin Inner (33) - : : : :- * ColumnarToRow (24) - : : : : +- CometSort (23) - : : : : +- CometColumnarExchange (22) - : : : : +- CometProject (21) - : : : : +- CometFilter (20) - : : : : +- CometScan parquet spark_catalog.default.web_returns (19) - : : : +- * Project (32) - : : : +- * SortMergeJoin Inner (31) - : : : :- * ColumnarToRow (27) - : : : : +- CometSort (26) - : : : : +- ReusedExchange (25) - : : : +- * ColumnarToRow (30) - : : : +- CometSort (29) - : : : +- ReusedExchange (28) - : : +- BroadcastExchange (40) - : : +- * ColumnarToRow (39) - : : +- CometProject (38) - : : +- CometFilter (37) - : : +- CometScan parquet spark_catalog.default.date_dim (36) - : +- BroadcastExchange (47) - : +- * ColumnarToRow (46) - : +- CometProject (45) - : +- CometFilter (44) - : +- CometScan parquet spark_catalog.default.customer_address (43) - +- BroadcastExchange (54) - +- * ColumnarToRow (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.web_site (50) +* HashAggregate (61) ++- Exchange (60) + +- * HashAggregate (59) + +- * HashAggregate (58) + +- * HashAggregate (57) + +- * Project (56) + +- * BroadcastHashJoin Inner BuildRight (55) + :- * Project (49) + : +- * BroadcastHashJoin Inner BuildRight (48) + : :- * Project (42) + : : +- * BroadcastHashJoin Inner BuildRight (41) + : : :- * SortMergeJoin LeftSemi (35) + : : : :- * SortMergeJoin LeftSemi (18) + : : : : :- * ColumnarToRow (6) + : : : : : +- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : : +- * Project (17) + : : : : +- * SortMergeJoin Inner (16) + : : : : :- * ColumnarToRow (12) + : : : : : +- CometSort (11) + : : : : : +- CometExchange (10) + : : : : : +- CometProject (9) + : : : : : +- CometFilter (8) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (7) + : : : : +- * ColumnarToRow (15) + : : : : +- CometSort (14) + : : : : +- ReusedExchange (13) + : : : +- * Project (34) + : : : +- * SortMergeJoin Inner (33) + : : : :- * ColumnarToRow (24) + : : : : +- CometSort (23) + : : : : +- CometExchange (22) + : : : : +- CometProject (21) + : : : : +- CometFilter (20) + : : : : +- CometScan parquet spark_catalog.default.web_returns (19) + : : : +- * Project (32) + : : : +- * SortMergeJoin Inner (31) + : : : :- * ColumnarToRow (27) + : : : : +- CometSort (26) + : : : : +- ReusedExchange (25) + : : : +- * ColumnarToRow (30) + : : : +- CometSort (29) + : : : +- ReusedExchange (28) + : : +- BroadcastExchange (40) + : : +- * ColumnarToRow (39) + : : +- CometProject (38) + : : +- CometFilter (37) + : : +- CometScan parquet spark_catalog.default.date_dim (36) + : +- BroadcastExchange (47) + : +- * ColumnarToRow (46) + : +- CometProject (45) + : +- CometFilter (44) + : +- CometScan parquet spark_catalog.default.customer_address (43) + +- BroadcastExchange (54) + +- * ColumnarToRow (53) + +- CometProject (52) + +- CometFilter (51) + +- CometScan parquet spark_catalog.default.web_site (50) (1) Scan parquet spark_catalog.default.web_sales @@ -79,9 +77,9 @@ Condition : ((isnotnull(ws_ship_date_sk#1) AND isnotnull(ws_ship_addr_sk#2)) AND Input [7]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6, ws_sold_date_sk#7] Arguments: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6], [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -(4) CometColumnarExchange +(4) CometExchange Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] -Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_order_number#4, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [6]: [ws_ship_date_sk#1, ws_ship_addr_sk#2, ws_web_site_sk#3, ws_order_number#4, ws_ext_ship_cost#5, ws_net_profit#6] @@ -105,9 +103,9 @@ Condition : (isnotnull(ws_order_number#9) AND isnotnull(ws_warehouse_sk#8)) Input [3]: [ws_warehouse_sk#8, ws_order_number#9, ws_sold_date_sk#10] Arguments: [ws_warehouse_sk#8, ws_order_number#9], [ws_warehouse_sk#8, ws_order_number#9] -(10) CometColumnarExchange +(10) CometExchange Input [2]: [ws_warehouse_sk#8, ws_order_number#9] -Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_order_number#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (11) CometSort Input [2]: [ws_warehouse_sk#8, ws_order_number#9] @@ -157,9 +155,9 @@ Condition : isnotnull(wr_order_number#13) Input [2]: [wr_order_number#13, wr_returned_date_sk#14] Arguments: [wr_order_number#13], [wr_order_number#13] -(22) CometColumnarExchange +(22) CometExchange Input [1]: [wr_order_number#13] -Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(wr_order_number#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (23) CometSort Input [1]: [wr_order_number#13] @@ -331,17 +329,11 @@ Functions [3]: [merge_sum(UnscaledValue(ws_ext_ship_cost#5)), merge_sum(Unscaled Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_ship_cost#5))#21, sum(UnscaledValue(ws_net_profit#6))#22, count(ws_order_number#4)#25] Results [3]: [sum#23, sum#24, count#26] -(60) RowToColumnar +(60) Exchange Input [3]: [sum#23, sum#24, count#26] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=7] -(61) CometColumnarExchange -Input [3]: [sum#23, sum#24, count#26] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(62) ColumnarToRow [codegen id : 15] -Input [3]: [sum#23, sum#24, count#26] - -(63) HashAggregate [codegen id : 15] +(61) HashAggregate [codegen id : 15] Input [3]: [sum#23, sum#24, count#26] Keys: [] Functions [3]: [sum(UnscaledValue(ws_ext_ship_cost#5)), sum(UnscaledValue(ws_net_profit#6)), count(distinct ws_order_number#4)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt index 2c736e6e5..5cfa7827a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q95/simplified.txt @@ -1,101 +1,99 @@ WholeStageCodegen (15) HashAggregate [sum,sum,count] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),order count ,total shipping cost ,total net profit ,sum,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #1 - RowToColumnar - WholeStageCodegen (14) - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] - HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] - Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_web_site_sk,web_site_sk] - Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] - Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - BroadcastHashJoin [ws_ship_date_sk,d_date_sk] - SortMergeJoin [ws_order_number,wr_order_number] - InputAdapter - WholeStageCodegen (5) - SortMergeJoin [ws_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometColumnarExchange [ws_order_number] #2 - CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] - CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (4) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] - InputAdapter - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - CometColumnarExchange [ws_order_number] #3 - CometProject [ws_warehouse_sk,ws_order_number] - CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] - InputAdapter - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - WholeStageCodegen (10) - Project [wr_order_number] - SortMergeJoin [wr_order_number,ws_order_number] - InputAdapter - WholeStageCodegen (6) - ColumnarToRow - InputAdapter - CometSort [wr_order_number] - CometColumnarExchange [wr_order_number] #4 - CometProject [wr_order_number] - CometFilter [wr_order_number,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + InputAdapter + Exchange #1 + WholeStageCodegen (14) + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),count(ws_order_number),sum,sum,count,sum,sum,count] + HashAggregate [ws_order_number] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + HashAggregate [ws_order_number,ws_ext_ship_cost,ws_net_profit] [sum(UnscaledValue(ws_ext_ship_cost)),sum(UnscaledValue(ws_net_profit)),sum,sum,sum,sum] + Project [ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_web_site_sk,web_site_sk] + Project [ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_addr_sk,ca_address_sk] + Project [ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + BroadcastHashJoin [ws_ship_date_sk,d_date_sk] + SortMergeJoin [ws_order_number,wr_order_number] + InputAdapter + WholeStageCodegen (5) + SortMergeJoin [ws_order_number,ws_order_number] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow InputAdapter - WholeStageCodegen (9) - Project [ws_order_number] - SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + CometSort [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometExchange [ws_order_number] #2 + CometProject [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit] + CometFilter [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_ship_date_sk,ws_ship_addr_sk,ws_web_site_sk,ws_order_number,ws_ext_ship_cost,ws_net_profit,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (4) + Project [ws_order_number] + SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + InputAdapter + WholeStageCodegen (2) + ColumnarToRow InputAdapter - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + CometSort [ws_warehouse_sk,ws_order_number] + CometExchange [ws_order_number] #3 + CometProject [ws_warehouse_sk,ws_order_number] + CometFilter [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_warehouse_sk,ws_order_number,ws_sold_date_sk] + InputAdapter + WholeStageCodegen (3) + ColumnarToRow InputAdapter - WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometSort [ws_warehouse_sk,ws_order_number] - ReusedExchange [ws_warehouse_sk,ws_order_number] #3 - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + InputAdapter + WholeStageCodegen (10) + Project [wr_order_number] + SortMergeJoin [wr_order_number,ws_order_number] + InputAdapter + WholeStageCodegen (6) + ColumnarToRow + InputAdapter + CometSort [wr_order_number] + CometExchange [wr_order_number] #4 + CometProject [wr_order_number] + CometFilter [wr_order_number,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_order_number,wr_returned_date_sk] + InputAdapter + WholeStageCodegen (9) + Project [ws_order_number] + SortMergeJoin [ws_order_number,ws_order_number,ws_warehouse_sk,ws_warehouse_sk] + InputAdapter + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 + InputAdapter + WholeStageCodegen (8) + ColumnarToRow + InputAdapter + CometSort [ws_warehouse_sk,ws_order_number] + ReusedExchange [ws_warehouse_sk,ws_order_number] #3 InputAdapter - BroadcastExchange #6 - WholeStageCodegen (12) + BroadcastExchange #5 + WholeStageCodegen (11) ColumnarToRow InputAdapter - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (13) + BroadcastExchange #6 + WholeStageCodegen (12) ColumnarToRow InputAdapter - CometProject [web_site_sk] - CometFilter [web_site_sk,web_company_name] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (13) + ColumnarToRow + InputAdapter + CometProject [web_site_sk] + CometFilter [web_site_sk,web_company_name] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt index 6f9d1a3f2..c4b2cf973 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/explain.txt @@ -1,7 +1,7 @@ == Physical Plan == * ColumnarToRow (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometBroadcastHashJoin (20) @@ -129,9 +129,9 @@ Input: [] Keys: [] Functions [1]: [partial_count(1)] -(23) CometColumnarExchange +(23) CometExchange Input [1]: [count#12] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (24) CometHashAggregate Input [1]: [count#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt index b8b3e211d..d9a87aa3c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q96/simplified.txt @@ -2,7 +2,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [count(1),count,count(1)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [count] CometProject CometBroadcastHashJoin [ss_store_sk,s_store_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt index e119edec6..2eec0079d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/explain.txt @@ -1,13 +1,13 @@ == Physical Plan == * ColumnarToRow (25) +- CometHashAggregate (24) - +- CometColumnarExchange (23) + +- CometExchange (23) +- CometHashAggregate (22) +- CometProject (21) +- CometSortMergeJoin (20) :- CometSort (11) : +- CometHashAggregate (10) - : +- CometColumnarExchange (9) + : +- CometExchange (9) : +- CometHashAggregate (8) : +- CometProject (7) : +- CometBroadcastHashJoin (6) @@ -18,7 +18,7 @@ : +- CometScan parquet spark_catalog.default.date_dim (2) +- CometSort (19) +- CometHashAggregate (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) +- CometProject (15) +- CometBroadcastHashJoin (14) @@ -66,9 +66,9 @@ Input [2]: [ss_item_sk#1, ss_customer_sk#2] Keys [2]: [ss_customer_sk#2, ss_item_sk#1] Functions: [] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [ss_customer_sk#2, ss_item_sk#1] -Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_customer_sk#2, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometHashAggregate Input [2]: [ss_customer_sk#2, ss_item_sk#1] @@ -103,9 +103,9 @@ Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] Keys [2]: [cs_bill_customer_sk#9, cs_item_sk#10] Functions: [] -(17) CometColumnarExchange +(17) CometExchange Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] -Arguments: hashpartitioning(cs_bill_customer_sk#9, cs_item_sk#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_bill_customer_sk#9, cs_item_sk#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometHashAggregate Input [2]: [cs_bill_customer_sk#9, cs_item_sk#10] @@ -130,9 +130,9 @@ Input [2]: [customer_sk#7, customer_sk#14] Keys: [] Functions [3]: [partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnull(customer_sk#14)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (isnotnull(customer_sk#7) AND isnotnull(customer_sk#14)) THEN 1 ELSE 0 END)] -(23) CometColumnarExchange +(23) CometExchange Input [3]: [sum#16, sum#17, sum#18] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (24) CometHashAggregate Input [3]: [sum#16, sum#17, sum#18] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt index da4461d84..0036a4bd8 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q97/simplified.txt @@ -2,13 +2,13 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [store_only,catalog_only,store_and_catalog,sum,sum,sum,sum(CASE WHEN (isnotnull(customer_sk) AND isnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END),sum(CASE WHEN (isnotnull(customer_sk) AND isnotnull(customer_sk)) THEN 1 ELSE 0 END)] - CometColumnarExchange #1 + CometExchange #1 CometHashAggregate [sum,sum,sum,customer_sk,customer_sk] CometProject [customer_sk,customer_sk] CometSortMergeJoin [customer_sk,item_sk,customer_sk,item_sk] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,ss_customer_sk,ss_item_sk] - CometColumnarExchange [ss_customer_sk,ss_item_sk] #2 + CometExchange [ss_customer_sk,ss_item_sk] #2 CometHashAggregate [ss_customer_sk,ss_item_sk] CometProject [ss_item_sk,ss_customer_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_sold_date_sk,d_date_sk] @@ -27,7 +27,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] CometSort [customer_sk,item_sk] CometHashAggregate [customer_sk,item_sk,cs_bill_customer_sk,cs_item_sk] - CometColumnarExchange [cs_bill_customer_sk,cs_item_sk] #5 + CometExchange [cs_bill_customer_sk,cs_item_sk] #5 CometHashAggregate [cs_bill_customer_sk,cs_item_sk] CometProject [cs_bill_customer_sk,cs_item_sk] CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_sold_date_sk,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt index 0239cc78c..3c017673f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/explain.txt @@ -8,9 +8,9 @@ +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -96,18 +96,18 @@ Input [6]: [ss_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15, i_item_id#6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt index 663bc2e3e..ac0911dee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q98/simplified.txt @@ -13,9 +13,9 @@ WholeStageCodegen (3) ColumnarToRow InputAdapter CometSort [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id] - CometColumnarExchange [i_class] #2 + CometExchange [i_class] #2 CometHashAggregate [i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,i_item_id,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt index ceb28b9e8..7935bb4c6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (28) +- CometTakeOrderedAndProject (27) +- CometHashAggregate (26) - +- CometColumnarExchange (25) + +- CometExchange (25) +- CometHashAggregate (24) +- CometProject (23) +- CometBroadcastHashJoin (22) @@ -145,9 +145,9 @@ Input [5]: [cs_ship_date_sk#1, cs_sold_date_sk#5, sm_type#9, cc_name#11, _groupi Keys [3]: [_groupingexpression#14, sm_type#9, cc_name#11] Functions [5]: [partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 30) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 30) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 60)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 60) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 90)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 90) AND ((cs_ship_date_sk#1 - cs_sold_date_sk#5) <= 120)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN ((cs_ship_date_sk#1 - cs_sold_date_sk#5) > 120) THEN 1 ELSE 0 END)] -(25) CometColumnarExchange +(25) CometExchange Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] -Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(_groupingexpression#14, sm_type#9, cc_name#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (26) CometHashAggregate Input [8]: [_groupingexpression#14, sm_type#9, cc_name#11, sum#15, sum#16, sum#17, sum#18, sum#19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/simplified.txt index f5ca9c40a..51599575d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q99/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ] CometHashAggregate [substr(w_warehouse_name, 1, 20),sm_type,cc_name,30 days ,31 - 60 days ,61 - 90 days ,91 - 120 days ,>120 days ,_groupingexpression,sum,sum,sum,sum,sum,sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) <= 30) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 30) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 60)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 60) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 90)) THEN 1 ELSE 0 END),sum(CASE WHEN (((cs_ship_date_sk - cs_sold_date_sk) > 90) AND ((cs_ship_date_sk - cs_sold_date_sk) <= 120)) THEN 1 ELSE 0 END),sum(CASE WHEN ((cs_ship_date_sk - cs_sold_date_sk) > 120) THEN 1 ELSE 0 END)] - CometColumnarExchange [_groupingexpression,sm_type,cc_name] #1 + CometExchange [_groupingexpression,sm_type,cc_name] #1 CometHashAggregate [_groupingexpression,sm_type,cc_name,sum,sum,sum,sum,sum,cs_ship_date_sk,cs_sold_date_sk] CometProject [w_warehouse_name] [cs_ship_date_sk,cs_sold_date_sk,sm_type,cc_name,_groupingexpression] CometBroadcastHashJoin [cs_ship_date_sk,cs_sold_date_sk,w_warehouse_name,sm_type,cc_name,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/explain.txt index f2de73f90..fc449ff52 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (39) +- CometTakeOrderedAndProject (38) +- CometHashAggregate (37) - +- CometColumnarExchange (36) + +- CometExchange (36) +- CometHashAggregate (35) +- CometProject (34) +- CometBroadcastHashJoin (33) @@ -207,9 +207,9 @@ Input [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purch Keys [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Functions [1]: [partial_count(1)] -(36) CometColumnarExchange +(36) CometExchange Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#31] -Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (37) CometHashAggregate Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt index 243bdbb91..2e8f2022c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q10a/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6] CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6,count,count(1)] - CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] CometProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometBroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt index 0ae630c49..52d523ca7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/explain.txt @@ -8,7 +8,7 @@ : :- CometBroadcastHashJoin (33) : : :- CometFilter (16) : : : +- CometHashAggregate (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometHashAggregate (13) : : : +- CometProject (12) : : : +- CometBroadcastHashJoin (11) @@ -24,7 +24,7 @@ : : : +- CometScan parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometProject (28) : : +- CometBroadcastHashJoin (27) @@ -41,7 +41,7 @@ : +- CometBroadcastExchange (48) : +- CometFilter (47) : +- CometHashAggregate (46) - : +- CometColumnarExchange (45) + : +- CometExchange (45) : +- CometHashAggregate (44) : +- CometProject (43) : +- CometBroadcastHashJoin (42) @@ -55,7 +55,7 @@ : +- ReusedExchange (41) +- CometBroadcastExchange (64) +- CometHashAggregate (63) - +- CometColumnarExchange (62) + +- CometExchange (62) +- CometHashAggregate (61) +- CometProject (60) +- CometBroadcastHashJoin (59) @@ -134,9 +134,9 @@ Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_fl Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -(14) CometColumnarExchange +(14) CometExchange Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] @@ -212,9 +212,9 @@ Input [10]: [c_customer_id#20, c_first_name#21, c_last_name#22, c_preferred_cust Keys [8]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#29 - ss_ext_discount_amt#28)))] -(30) CometColumnarExchange +(30) CometExchange Input [9]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, sum#34] -Arguments: hashpartitioning(c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate Input [9]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, sum#34] @@ -283,9 +283,9 @@ Input [10]: [c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust Keys [8]: [c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_country#45, c_login#46, c_email_address#47, d_year#54] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#50 - ws_ext_discount_amt#49)))] -(45) CometColumnarExchange +(45) CometExchange Input [9]: [c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_country#45, c_login#46, c_email_address#47, d_year#54, sum#55] -Arguments: hashpartitioning(c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_country#45, c_login#46, c_email_address#47, d_year#54, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_country#45, c_login#46, c_email_address#47, d_year#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometHashAggregate Input [9]: [c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_country#45, c_login#46, c_email_address#47, d_year#54, sum#55] @@ -362,9 +362,9 @@ Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#72] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -(62) CometColumnarExchange +(62) CometExchange Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#72, sum#73] -Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#72, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (63) CometHashAggregate Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#72, sum#73] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt index c4811d292..8743dd42e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q11/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -32,7 +32,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -56,7 +56,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -71,7 +71,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt index fbe5feb3e..155728936 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (22) +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -92,18 +92,18 @@ Input [6]: [ws_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt index e2df89eec..6f51be0c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q12/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c ColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometColumnarExchange [i_class] #1 + CometExchange [i_class] #1 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt index 9fc14889b..c30e67082 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/explain.txt @@ -4,7 +4,7 @@ +- CometBroadcastHashJoin (83) :- CometFilter (64) : +- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometProject (60) : +- CometBroadcastHashJoin (59) @@ -21,7 +21,7 @@ : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) - : : : : +- CometColumnarExchange (31) + : : : : +- CometExchange (31) : : : : +- CometHashAggregate (30) : : : : +- CometProject (29) : : : : +- CometBroadcastHashJoin (28) @@ -69,7 +69,7 @@ +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) - +- CometColumnarExchange (79) + +- CometExchange (79) +- CometHashAggregate (78) +- CometProject (77) +- CometBroadcastHashJoin (76) @@ -235,9 +235,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -376,9 +376,9 @@ Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_cate Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(62) CometColumnarExchange +(62) CometExchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] @@ -454,9 +454,9 @@ Input [5]: [ss_quantity#56, ss_list_price#57, i_brand_id#62, i_class_id#63, i_ca Keys [3]: [i_brand_id#62, i_class_id#63, i_category_id#64] Functions [2]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), partial_count(1)] -(79) CometColumnarExchange +(79) CometExchange Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] -Arguments: hashpartitioning(i_brand_id#62, i_class_id#63, i_category_id#64, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_brand_id#62, i_class_id#63, i_category_id#64, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (80) CometHashAggregate Input [6]: [i_brand_id#62, i_class_id#63, i_category_id#64, sum#69, isEmpty#70, count#71] @@ -488,7 +488,7 @@ Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#5 Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#53, [id=#54] * ColumnarToRow (102) +- CometHashAggregate (101) - +- CometColumnarExchange (100) + +- CometExchange (100) +- CometHashAggregate (99) +- CometUnion (98) :- CometProject (89) @@ -572,9 +572,9 @@ Input [2]: [quantity#80, list_price#81] Keys: [] Functions [1]: [partial_avg((cast(quantity#80 as decimal(10,0)) * list_price#81))] -(100) CometColumnarExchange +(100) CometExchange Input [2]: [sum#96, count#97] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (101) CometHashAggregate Input [2]: [sum#96, count#97] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt index a51d1d007..bb4be4d3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometColumnarExchange #14 + CometExchange #14 CometHashAggregate [sum,count,quantity,list_price] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] @@ -28,7 +28,7 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #1 + CometExchange [i_brand_id,i_class_id,i_category_id] #1 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -60,7 +60,7 @@ WholeStageCodegen (1) CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #5 + CometExchange [brand_id,class_id,category_id] #5 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -120,7 +120,7 @@ WholeStageCodegen (1) CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #16 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt index 832fa1381..6d048ce94 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/explain.txt @@ -2,16 +2,16 @@ * ColumnarToRow (122) +- CometTakeOrderedAndProject (121) +- CometHashAggregate (120) - +- CometColumnarExchange (119) + +- CometExchange (119) +- CometHashAggregate (118) +- CometUnion (117) :- CometHashAggregate (96) - : +- CometColumnarExchange (95) + : +- CometExchange (95) : +- CometHashAggregate (94) : +- CometUnion (93) : :- CometFilter (64) : : +- CometHashAggregate (63) - : : +- CometColumnarExchange (62) + : : +- CometExchange (62) : : +- CometHashAggregate (61) : : +- CometProject (60) : : +- CometBroadcastHashJoin (59) @@ -28,7 +28,7 @@ : : : : +- CometBroadcastExchange (43) : : : : +- CometBroadcastHashJoin (42) : : : : :- CometHashAggregate (32) - : : : : : +- CometColumnarExchange (31) + : : : : : +- CometExchange (31) : : : : : +- CometHashAggregate (30) : : : : : +- CometProject (29) : : : : : +- CometBroadcastHashJoin (28) @@ -75,7 +75,7 @@ : : +- CometScan parquet spark_catalog.default.date_dim (55) : :- CometFilter (78) : : +- CometHashAggregate (77) - : : +- CometColumnarExchange (76) + : : +- CometExchange (76) : : +- CometHashAggregate (75) : : +- CometProject (74) : : +- CometBroadcastHashJoin (73) @@ -89,7 +89,7 @@ : : +- ReusedExchange (72) : +- CometFilter (92) : +- CometHashAggregate (91) - : +- CometColumnarExchange (90) + : +- CometExchange (90) : +- CometHashAggregate (89) : +- CometProject (88) : +- CometBroadcastHashJoin (87) @@ -102,22 +102,22 @@ : : +- ReusedExchange (83) : +- ReusedExchange (86) :- CometHashAggregate (101) - : +- CometColumnarExchange (100) + : +- CometExchange (100) : +- CometHashAggregate (99) : +- CometHashAggregate (98) : +- ReusedExchange (97) :- CometHashAggregate (106) - : +- CometColumnarExchange (105) + : +- CometExchange (105) : +- CometHashAggregate (104) : +- CometHashAggregate (103) : +- ReusedExchange (102) :- CometHashAggregate (111) - : +- CometColumnarExchange (110) + : +- CometExchange (110) : +- CometHashAggregate (109) : +- CometHashAggregate (108) : +- ReusedExchange (107) +- CometHashAggregate (116) - +- CometColumnarExchange (115) + +- CometExchange (115) +- CometHashAggregate (114) +- CometHashAggregate (113) +- ReusedExchange (112) @@ -272,9 +272,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -413,9 +413,9 @@ Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_cate Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(62) CometColumnarExchange +(62) CometExchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] @@ -475,9 +475,9 @@ Input [5]: [cs_quantity#55, cs_list_price#56, i_brand_id#61, i_class_id#62, i_ca Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] Functions [2]: [partial_sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), partial_count(1)] -(76) CometColumnarExchange +(76) CometExchange Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#65, isEmpty#66, count#67] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (77) CometHashAggregate Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#65, isEmpty#66, count#67] @@ -537,9 +537,9 @@ Input [5]: [ws_quantity#72, ws_list_price#73, i_brand_id#78, i_class_id#79, i_ca Keys [3]: [i_brand_id#78, i_class_id#79, i_category_id#80] Functions [2]: [partial_sum((cast(ws_quantity#72 as decimal(10,0)) * ws_list_price#73)), partial_count(1)] -(90) CometColumnarExchange +(90) CometExchange Input [6]: [i_brand_id#78, i_class_id#79, i_category_id#80, sum#82, isEmpty#83, count#84] -Arguments: hashpartitioning(i_brand_id#78, i_class_id#79, i_category_id#80, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_brand_id#78, i_class_id#79, i_category_id#80, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (91) CometHashAggregate Input [6]: [i_brand_id#78, i_class_id#79, i_category_id#80, sum#82, isEmpty#83, count#84] @@ -560,9 +560,9 @@ Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50 Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum(sales#50), partial_sum(number_sales#51)] -(95) CometColumnarExchange +(95) CometExchange Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#88, isEmpty#89, sum#90] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (96) CometHashAggregate Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#88, isEmpty#89, sum#90] @@ -582,9 +582,9 @@ Input [5]: [channel#49, i_brand_id#91, i_class_id#92, sum_sales#94, number_sales Keys [3]: [channel#49, i_brand_id#91, i_class_id#92] Functions [2]: [partial_sum(sum_sales#94), partial_sum(number_sales#95)] -(100) CometColumnarExchange +(100) CometExchange Input [6]: [channel#49, i_brand_id#91, i_class_id#92, sum#96, isEmpty#97, sum#98] -Arguments: hashpartitioning(channel#49, i_brand_id#91, i_class_id#92, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(channel#49, i_brand_id#91, i_class_id#92, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (101) CometHashAggregate Input [6]: [channel#49, i_brand_id#91, i_class_id#92, sum#96, isEmpty#97, sum#98] @@ -604,9 +604,9 @@ Input [4]: [channel#49, i_brand_id#99, sum_sales#102, number_sales#103] Keys [2]: [channel#49, i_brand_id#99] Functions [2]: [partial_sum(sum_sales#102), partial_sum(number_sales#103)] -(105) CometColumnarExchange +(105) CometExchange Input [5]: [channel#49, i_brand_id#99, sum#104, isEmpty#105, sum#106] -Arguments: hashpartitioning(channel#49, i_brand_id#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(channel#49, i_brand_id#99, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (106) CometHashAggregate Input [5]: [channel#49, i_brand_id#99, sum#104, isEmpty#105, sum#106] @@ -626,9 +626,9 @@ Input [3]: [channel#49, sum_sales#110, number_sales#111] Keys [1]: [channel#49] Functions [2]: [partial_sum(sum_sales#110), partial_sum(number_sales#111)] -(110) CometColumnarExchange +(110) CometExchange Input [4]: [channel#49, sum#112, isEmpty#113, sum#114] -Arguments: hashpartitioning(channel#49, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(channel#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (111) CometHashAggregate Input [4]: [channel#49, sum#112, isEmpty#113, sum#114] @@ -648,9 +648,9 @@ Input [2]: [sum_sales#118, number_sales#119] Keys: [] Functions [2]: [partial_sum(sum_sales#118), partial_sum(number_sales#119)] -(115) CometColumnarExchange +(115) CometExchange Input [3]: [sum#120, isEmpty#121, sum#122] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (116) CometHashAggregate Input [3]: [sum#120, isEmpty#121, sum#122] @@ -669,9 +669,9 @@ Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sale Keys [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] Functions: [] -(119) CometColumnarExchange +(119) CometExchange Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (120) CometHashAggregate Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#123, number_sales#124] @@ -690,7 +690,7 @@ Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sale Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#52, [id=#53] * ColumnarToRow (142) +- CometHashAggregate (141) - +- CometColumnarExchange (140) + +- CometExchange (140) +- CometHashAggregate (139) +- CometUnion (138) :- CometProject (126) @@ -793,9 +793,9 @@ Input [2]: [quantity#148, list_price#149] Keys: [] Functions [1]: [partial_avg((cast(quantity#148 as decimal(10,0)) * list_price#149))] -(140) CometColumnarExchange +(140) CometExchange Input [2]: [sum#165, count#166] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (141) CometHashAggregate Input [2]: [sum#165, count#166] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt index 01803108f..760d0963e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q14a/simplified.txt @@ -3,11 +3,11 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - CometColumnarExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 + CometExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum,isEmpty,sum,sum(sales),sum(number_salesL)] - CometColumnarExchange [channel,i_brand_id,i_class_id,i_category_id] #2 + CometExchange [channel,i_brand_id,i_class_id,i_category_id] #2 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sales,number_sales] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] @@ -16,7 +16,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometColumnarExchange #16 + CometExchange #16 CometHashAggregate [sum,count,quantity,list_price] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] @@ -45,7 +45,7 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #4 ReusedExchange [d_date_sk] #18 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #3 + CometExchange [i_brand_id,i_class_id,i_category_id] #3 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -70,7 +70,7 @@ WholeStageCodegen (1) CometBroadcastExchange [brand_id,class_id,category_id] #6 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #7 + CometExchange [brand_id,class_id,category_id] #7 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -128,7 +128,7 @@ WholeStageCodegen (1) CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #19 + CometExchange [i_brand_id,i_class_id,i_category_id] #19 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,cs_quantity,cs_list_price] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -144,7 +144,7 @@ WholeStageCodegen (1) CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #20 + CometExchange [i_brand_id,i_class_id,i_category_id] #20 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ws_quantity,ws_list_price] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -158,22 +158,22 @@ WholeStageCodegen (1) ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 ReusedExchange [d_date_sk] #15 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometColumnarExchange [channel,i_brand_id,i_class_id] #21 + CometExchange [channel,i_brand_id,i_class_id] #21 CometHashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum,sum_sales,number_sales] CometHashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometColumnarExchange [channel,i_brand_id] #22 + CometExchange [channel,i_brand_id] #22 CometHashAggregate [channel,i_brand_id,sum,isEmpty,sum,sum_sales,number_sales] CometHashAggregate [channel,i_brand_id,sum_sales,number_sales,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometColumnarExchange [channel] #23 + CometExchange [channel] #23 CometHashAggregate [channel,sum,isEmpty,sum,sum_sales,number_sales] CometHashAggregate [channel,sum_sales,number_sales,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometColumnarExchange #24 + CometExchange #24 CometHashAggregate [sum,isEmpty,sum,sum_sales,number_sales] CometHashAggregate [sum_sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt index 144f9eea9..322481b94 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (142) +- CometUnion (141) :- CometHashAggregate (38) - : +- CometColumnarExchange (37) + : +- CometExchange (37) : +- CometHashAggregate (36) : +- CometProject (35) : +- CometBroadcastHashJoin (34) @@ -41,7 +41,7 @@ : +- CometFilter (32) : +- CometScan parquet spark_catalog.default.item (31) :- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometProject (60) : +- CometBroadcastHashJoin (59) @@ -66,7 +66,7 @@ : : +- ReusedExchange (55) : +- ReusedExchange (58) :- CometHashAggregate (89) - : +- CometColumnarExchange (88) + : +- CometExchange (88) : +- CometHashAggregate (87) : +- CometProject (86) : +- CometBroadcastHashJoin (85) @@ -92,7 +92,7 @@ : : +- ReusedExchange (81) : +- ReusedExchange (84) :- CometHashAggregate (115) - : +- CometColumnarExchange (114) + : +- CometExchange (114) : +- CometHashAggregate (113) : +- CometProject (112) : +- CometBroadcastHashJoin (111) @@ -118,7 +118,7 @@ : : +- ReusedExchange (107) : +- ReusedExchange (110) +- CometHashAggregate (140) - +- CometColumnarExchange (139) + +- CometExchange (139) +- CometHashAggregate (138) +- CometProject (137) +- CometBroadcastHashJoin (136) @@ -317,9 +317,9 @@ Input [11]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, agg1#29, ag Keys [4]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22] Functions [7]: [partial_avg(agg1#29), partial_avg(agg2#30), partial_avg(agg3#31), partial_avg(agg4#32), partial_avg(agg5#33), partial_avg(agg6#34), partial_avg(agg7#35)] -(37) CometColumnarExchange +(37) CometExchange Input [18]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] -Arguments: hashpartitioning(i_item_id#28, ca_country#24, ca_state#23, ca_county#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#28, ca_country#24, ca_state#23, ca_county#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (38) CometHashAggregate Input [18]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] @@ -427,9 +427,9 @@ Input [10]: [i_item_id#72, ca_country#69, ca_state#68, agg1#73, agg2#74, agg3#75 Keys [3]: [i_item_id#72, ca_country#69, ca_state#68] Functions [7]: [partial_avg(agg1#73), partial_avg(agg2#74), partial_avg(agg3#75), partial_avg(agg4#76), partial_avg(agg5#77), partial_avg(agg6#78), partial_avg(agg7#79)] -(62) CometColumnarExchange +(62) CometExchange Input [17]: [i_item_id#72, ca_country#69, ca_state#68, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89, sum#90, count#91, sum#92, count#93] -Arguments: hashpartitioning(i_item_id#72, ca_country#69, ca_state#68, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#72, ca_country#69, ca_state#68, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate Input [17]: [i_item_id#72, ca_country#69, ca_state#68, sum#80, count#81, sum#82, count#83, sum#84, count#85, sum#86, count#87, sum#88, count#89, sum#90, count#91, sum#92, count#93] @@ -541,9 +541,9 @@ Input [9]: [i_item_id#116, ca_country#113, agg1#117, agg2#118, agg3#119, agg4#12 Keys [2]: [i_item_id#116, ca_country#113] Functions [7]: [partial_avg(agg1#117), partial_avg(agg2#118), partial_avg(agg3#119), partial_avg(agg4#120), partial_avg(agg5#121), partial_avg(agg6#122), partial_avg(agg7#123)] -(88) CometColumnarExchange +(88) CometExchange Input [16]: [i_item_id#116, ca_country#113, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135, sum#136, count#137] -Arguments: hashpartitioning(i_item_id#116, ca_country#113, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#116, ca_country#113, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (89) CometHashAggregate Input [16]: [i_item_id#116, ca_country#113, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135, sum#136, count#137] @@ -655,9 +655,9 @@ Input [8]: [i_item_id#159, agg1#160, agg2#161, agg3#162, agg4#163, agg5#164, agg Keys [1]: [i_item_id#159] Functions [7]: [partial_avg(agg1#160), partial_avg(agg2#161), partial_avg(agg3#162), partial_avg(agg4#163), partial_avg(agg5#164), partial_avg(agg6#165), partial_avg(agg7#166)] -(114) CometColumnarExchange +(114) CometExchange Input [15]: [i_item_id#159, sum#167, count#168, sum#169, count#170, sum#171, count#172, sum#173, count#174, sum#175, count#176, sum#177, count#178, sum#179, count#180] -Arguments: hashpartitioning(i_item_id#159, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_item_id#159, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (115) CometHashAggregate Input [15]: [i_item_id#159, sum#167, count#168, sum#169, count#170, sum#171, count#172, sum#173, count#174, sum#175, count#176, sum#177, count#178, sum#179, count#180] @@ -765,9 +765,9 @@ Input [7]: [agg1#201, agg2#202, agg3#203, agg4#204, agg5#205, agg6#206, agg7#207 Keys: [] Functions [7]: [partial_avg(agg1#201), partial_avg(agg2#202), partial_avg(agg3#203), partial_avg(agg4#204), partial_avg(agg5#205), partial_avg(agg6#206), partial_avg(agg7#207)] -(139) CometColumnarExchange +(139) CometExchange Input [14]: [sum#208, count#209, sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219, sum#220, count#221] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (140) CometHashAggregate Input [14]: [sum#208, count#209, sum#210, count#211, sum#212, count#213, sum#214, count#215, sum#216, count#217, sum#218, count#219, sum#220, count#221] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/simplified.txt index aef60ba9b..add76afad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q18a/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometUnion [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometColumnarExchange [i_item_id,ca_country,ca_state,ca_county] #1 + CometExchange [i_item_id,ca_country,ca_state,ca_county] #1 CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] @@ -50,7 +50,7 @@ WholeStageCodegen (1) CometFilter [i_item_sk,i_item_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometColumnarExchange [i_item_id,ca_country,ca_state] #9 + CometExchange [i_item_id,ca_country,ca_state] #9 CometHashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] @@ -76,7 +76,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk] #7 ReusedExchange [i_item_sk,i_item_id] #8 CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometColumnarExchange [i_item_id,ca_country] #11 + CometExchange [i_item_id,ca_country] #11 CometHashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] @@ -103,7 +103,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk] #7 ReusedExchange [i_item_sk,i_item_id] #8 CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometColumnarExchange [i_item_id] #13 + CometExchange [i_item_id] #13 CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] @@ -130,7 +130,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk] #7 ReusedExchange [i_item_sk,i_item_id] #8 CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometColumnarExchange #15 + CometExchange #15 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/explain.txt index e2ecb0b16..3de2b0499 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (22) +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -92,18 +92,18 @@ Input [6]: [cs_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt index 7b17378dc..c4d5e48ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q20/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c ColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometColumnarExchange [i_class] #1 + CometExchange [i_class] #1 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt index 1975ac486..2810779ed 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/explain.txt @@ -1,30 +1,28 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometColumnarExchange (23) - +- RowToColumnar (22) - +- * HashAggregate (21) - +- * Expand (20) - +- * Project (19) - +- * BroadcastNestedLoopJoin Inner BuildRight (18) - :- * ColumnarToRow (14) - : +- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.inventory (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.item (9) - +- BroadcastExchange (17) - +- * ColumnarToRow (16) - +- CometScan parquet spark_catalog.default.warehouse (15) +TakeOrderedAndProject (24) ++- * HashAggregate (23) + +- Exchange (22) + +- * HashAggregate (21) + +- * Expand (20) + +- * Project (19) + +- * BroadcastNestedLoopJoin Inner BuildRight (18) + :- * ColumnarToRow (14) + : +- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.item (9) + +- BroadcastExchange (17) + +- * ColumnarToRow (16) + +- CometScan parquet spark_catalog.default.warehouse (15) (1) Scan parquet spark_catalog.default.inventory @@ -126,56 +124,50 @@ Functions [1]: [partial_avg(inv_quantity_on_hand#2)] Aggregate Attributes [2]: [sum#17, count#18] Results [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] -(22) RowToColumnar +(22) Exchange Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] +Arguments: hashpartitioning(i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(23) CometColumnarExchange -Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] -Arguments: hashpartitioning(i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(24) ColumnarToRow [codegen id : 3] -Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] - -(25) HashAggregate [codegen id : 3] +(23) HashAggregate [codegen id : 3] Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] Keys [5]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16] Functions [1]: [avg(inv_quantity_on_hand#2)] Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#21] Results [5]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, avg(inv_quantity_on_hand#2)#21 AS qoh#22] -(26) TakeOrderedAndProject +(24) TakeOrderedAndProject Input [5]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, qoh#22] Arguments: 100, [qoh#22 ASC NULLS FIRST, i_product_name#12 ASC NULLS FIRST, i_brand#13 ASC NULLS FIRST, i_class#14 ASC NULLS FIRST, i_category#15 ASC NULLS FIRST], [i_product_name#12, i_brand#13, i_class#14, i_category#15, qoh#22] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * ColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.date_dim (27) +BroadcastExchange (29) ++- * ColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.date_dim (25) -(27) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) -(29) CometProject +(27) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(30) ColumnarToRow [codegen id : 1] +(28) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(31) BroadcastExchange +(29) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt index f5b281ac6..0e864ab70 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22/simplified.txt @@ -1,41 +1,39 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] WholeStageCodegen (3) HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastNestedLoopJoin - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.warehouse + InputAdapter + Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + WholeStageCodegen (2) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] + Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + BroadcastNestedLoopJoin + ColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometScan parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt index 100e35413..90324f2cc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/explain.txt @@ -5,7 +5,7 @@ :- CometHashAggregate (23) : +- CometHashAggregate (22) : +- CometHashAggregate (21) - : +- CometColumnarExchange (20) + : +- CometExchange (20) : +- CometHashAggregate (19) : +- CometProject (18) : +- CometBroadcastHashJoin (17) @@ -26,22 +26,22 @@ : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.warehouse (14) :- CometHashAggregate (28) - : +- CometColumnarExchange (27) + : +- CometExchange (27) : +- CometHashAggregate (26) : +- CometHashAggregate (25) : +- ReusedExchange (24) :- CometHashAggregate (33) - : +- CometColumnarExchange (32) + : +- CometExchange (32) : +- CometHashAggregate (31) : +- CometHashAggregate (30) : +- ReusedExchange (29) :- CometHashAggregate (38) - : +- CometColumnarExchange (37) + : +- CometExchange (37) : +- CometHashAggregate (36) : +- CometHashAggregate (35) : +- ReusedExchange (34) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometHashAggregate (40) +- ReusedExchange (39) @@ -140,9 +140,9 @@ Input [5]: [inv_quantity_on_hand#3, i_brand#9, i_class#10, i_category#11, i_prod Keys [4]: [i_product_name#12, i_brand#9, i_class#10, i_category#11] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -(20) CometColumnarExchange +(20) CometExchange Input [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#14, count#15] -Arguments: hashpartitioning(i_product_name#12, i_brand#9, i_class#10, i_category#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_product_name#12, i_brand#9, i_class#10, i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#14, count#15] @@ -172,9 +172,9 @@ Input [4]: [i_product_name#19, i_brand#20, i_class#21, qoh#26] Keys [3]: [i_product_name#19, i_brand#20, i_class#21] Functions [1]: [partial_avg(qoh#26)] -(27) CometColumnarExchange +(27) CometExchange Input [5]: [i_product_name#19, i_brand#20, i_class#21, sum#27, count#28] -Arguments: hashpartitioning(i_product_name#19, i_brand#20, i_class#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_product_name#19, i_brand#20, i_class#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [5]: [i_product_name#19, i_brand#20, i_class#21, sum#27, count#28] @@ -194,9 +194,9 @@ Input [3]: [i_product_name#29, i_brand#30, qoh#36] Keys [2]: [i_product_name#29, i_brand#30] Functions [1]: [partial_avg(qoh#36)] -(32) CometColumnarExchange +(32) CometExchange Input [4]: [i_product_name#29, i_brand#30, sum#37, count#38] -Arguments: hashpartitioning(i_product_name#29, i_brand#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_product_name#29, i_brand#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate Input [4]: [i_product_name#29, i_brand#30, sum#37, count#38] @@ -216,9 +216,9 @@ Input [2]: [i_product_name#39, qoh#46] Keys [1]: [i_product_name#39] Functions [1]: [partial_avg(qoh#46)] -(37) CometColumnarExchange +(37) CometExchange Input [3]: [i_product_name#39, sum#47, count#48] -Arguments: hashpartitioning(i_product_name#39, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_product_name#39, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (38) CometHashAggregate Input [3]: [i_product_name#39, sum#47, count#48] @@ -238,9 +238,9 @@ Input [1]: [qoh#56] Keys: [] Functions [1]: [partial_avg(qoh#56)] -(42) CometColumnarExchange +(42) CometExchange Input [2]: [sum#57, count#58] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (43) CometHashAggregate Input [2]: [sum#57, count#58] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/simplified.txt index d5a409c3e..63eda27a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q22a/simplified.txt @@ -6,7 +6,7 @@ WholeStageCodegen (1) CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count,qoh] CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(inv_quantity_on_hand)] - CometColumnarExchange [i_product_name,i_brand,i_class,i_category] #1 + CometExchange [i_product_name,i_brand,i_class,i_category] #1 CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] @@ -35,22 +35,22 @@ WholeStageCodegen (1) CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk] CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometColumnarExchange [i_product_name,i_brand,i_class] #6 + CometExchange [i_product_name,i_brand,i_class] #6 CometHashAggregate [i_product_name,i_brand,i_class,sum,count,qoh] CometHashAggregate [i_product_name,i_brand,i_class,qoh,i_category,sum,count,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometColumnarExchange [i_product_name,i_brand] #7 + CometExchange [i_product_name,i_brand] #7 CometHashAggregate [i_product_name,i_brand,sum,count,qoh] CometHashAggregate [i_product_name,i_brand,qoh,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometColumnarExchange [i_product_name] #8 + CometExchange [i_product_name] #8 CometHashAggregate [i_product_name,sum,count,qoh] CometHashAggregate [i_product_name,qoh,i_brand,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometColumnarExchange #9 + CometExchange #9 CometHashAggregate [sum,count,qoh] CometHashAggregate [qoh,i_product_name,i_brand,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24/explain.txt index 118902e57..9dd9f17b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24/explain.txt @@ -1,54 +1,50 @@ == Physical Plan == -* ColumnarToRow (50) -+- CometSort (49) - +- CometColumnarExchange (48) - +- RowToColumnar (47) - +- * Filter (46) - +- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometColumnarExchange (38) - +- RowToColumnar (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) +* ColumnarToRow (46) ++- CometSort (45) + +- CometColumnarExchange (44) + +- RowToColumnar (43) + +- * Filter (42) + +- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (21) + : : +- CometFilter (20) + : : +- CometScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.customer (24) + +- BroadcastExchange (33) + +- * ColumnarToRow (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.customer_address (30) (1) Scan parquet spark_catalog.default.store_sales @@ -66,9 +62,9 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(4) CometColumnarExchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -89,9 +85,9 @@ Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -220,230 +216,202 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#30] Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -(37) RowToColumnar +(37) Exchange Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(38) CometColumnarExchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(39) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] - -(40) HashAggregate [codegen id : 3] +(38) HashAggregate [codegen id : 3] Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, netpaid#33] Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] Functions [1]: [partial_sum(netpaid#33)] Aggregate Attributes [2]: [sum#34, isEmpty#35] Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -(42) RowToColumnar -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] - -(43) CometColumnarExchange -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] Functions [1]: [sum(netpaid#33)] Aggregate Attributes [1]: [sum(netpaid#33)#38] Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, sum(netpaid#33)#38 AS paid#39] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subquery scalar-subquery#40, [id=#41] as decimal(33,8)))) -(47) RowToColumnar +(43) RowToColumnar Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -(48) CometColumnarExchange +(44) CometColumnarExchange Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(49) CometSort +(45) CometSort Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] Arguments: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] -(50) ColumnarToRow [codegen id : 5] +(46) ColumnarToRow [codegen id : 5] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#40, [id=#41] -* HashAggregate (81) -+- * ColumnarToRow (80) - +- CometColumnarExchange (79) - +- RowToColumnar (78) - +- * HashAggregate (77) - +- * HashAggregate (76) - +- * ColumnarToRow (75) - +- CometColumnarExchange (74) - +- RowToColumnar (73) - +- * HashAggregate (72) - +- * Project (71) - +- * BroadcastHashJoin Inner BuildRight (70) - :- * ColumnarToRow (68) - : +- CometProject (67) - : +- CometBroadcastHashJoin (66) - : :- CometProject (64) - : : +- CometBroadcastHashJoin (63) - : : :- CometProject (59) - : : : +- CometBroadcastHashJoin (58) - : : : :- CometProject (56) - : : : : +- CometSortMergeJoin (55) - : : : : :- CometSort (52) - : : : : : +- ReusedExchange (51) - : : : : +- CometSort (54) - : : : : +- ReusedExchange (53) - : : : +- ReusedExchange (57) - : : +- CometBroadcastExchange (62) - : : +- CometFilter (61) - : : +- CometScan parquet spark_catalog.default.item (60) - : +- ReusedExchange (65) - +- ReusedExchange (69) - - -(51) ReusedExchange [Reuses operator id: 4] +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#40, [id=#41] +* HashAggregate (73) ++- Exchange (72) + +- * HashAggregate (71) + +- * HashAggregate (70) + +- Exchange (69) + +- * HashAggregate (68) + +- * Project (67) + +- * BroadcastHashJoin Inner BuildRight (66) + :- * ColumnarToRow (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (55) + : : : +- CometBroadcastHashJoin (54) + : : : :- CometProject (52) + : : : : +- CometSortMergeJoin (51) + : : : : :- CometSort (48) + : : : : : +- ReusedExchange (47) + : : : : +- CometSort (50) + : : : : +- ReusedExchange (49) + : : : +- ReusedExchange (53) + : : +- CometBroadcastExchange (58) + : : +- CometFilter (57) + : : +- CometScan parquet spark_catalog.default.item (56) + : +- ReusedExchange (61) + +- ReusedExchange (65) + + +(47) ReusedExchange [Reuses operator id: 4] Output [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] -(52) CometSort +(48) CometSort Input [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46], [ss_ticket_number#45 ASC NULLS FIRST, ss_item_sk#42 ASC NULLS FIRST] -(53) ReusedExchange [Reuses operator id: 9] +(49) ReusedExchange [Reuses operator id: 9] Output [2]: [sr_item_sk#47, sr_ticket_number#48] -(54) CometSort +(50) CometSort Input [2]: [sr_item_sk#47, sr_ticket_number#48] Arguments: [sr_item_sk#47, sr_ticket_number#48], [sr_ticket_number#48 ASC NULLS FIRST, sr_item_sk#47 ASC NULLS FIRST] -(55) CometSortMergeJoin +(51) CometSortMergeJoin Left output [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] Right output [2]: [sr_item_sk#47, sr_ticket_number#48] Arguments: [ss_ticket_number#45, ss_item_sk#42], [sr_ticket_number#48, sr_item_sk#47], Inner -(56) CometProject +(52) CometProject Input [7]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46, sr_item_sk#47, sr_ticket_number#48] Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46], [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46] -(57) ReusedExchange [Reuses operator id: 16] +(53) ReusedExchange [Reuses operator id: 16] Output [4]: [s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] -(58) CometBroadcastHashJoin +(54) CometBroadcastHashJoin Left output [4]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46] Right output [4]: [s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] Arguments: [ss_store_sk#44], [s_store_sk#49], Inner, BuildRight -(59) CometProject +(55) CometProject Input [8]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46, s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52], [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52] -(60) Scan parquet spark_catalog.default.item +(56) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(61) CometFilter +(57) CometFilter Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Condition : isnotnull(i_item_sk#53) -(62) CometBroadcastExchange +(58) CometBroadcastExchange Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Arguments: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -(63) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [6]: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52] Right output [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Arguments: [ss_item_sk#42], [i_item_sk#53], Inner, BuildRight -(64) CometProject +(60) CometProject Input [12]: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Arguments: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58], [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -(65) ReusedExchange [Reuses operator id: 26] +(61) ReusedExchange [Reuses operator id: 26] Output [5]: [c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -(66) CometBroadcastHashJoin +(62) CometBroadcastHashJoin Left output [10]: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Right output [5]: [c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] Arguments: [ss_customer_sk#43], [c_customer_sk#59], Inner, BuildRight -(67) CometProject +(63) CometProject Input [15]: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] Arguments: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63], [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -(68) ColumnarToRow [codegen id : 2] +(64) ColumnarToRow [codegen id : 2] Input [13]: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -(69) ReusedExchange [Reuses operator id: 33] +(65) ReusedExchange [Reuses operator id: 33] Output [4]: [ca_address_sk#64, ca_state#65, ca_zip#66, ca_country#67] -(70) BroadcastHashJoin [codegen id : 2] +(66) BroadcastHashJoin [codegen id : 2] Left keys [3]: [c_current_addr_sk#60, c_birth_country#63, s_zip#52] Right keys [3]: [ca_address_sk#64, upper(ca_country#67), ca_zip#66] Join type: Inner Join condition: None -(71) Project [codegen id : 2] +(67) Project [codegen id : 2] Output [11]: [ss_net_paid#46, s_store_name#50, s_state#51, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_first_name#61, c_last_name#62, ca_state#65] Input [17]: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63, ca_address_sk#64, ca_state#65, ca_zip#66, ca_country#67] -(72) HashAggregate [codegen id : 2] +(68) HashAggregate [codegen id : 2] Input [11]: [ss_net_paid#46, s_store_name#50, s_state#51, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_first_name#61, c_last_name#62, ca_state#65] Keys [10]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#46))] Aggregate Attributes [1]: [sum#68] Results [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] -(73) RowToColumnar +(69) Exchange Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] +Arguments: hashpartitioning(c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(74) CometColumnarExchange -Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] -Arguments: hashpartitioning(c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(75) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] - -(76) HashAggregate [codegen id : 3] +(70) HashAggregate [codegen id : 3] Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] Keys [10]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55] Functions [1]: [sum(UnscaledValue(ss_net_paid#46))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#46))#32] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#46))#32,17,2) AS netpaid#70] -(77) HashAggregate [codegen id : 3] +(71) HashAggregate [codegen id : 3] Input [1]: [netpaid#70] Keys: [] Functions [1]: [partial_avg(netpaid#70)] Aggregate Attributes [2]: [sum#71, count#72] Results [2]: [sum#73, count#74] -(78) RowToColumnar -Input [2]: [sum#73, count#74] - -(79) CometColumnarExchange -Input [2]: [sum#73, count#74] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(80) ColumnarToRow [codegen id : 4] +(72) Exchange Input [2]: [sum#73, count#74] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] -(81) HashAggregate [codegen id : 4] +(73) HashAggregate [codegen id : 4] Input [2]: [sum#73, count#74] Keys: [] Functions [1]: [avg(netpaid#70)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24/simplified.txt index 42c283018..1e6a1a1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q24/simplified.txt @@ -9,92 +9,84 @@ WholeStageCodegen (5) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #10 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #5 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - InputAdapter - ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name] #2 - RowToColumnar + InputAdapter + Exchange #10 WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [netpaid] [sum,count,sum,count] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #4 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #5 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #5 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + InputAdapter + ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name] #2 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #4 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #5 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (1) + ColumnarToRow InputAdapter - BroadcastExchange #9 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] + CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/explain.txt index 33a1e3542..e181d2ce2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (68) +- CometUnion (67) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -30,7 +30,7 @@ : +- CometFilter (21) : +- CometScan parquet spark_catalog.default.item (20) :- CometHashAggregate (47) - : +- CometColumnarExchange (46) + : +- CometExchange (46) : +- CometHashAggregate (45) : +- CometProject (44) : +- CometBroadcastHashJoin (43) @@ -50,7 +50,7 @@ : : +- CometScan parquet spark_catalog.default.store (36) : +- ReusedExchange (42) +- CometHashAggregate (66) - +- CometColumnarExchange (65) + +- CometExchange (65) +- CometHashAggregate (64) +- CometProject (63) +- CometBroadcastHashJoin (62) @@ -191,9 +191,9 @@ Input [6]: [i_item_id#19, s_state#17, agg1#20, agg2#21, agg3#22, agg4#23] Keys [2]: [i_item_id#19, s_state#17] Functions [4]: [partial_avg(agg1#20), partial_avg(UnscaledValue(agg2#21)), partial_avg(UnscaledValue(agg3#22)), partial_avg(UnscaledValue(agg4#23))] -(26) CometColumnarExchange +(26) CometExchange Input [10]: [i_item_id#19, s_state#17, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] -Arguments: hashpartitioning(i_item_id#19, s_state#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#19, s_state#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [10]: [i_item_id#19, s_state#17, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] @@ -281,9 +281,9 @@ Input [5]: [i_item_id#46, agg1#47, agg2#48, agg3#49, agg4#50] Keys [1]: [i_item_id#46] Functions [4]: [partial_avg(agg1#47), partial_avg(UnscaledValue(agg2#48)), partial_avg(UnscaledValue(agg3#49)), partial_avg(UnscaledValue(agg4#50))] -(46) CometColumnarExchange +(46) CometExchange Input [9]: [i_item_id#46, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58] -Arguments: hashpartitioning(i_item_id#46, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#46, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (47) CometHashAggregate Input [9]: [i_item_id#46, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58] @@ -367,9 +367,9 @@ Input [4]: [agg1#72, agg2#73, agg3#74, agg4#75] Keys: [] Functions [4]: [partial_avg(agg1#72), partial_avg(UnscaledValue(agg2#73)), partial_avg(UnscaledValue(agg3#74)), partial_avg(UnscaledValue(agg4#75))] -(65) CometColumnarExchange +(65) CometExchange Input [8]: [sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (66) CometHashAggregate Input [8]: [sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/simplified.txt index 43f9cbc6e..ed169acda 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q27a/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] CometUnion [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometColumnarExchange [i_item_id,s_state] #1 + CometExchange [i_item_id,s_state] #1 CometHashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] @@ -39,7 +39,7 @@ WholeStageCodegen (1) CometFilter [i_item_sk,i_item_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometColumnarExchange [i_item_id] #7 + CometExchange [i_item_id] #7 CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] @@ -60,7 +60,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [i_item_sk,i_item_id] #6 CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometColumnarExchange #9 + CometExchange #9 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/explain.txt index d978de885..a8800a70f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/explain.txt @@ -6,7 +6,7 @@ +- CometBroadcastHashJoin (28) :- CometFilter (24) : +- CometHashAggregate (23) - : +- CometColumnarExchange (22) + : +- CometExchange (22) : +- CometHashAggregate (21) : +- CometProject (20) : +- CometBroadcastHashJoin (19) @@ -134,9 +134,9 @@ Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt index fafc612d6..7da6076f5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q34/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometColumnarExchange [ss_ticket_number,ss_customer_sk] #2 + CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt index 721036bfb..790000085 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/explain.txt @@ -1,50 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- Exchange (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * ColumnarToRow (31) + : +- CometFilter (30) + : +- CometScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometFilter (36) + +- CometScan parquet spark_catalog.default.customer_demographics (35) (1) Scan parquet spark_catalog.default.customer @@ -240,56 +238,50 @@ Functions [10]: [partial_count(1), partial_avg(cd_dep_count#25), partial_max(cd_ Aggregate Attributes [13]: [count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] Results [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] -(42) RowToColumnar +(42) Exchange Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) CometColumnarExchange -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] -Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 6] -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] - -(45) HashAggregate [codegen id : 6] +(43) HashAggregate [codegen id : 6] Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] Keys [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [10]: [count(1), avg(cd_dep_count#25), max(cd_dep_count#25), sum(cd_dep_count#25), avg(cd_dep_employed_count#26), max(cd_dep_employed_count#26), sum(cd_dep_employed_count#26), avg(cd_dep_college_count#27), max(cd_dep_college_count#27), sum(cd_dep_college_count#27)] Aggregate Attributes [10]: [count(1)#54, avg(cd_dep_count#25)#55, max(cd_dep_count#25)#56, sum(cd_dep_count#25)#57, avg(cd_dep_employed_count#26)#58, max(cd_dep_employed_count#26)#59, sum(cd_dep_employed_count#26)#60, avg(cd_dep_college_count#27)#61, max(cd_dep_college_count#27)#62, sum(cd_dep_college_count#27)#63] Results [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, count(1)#54 AS cnt1#64, avg(cd_dep_count#25)#55 AS avg(cd_dep_count)#65, max(cd_dep_count#25)#56 AS max(cd_dep_count)#66, sum(cd_dep_count#25)#57 AS sum(cd_dep_count)#67, cd_dep_employed_count#26, count(1)#54 AS cnt2#68, avg(cd_dep_employed_count#26)#58 AS avg(cd_dep_employed_count)#69, max(cd_dep_employed_count#26)#59 AS max(cd_dep_employed_count)#70, sum(cd_dep_employed_count#26)#60 AS sum(cd_dep_employed_count)#71, cd_dep_college_count#27, count(1)#54 AS cnt3#72, avg(cd_dep_college_count#27)#61 AS avg(cd_dep_college_count)#73, max(cd_dep_college_count#27)#62 AS max(cd_dep_college_count)#74, sum(cd_dep_college_count#27)#63 AS sum(cd_dep_college_count)#75] -(46) TakeOrderedAndProject +(44) TakeOrderedAndProject Input [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] Arguments: 100, [ca_state#21 ASC NULLS FIRST, cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (51) -+- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.date_dim (47) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan parquet spark_catalog.default.date_dim (45) -(47) Scan parquet spark_catalog.default.date_dim +(45) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(48) CometFilter +(46) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) -(49) CometProject +(47) CometProject Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Arguments: [d_date_sk#9], [d_date_sk#9] -(50) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(51) BroadcastExchange +(49) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt index 221648657..0b55e23ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35/simplified.txt @@ -1,72 +1,70 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + InputAdapter + Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt index 476dab310..9f321ebda 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (38) +- CometTakeOrderedAndProject (37) +- CometHashAggregate (36) - +- CometColumnarExchange (35) + +- CometExchange (35) +- CometHashAggregate (34) +- CometProject (33) +- CometBroadcastHashJoin (32) @@ -202,9 +202,9 @@ Input [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd Keys [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [10]: [partial_count(1), partial_avg(cd_dep_count#25), partial_max(cd_dep_count#25), partial_sum(cd_dep_count#25), partial_avg(cd_dep_employed_count#26), partial_max(cd_dep_employed_count#26), partial_sum(cd_dep_employed_count#26), partial_avg(cd_dep_college_count#27), partial_max(cd_dep_college_count#27), partial_sum(cd_dep_college_count#27)] -(35) CometColumnarExchange +(35) CometExchange Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] -Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (36) CometHashAggregate Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt index 25f72e37b..5ebac3026 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q35a/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometBroadcastHashJoin [c_current_cdemo_sk,ca_state,cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt index 942fd184c..e1b4302ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/explain.txt @@ -4,13 +4,13 @@ TakeOrderedAndProject (42) +- Window (40) +- * ColumnarToRow (39) +- CometSort (38) - +- CometColumnarExchange (37) + +- CometExchange (37) +- CometHashAggregate (36) - +- CometColumnarExchange (35) + +- CometExchange (35) +- CometHashAggregate (34) +- CometUnion (33) :- CometHashAggregate (22) - : +- CometColumnarExchange (21) + : +- CometExchange (21) : +- CometHashAggregate (20) : +- CometProject (19) : +- CometBroadcastHashJoin (18) @@ -32,12 +32,12 @@ TakeOrderedAndProject (42) : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.store (14) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometHashAggregate (24) : +- ReusedExchange (23) +- CometHashAggregate (32) - +- CometColumnarExchange (31) + +- CometExchange (31) +- CometHashAggregate (30) +- CometHashAggregate (29) +- ReusedExchange (28) @@ -140,9 +140,9 @@ Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#10, i_category#11] Keys [2]: [i_category#11, i_class#10] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(21) CometColumnarExchange +(21) CometExchange Input [4]: [i_category#11, i_class#10, sum#14, sum#15] -Arguments: hashpartitioning(i_category#11, i_class#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#11, i_class#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate Input [4]: [i_category#11, i_class#10, sum#14, sum#15] @@ -162,9 +162,9 @@ Input [3]: [ss_net_profit#22, ss_ext_sales_price#23, i_category#16] Keys [1]: [i_category#16] Functions [2]: [partial_sum(ss_net_profit#22), partial_sum(ss_ext_sales_price#23)] -(26) CometColumnarExchange +(26) CometExchange Input [5]: [i_category#16, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate Input [5]: [i_category#16, sum#24, isEmpty#25, sum#26, isEmpty#27] @@ -184,9 +184,9 @@ Input [2]: [ss_net_profit#34, ss_ext_sales_price#35] Keys: [] Functions [2]: [partial_sum(ss_net_profit#34), partial_sum(ss_ext_sales_price#35)] -(31) CometColumnarExchange +(31) CometExchange Input [4]: [sum#36, isEmpty#37, sum#38, isEmpty#39] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometHashAggregate Input [4]: [sum#36, isEmpty#37, sum#38, isEmpty#39] @@ -203,18 +203,18 @@ Input [6]: [gross_margin#40, i_category#11, i_class#10, t_category#41, t_class#4 Keys [6]: [gross_margin#40, i_category#11, i_class#10, t_category#41, t_class#42, lochierarchy#43] Functions: [] -(35) CometColumnarExchange +(35) CometExchange Input [6]: [gross_margin#40, i_category#11, i_class#10, t_category#41, t_class#42, lochierarchy#43] -Arguments: hashpartitioning(gross_margin#40, i_category#11, i_class#10, t_category#41, t_class#42, lochierarchy#43, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(gross_margin#40, i_category#11, i_class#10, t_category#41, t_class#42, lochierarchy#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (36) CometHashAggregate Input [6]: [gross_margin#40, i_category#11, i_class#10, t_category#41, t_class#42, lochierarchy#43] Keys [6]: [gross_margin#40, i_category#11, i_class#10, t_category#41, t_class#42, lochierarchy#43] Functions: [] -(37) CometColumnarExchange +(37) CometExchange Input [5]: [gross_margin#40, i_category#11, i_class#10, lochierarchy#43, _w0#55] -Arguments: hashpartitioning(lochierarchy#43, _w0#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(lochierarchy#43, _w0#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (38) CometSort Input [5]: [gross_margin#40, i_category#11, i_class#10, lochierarchy#43, _w0#55] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/simplified.txt index 9fc6f31f9..6eb469dae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q36a/simplified.txt @@ -7,13 +7,13 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i ColumnarToRow InputAdapter CometSort [gross_margin,i_category,i_class,lochierarchy,_w0] - CometColumnarExchange [lochierarchy,_w0] #1 + CometExchange [lochierarchy,_w0] #1 CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,t_category,t_class] - CometColumnarExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 + CometExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] CometUnion [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_category,i_class] #3 + CometExchange [i_category,i_class] #3 CometHashAggregate [i_category,i_class,sum,sum,ss_net_profit,ss_ext_sales_price] CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] @@ -43,12 +43,12 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometFilter [s_store_sk,s_state] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty,sum(ss_net_profit),sum(ss_ext_sales_price)] - CometColumnarExchange [i_category] #8 + CometExchange [i_category] #8 CometHashAggregate [i_category,sum,isEmpty,sum,isEmpty,ss_net_profit,ss_ext_sales_price] CometHashAggregate [ss_net_profit,ss_ext_sales_price,i_category,i_class,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty,sum(ss_net_profit),sum(ss_ext_sales_price)] - CometColumnarExchange #9 + CometExchange #9 CometHashAggregate [sum,isEmpty,sum,isEmpty,ss_net_profit,ss_ext_sales_price] CometHashAggregate [ss_net_profit,ss_ext_sales_price,i_category,i_class,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/explain.txt index b6ab28625..3bc9bb6e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/explain.txt @@ -11,9 +11,9 @@ TakeOrderedAndProject (45) : : +- Window (24) : : +- * ColumnarToRow (23) : : +- CometSort (22) - : : +- CometColumnarExchange (21) + : : +- CometExchange (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -135,18 +135,18 @@ Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#10, d_moy#11, s_st Keys [6]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -(19) CometColumnarExchange +(19) CometExchange Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (20) CometHashAggregate Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#15] Keys [6]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -(21) CometColumnarExchange +(21) CometExchange Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (22) CometSort Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#16, _w0#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/simplified.txt index 3c44ef74c..c16c278d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q47/simplified.txt @@ -16,9 +16,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, ColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum,ss_sales_price] CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/explain.txt index ed68082f6..7d9198ed5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/explain.txt @@ -1,83 +1,81 @@ == Physical Plan == -* ColumnarToRow (79) -+- CometTakeOrderedAndProject (78) - +- CometHashAggregate (77) - +- CometColumnarExchange (76) - +- RowToColumnar (75) - +- * HashAggregate (74) - +- Union (73) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * ColumnarToRow (21) - : +- CometSort (20) - : +- CometColumnarExchange (19) - : +- CometHashAggregate (18) - : +- CometColumnarExchange (17) - : +- CometHashAggregate (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometScan parquet spark_catalog.default.web_returns (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.date_dim (10) - :- * Project (49) - : +- * Filter (48) - : +- Window (47) - : +- * Sort (46) - : +- Window (45) - : +- * ColumnarToRow (44) - : +- CometSort (43) - : +- CometColumnarExchange (42) - : +- CometHashAggregate (41) - : +- CometColumnarExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometBroadcastExchange (30) - : : : +- CometProject (29) - : : : +- CometFilter (28) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) - : : +- CometProject (33) - : : +- CometFilter (32) - : : +- CometScan parquet spark_catalog.default.catalog_returns (31) - : +- ReusedExchange (36) - +- * Project (72) - +- * Filter (71) - +- Window (70) - +- * Sort (69) - +- Window (68) - +- * ColumnarToRow (67) - +- CometSort (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometColumnarExchange (63) - +- CometHashAggregate (62) - +- CometProject (61) - +- CometBroadcastHashJoin (60) - :- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometBroadcastExchange (53) - : : +- CometProject (52) - : : +- CometFilter (51) - : : +- CometScan parquet spark_catalog.default.store_sales (50) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometScan parquet spark_catalog.default.store_returns (54) - +- ReusedExchange (59) +TakeOrderedAndProject (77) ++- * HashAggregate (76) + +- Exchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * ColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * ColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * ColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) (1) Scan parquet spark_catalog.default.web_sales @@ -157,18 +155,18 @@ Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, w Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -(19) CometColumnarExchange +(19) CometExchange Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort Input [3]: [item#22, return_ratio#23, currency_ratio#24] @@ -258,18 +256,18 @@ Input [5]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37 Keys [1]: [cs_item_sk#28] Functions [4]: [partial_sum(coalesce(cr_return_quantity#37, 0)), partial_sum(coalesce(cs_quantity#30, 0)), partial_sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] -(40) CometColumnarExchange +(40) CometExchange Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] Keys [1]: [cs_item_sk#28] Functions [4]: [sum(coalesce(cr_return_quantity#37, 0)), sum(coalesce(cs_quantity#30, 0)), sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] -(42) CometColumnarExchange +(42) CometExchange Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort Input [3]: [item#47, return_ratio#48, currency_ratio#49] @@ -359,18 +357,18 @@ Input [5]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62 Keys [1]: [ss_item_sk#53] Functions [4]: [partial_sum(coalesce(sr_return_quantity#62, 0)), partial_sum(coalesce(ss_quantity#55, 0)), partial_sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] -(63) CometColumnarExchange +(63) CometExchange Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (64) CometHashAggregate Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] Keys [1]: [ss_item_sk#53] Functions [4]: [sum(coalesce(sr_return_quantity#62, 0)), sum(coalesce(ss_quantity#55, 0)), sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] -(65) CometColumnarExchange +(65) CometExchange Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (66) CometSort Input [3]: [item#72, return_ratio#73, currency_ratio#74] @@ -408,54 +406,50 @@ Functions: [] Aggregate Attributes: [] Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -(75) RowToColumnar +(75) Exchange Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(76) CometColumnarExchange -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(77) CometHashAggregate +(76) HashAggregate [codegen id : 11] Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -(78) CometTakeOrderedAndProject -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#27 ASC NULLS FIRST,return_rank#25 ASC NULLS FIRST,currency_rank#26 ASC NULLS FIRST,item#22 ASC NULLS FIRST], output=[channel#27,item#22,return_ratio#23,return_rank#25,currency_rank#26]), [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26], 100, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST, item#22 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] - -(79) ColumnarToRow [codegen id : 11] +(77) TakeOrderedAndProject Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Arguments: 100, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST, item#22 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (84) -+- * ColumnarToRow (83) - +- CometProject (82) - +- CometFilter (81) - +- CometScan parquet spark_catalog.default.date_dim (80) +BroadcastExchange (82) ++- * ColumnarToRow (81) + +- CometProject (80) + +- CometFilter (79) + +- CometScan parquet spark_catalog.default.date_dim (78) -(80) Scan parquet spark_catalog.default.date_dim +(78) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(81) CometFilter +(79) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) -(82) CometProject +(80) CometProject Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Arguments: [d_date_sk#13], [d_date_sk#13] -(83) ColumnarToRow [codegen id : 1] +(81) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#13] -(84) BroadcastExchange +(82) BroadcastExchange Input [1]: [d_date_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/simplified.txt index 0d02a9c39..d98b2b0a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q49/simplified.txt @@ -1,111 +1,109 @@ -WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] - CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] - CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (3) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (2) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #2 - CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [ws_item_sk] #3 - CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (6) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (5) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #7 - CometHashAggregate [item,return_ratio,currency_ratio,cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [cs_item_sk] #8 - CometHashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - WholeStageCodegen (9) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (8) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #10 - CometHashAggregate [item,return_ratio,currency_ratio,ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [ss_item_sk] #11 - CometHashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 +TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] + WholeStageCodegen (11) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #7 + CometHashAggregate [item,return_ratio,currency_ratio,cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #8 + CometHashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #10 + CometHashAggregate [item,return_ratio,currency_ratio,ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #11 + CometHashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/explain.txt index 20a3f1c52..5d728a1c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/explain.txt @@ -1,84 +1,76 @@ == Physical Plan == -TakeOrderedAndProject (80) -+- * Filter (79) - +- * HashAggregate (78) - +- * HashAggregate (77) - +- * Project (76) - +- * BroadcastHashJoin Inner BuildRight (75) - :- Window (68) - : +- * ColumnarToRow (67) - : +- CometSort (66) - : +- CometColumnarExchange (65) - : +- CometProject (64) - : +- CometFilter (63) - : +- CometSortMergeJoin (62) - : :- CometSort (32) - : : +- CometColumnarExchange (31) - : : +- RowToColumnar (30) - : : +- * HashAggregate (29) - : : +- * ColumnarToRow (28) - : : +- CometColumnarExchange (27) - : : +- RowToColumnar (26) - : : +- * HashAggregate (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * Project (16) - : : : +- Window (15) - : : : +- * ColumnarToRow (14) - : : : +- CometSort (13) - : : : +- CometColumnarExchange (12) - : : : +- CometHashAggregate (11) - : : : +- CometColumnarExchange (10) - : : : +- CometHashAggregate (9) - : : : +- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- Window (20) - : : +- * ColumnarToRow (19) - : : +- CometSort (18) - : : +- ReusedExchange (17) - : +- CometSort (61) - : +- CometColumnarExchange (60) - : +- RowToColumnar (59) - : +- * HashAggregate (58) - : +- * ColumnarToRow (57) - : +- CometColumnarExchange (56) - : +- RowToColumnar (55) - : +- * HashAggregate (54) - : +- * Project (53) - : +- * BroadcastHashJoin Inner BuildRight (52) - : :- * Project (45) - : : +- Window (44) - : : +- * ColumnarToRow (43) - : : +- CometSort (42) - : : +- CometColumnarExchange (41) - : : +- CometHashAggregate (40) - : : +- CometColumnarExchange (39) - : : +- CometHashAggregate (38) - : : +- CometProject (37) - : : +- CometBroadcastHashJoin (36) - : : :- CometFilter (34) - : : : +- CometScan parquet spark_catalog.default.store_sales (33) - : : +- ReusedExchange (35) - : +- BroadcastExchange (51) - : +- * Project (50) - : +- Window (49) - : +- * ColumnarToRow (48) - : +- CometSort (47) - : +- ReusedExchange (46) - +- BroadcastExchange (74) - +- * Project (73) - +- Window (72) - +- * ColumnarToRow (71) - +- CometSort (70) - +- ReusedExchange (69) +TakeOrderedAndProject (72) ++- * Filter (71) + +- * HashAggregate (70) + +- * HashAggregate (69) + +- * Project (68) + +- * BroadcastHashJoin Inner BuildRight (67) + :- Window (61) + : +- * Sort (60) + : +- Exchange (59) + : +- * Project (58) + : +- * Filter (57) + : +- * SortMergeJoin FullOuter (56) + : :- * Sort (29) + : : +- Exchange (28) + : : +- * HashAggregate (27) + : : +- Exchange (26) + : : +- * HashAggregate (25) + : : +- * Project (24) + : : +- * BroadcastHashJoin Inner BuildRight (23) + : : :- * Project (16) + : : : +- Window (15) + : : : +- * ColumnarToRow (14) + : : : +- CometSort (13) + : : : +- CometExchange (12) + : : : +- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (22) + : : +- * Project (21) + : : +- Window (20) + : : +- * ColumnarToRow (19) + : : +- CometSort (18) + : : +- ReusedExchange (17) + : +- * Sort (55) + : +- Exchange (54) + : +- * HashAggregate (53) + : +- Exchange (52) + : +- * HashAggregate (51) + : +- * Project (50) + : +- * BroadcastHashJoin Inner BuildRight (49) + : :- * Project (42) + : : +- Window (41) + : : +- * ColumnarToRow (40) + : : +- CometSort (39) + : : +- CometExchange (38) + : : +- CometHashAggregate (37) + : : +- CometExchange (36) + : : +- CometHashAggregate (35) + : : +- CometProject (34) + : : +- CometBroadcastHashJoin (33) + : : :- CometFilter (31) + : : : +- CometScan parquet spark_catalog.default.store_sales (30) + : : +- ReusedExchange (32) + : +- BroadcastExchange (48) + : +- * Project (47) + : +- Window (46) + : +- * ColumnarToRow (45) + : +- CometSort (44) + : +- ReusedExchange (43) + +- BroadcastExchange (66) + +- * Project (65) + +- Window (64) + +- * Sort (63) + +- ReusedExchange (62) (1) Scan parquet spark_catalog.default.web_sales @@ -126,18 +118,18 @@ Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6] Keys [2]: [ws_item_sk#1, d_date#6] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -(10) CometColumnarExchange +(10) CometExchange Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate Input [3]: [ws_item_sk#1, d_date#6, sum#8] Keys [2]: [ws_item_sk#1, d_date#6] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -(12) CometColumnarExchange +(12) CometExchange Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (13) CometSort Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] @@ -193,35 +185,26 @@ Functions [1]: [partial_sum(sumws#16)] Aggregate Attributes [2]: [sum#17, isEmpty#18] Results [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] -(26) RowToColumnar +(26) Exchange Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] +Arguments: hashpartitioning(item_sk#9, d_date#6, sumws#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(27) CometColumnarExchange -Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] -Arguments: hashpartitioning(item_sk#9, d_date#6, sumws#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(28) ColumnarToRow [codegen id : 5] -Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] - -(29) HashAggregate [codegen id : 5] +(27) HashAggregate [codegen id : 5] Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] Keys [3]: [item_sk#9, d_date#6, sumws#10] Functions [1]: [sum(sumws#16)] Aggregate Attributes [1]: [sum(sumws#16)#21] Results [3]: [item_sk#9, d_date#6, sum(sumws#16)#21 AS cume_sales#22] -(30) RowToColumnar +(28) Exchange Input [3]: [item_sk#9, d_date#6, cume_sales#22] +Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(31) CometColumnarExchange +(29) Sort [codegen id : 6] Input [3]: [item_sk#9, d_date#6, cume_sales#22] -Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0 -(32) CometSort -Input [3]: [item_sk#9, d_date#6, cume_sales#22] -Arguments: [item_sk#9, d_date#6, cume_sales#22], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] - -(33) Scan parquet spark_catalog.default.store_sales +(30) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -229,236 +212,222 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#25), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(34) CometFilter +(31) CometFilter Input [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] Condition : isnotnull(ss_item_sk#23) -(35) ReusedExchange [Reuses operator id: 6] +(32) ReusedExchange [Reuses operator id: 6] Output [2]: [d_date_sk#27, d_date#28] -(36) CometBroadcastHashJoin +(33) CometBroadcastHashJoin Left output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] Right output [2]: [d_date_sk#27, d_date#28] Arguments: [ss_sold_date_sk#25], [d_date_sk#27], Inner, BuildRight -(37) CometProject +(34) CometProject Input [5]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25, d_date_sk#27, d_date#28] Arguments: [ss_item_sk#23, ss_sales_price#24, d_date#28], [ss_item_sk#23, ss_sales_price#24, d_date#28] -(38) CometHashAggregate +(35) CometHashAggregate Input [3]: [ss_item_sk#23, ss_sales_price#24, d_date#28] Keys [2]: [ss_item_sk#23, d_date#28] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#24))] -(39) CometColumnarExchange +(36) CometExchange Input [3]: [ss_item_sk#23, d_date#28, sum#29] -Arguments: hashpartitioning(ss_item_sk#23, d_date#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ss_item_sk#23, d_date#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(40) CometHashAggregate +(37) CometHashAggregate Input [3]: [ss_item_sk#23, d_date#28, sum#29] Keys [2]: [ss_item_sk#23, d_date#28] Functions [1]: [sum(UnscaledValue(ss_sales_price#24))] -(41) CometColumnarExchange +(38) CometExchange Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] -Arguments: hashpartitioning(ss_item_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(ss_item_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(42) CometSort +(39) CometSort Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] Arguments: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23], [ss_item_sk#23 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST] -(43) ColumnarToRow [codegen id : 6] +(40) ColumnarToRow [codegen id : 7] Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] -(44) Window +(41) Window Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] Arguments: [row_number() windowspecdefinition(ss_item_sk#23, d_date#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#32], [ss_item_sk#23], [d_date#28 ASC NULLS FIRST] -(45) Project [codegen id : 9] +(42) Project [codegen id : 10] Output [4]: [item_sk#30, d_date#28, sumss#31, rk#32] Input [5]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23, rk#32] -(46) ReusedExchange [Reuses operator id: 41] +(43) ReusedExchange [Reuses operator id: 38] Output [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] -(47) CometSort +(44) CometSort Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] Arguments: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34], [ss_item_sk#34 ASC NULLS FIRST, d_date#33 ASC NULLS FIRST] -(48) ColumnarToRow [codegen id : 7] +(45) ColumnarToRow [codegen id : 8] Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] -(49) Window +(46) Window Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] Arguments: [row_number() windowspecdefinition(ss_item_sk#34, d_date#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [ss_item_sk#34], [d_date#33 ASC NULLS FIRST] -(50) Project [codegen id : 8] +(47) Project [codegen id : 9] Output [3]: [item_sk#30 AS item_sk#36, sumss#31 AS sumss#37, rk#35] Input [5]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34, rk#35] -(51) BroadcastExchange +(48) BroadcastExchange Input [3]: [item_sk#36, sumss#37, rk#35] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -(52) BroadcastHashJoin [codegen id : 9] +(49) BroadcastHashJoin [codegen id : 10] Left keys [1]: [item_sk#30] Right keys [1]: [item_sk#36] Join type: Inner Join condition: (rk#32 >= rk#35) -(53) Project [codegen id : 9] +(50) Project [codegen id : 10] Output [4]: [item_sk#30, d_date#28, sumss#31, sumss#37] Input [7]: [item_sk#30, d_date#28, sumss#31, rk#32, item_sk#36, sumss#37, rk#35] -(54) HashAggregate [codegen id : 9] +(51) HashAggregate [codegen id : 10] Input [4]: [item_sk#30, d_date#28, sumss#31, sumss#37] Keys [3]: [item_sk#30, d_date#28, sumss#31] Functions [1]: [partial_sum(sumss#37)] Aggregate Attributes [2]: [sum#38, isEmpty#39] Results [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] -(55) RowToColumnar -Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] - -(56) CometColumnarExchange +(52) Exchange Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] -Arguments: hashpartitioning(item_sk#30, d_date#28, sumss#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: hashpartitioning(item_sk#30, d_date#28, sumss#31, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(57) ColumnarToRow [codegen id : 10] -Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] - -(58) HashAggregate [codegen id : 10] +(53) HashAggregate [codegen id : 11] Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] Keys [3]: [item_sk#30, d_date#28, sumss#31] Functions [1]: [sum(sumss#37)] Aggregate Attributes [1]: [sum(sumss#37)#42] Results [3]: [item_sk#30, d_date#28, sum(sumss#37)#42 AS cume_sales#43] -(59) RowToColumnar -Input [3]: [item_sk#30, d_date#28, cume_sales#43] - -(60) CometColumnarExchange +(54) Exchange Input [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: hashpartitioning(item_sk#30, d_date#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(item_sk#30, d_date#28, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(61) CometSort +(55) Sort [codegen id : 12] Input [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#30, d_date#28, cume_sales#43], [item_sk#30 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST] +Arguments: [item_sk#30 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST], false, 0 -(62) CometSortMergeJoin -Left output [3]: [item_sk#9, d_date#6, cume_sales#22] -Right output [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#9, d_date#6], [item_sk#30, d_date#28], FullOuter +(56) SortMergeJoin [codegen id : 13] +Left keys [2]: [item_sk#9, d_date#6] +Right keys [2]: [item_sk#30, d_date#28] +Join type: FullOuter +Join condition: None -(63) CometFilter +(57) Filter [codegen id : 13] Input [6]: [item_sk#9, d_date#6, cume_sales#22, item_sk#30, d_date#28, cume_sales#43] Condition : isnotnull(CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END) -(64) CometProject +(58) Project [codegen id : 13] +Output [4]: [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END AS item_sk#44, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#28 END AS d_date#45, cume_sales#22 AS web_sales#46, cume_sales#43 AS store_sales#47] Input [6]: [item_sk#9, d_date#6, cume_sales#22, item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END AS item_sk#44, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#28 END AS d_date#45, cume_sales#22 AS web_sales#46, cume_sales#43 AS store_sales#47] -(65) CometColumnarExchange +(59) Exchange Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: hashpartitioning(item_sk#44, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(item_sk#44, 5), ENSURE_REQUIREMENTS, [plan_id=11] -(66) CometSort +(60) Sort [codegen id : 14] Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] +Arguments: [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], false, 0 -(67) ColumnarToRow [codegen id : 11] -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] - -(68) Window +(61) Window Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#48], [item_sk#44], [d_date#45 ASC NULLS FIRST] -(69) ReusedExchange [Reuses operator id: 65] +(62) ReusedExchange [Reuses operator id: 59] Output [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -(70) CometSort -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] - -(71) ColumnarToRow [codegen id : 22] +(63) Sort [codegen id : 28] Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], false, 0 -(72) Window +(64) Window Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#49], [item_sk#44], [d_date#45 ASC NULLS FIRST] -(73) Project [codegen id : 23] +(65) Project [codegen id : 29] Output [4]: [item_sk#44 AS item_sk#50, web_sales#46 AS web_sales#51, store_sales#47 AS store_sales#52, rk#49] Input [5]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#49] -(74) BroadcastExchange +(66) BroadcastExchange Input [4]: [item_sk#50, web_sales#51, store_sales#52, rk#49] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] -(75) BroadcastHashJoin [codegen id : 24] +(67) BroadcastHashJoin [codegen id : 30] Left keys [1]: [item_sk#44] Right keys [1]: [item_sk#50] Join type: Inner Join condition: (rk#48 >= rk#49) -(76) Project [codegen id : 24] +(68) Project [codegen id : 30] Output [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] Input [9]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#48, item_sk#50, web_sales#51, store_sales#52, rk#49] -(77) HashAggregate [codegen id : 24] +(69) HashAggregate [codegen id : 30] Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] Functions [2]: [partial_max(web_sales#51), partial_max(store_sales#52)] Aggregate Attributes [2]: [max#53, max#54] Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#55, max#56] -(78) HashAggregate [codegen id : 24] +(70) HashAggregate [codegen id : 30] Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#55, max#56] Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] Functions [2]: [max(web_sales#51), max(store_sales#52)] Aggregate Attributes [2]: [max(web_sales#51)#57, max(store_sales#52)#58] Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max(web_sales#51)#57 AS web_cumulative#59, max(store_sales#52)#58 AS store_cumulative#60] -(79) Filter [codegen id : 24] +(71) Filter [codegen id : 30] Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] Condition : ((isnotnull(web_cumulative#59) AND isnotnull(store_cumulative#60)) AND (web_cumulative#59 > store_cumulative#60)) -(80) TakeOrderedAndProject +(72) TakeOrderedAndProject Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] Arguments: 100, [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (85) -+- * ColumnarToRow (84) - +- CometProject (83) - +- CometFilter (82) - +- CometScan parquet spark_catalog.default.date_dim (81) +BroadcastExchange (77) ++- * ColumnarToRow (76) + +- CometProject (75) + +- CometFilter (74) + +- CometScan parquet spark_catalog.default.date_dim (73) -(81) Scan parquet spark_catalog.default.date_dim +(73) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(82) CometFilter +(74) CometFilter Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5)) -(83) CometProject +(75) CometProject Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] -(84) ColumnarToRow [codegen id : 1] +(76) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#5, d_date#6] -(85) BroadcastExchange +(77) BroadcastExchange Input [2]: [d_date_sk#5, d_date#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] -Subquery:2 Hosting operator id = 33 Hosting Expression = ss_sold_date_sk#25 IN dynamicpruning#4 +Subquery:2 Hosting operator id = 30 Hosting Expression = ss_sold_date_sk#25 IN dynamicpruning#4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt index 7fe01fcc8..f8e1ba235 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q51a/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (24) + WholeStageCodegen (30) Filter [web_cumulative,store_cumulative] HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] @@ -7,23 +7,23 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store BroadcastHashJoin [item_sk,item_sk,rk,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (11) - ColumnarToRow + WholeStageCodegen (14) + Sort [item_sk,d_date] InputAdapter - CometSort [item_sk,d_date,web_sales,store_sales] - CometColumnarExchange [item_sk] #1 - CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] - CometFilter [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] - CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #2 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [item_sk,d_date,sumws] #3 - RowToColumnar + Exchange [item_sk] #1 + WholeStageCodegen (13) + Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] + Filter [item_sk,item_sk] + SortMergeJoin [item_sk,d_date,item_sk,d_date] + InputAdapter + WholeStageCodegen (6) + Sort [item_sk,d_date] + InputAdapter + Exchange [item_sk,d_date] #2 + WholeStageCodegen (5) + HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] + InputAdapter + Exchange [item_sk,d_date,sumws] #3 WholeStageCodegen (4) HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] Project [item_sk,d_date,sumws,sumws] @@ -35,9 +35,9 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store ColumnarToRow InputAdapter CometSort [item_sk,d_date,sumws,ws_item_sk] - CometColumnarExchange [ws_item_sk] #4 + CometExchange [ws_item_sk] #4 CometHashAggregate [item_sk,d_date,sumws,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] - CometColumnarExchange [ws_item_sk,d_date] #5 + CometExchange [ws_item_sk,d_date] #5 CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] CometProject [ws_item_sk,ws_sales_price,d_date] CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] @@ -66,29 +66,29 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometSort [item_sk,d_date,sumws,ws_item_sk] ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #9 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [item_sk,d_date,sumss] #10 - RowToColumnar - WholeStageCodegen (9) + InputAdapter + WholeStageCodegen (12) + Sort [item_sk,d_date] + InputAdapter + Exchange [item_sk,d_date] #9 + WholeStageCodegen (11) + HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] + InputAdapter + Exchange [item_sk,d_date,sumss] #10 + WholeStageCodegen (10) HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] Project [item_sk,d_date,sumss,sumss] BroadcastHashJoin [item_sk,item_sk,rk,rk] Project [item_sk,d_date,sumss,rk] InputAdapter Window [ss_item_sk,d_date] - WholeStageCodegen (6) + WholeStageCodegen (7) ColumnarToRow InputAdapter CometSort [item_sk,d_date,sumss,ss_item_sk] - CometColumnarExchange [ss_item_sk] #11 + CometExchange [ss_item_sk] #11 CometHashAggregate [item_sk,d_date,sumss,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [ss_item_sk,d_date] #12 + CometExchange [ss_item_sk,d_date] #12 CometHashAggregate [ss_item_sk,d_date,sum,ss_sales_price] CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] @@ -98,23 +98,22 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store ReusedExchange [d_date_sk,d_date] #7 InputAdapter BroadcastExchange #13 - WholeStageCodegen (8) + WholeStageCodegen (9) Project [item_sk,sumss,rk] InputAdapter Window [ss_item_sk,d_date] - WholeStageCodegen (7) + WholeStageCodegen (8) ColumnarToRow InputAdapter CometSort [item_sk,d_date,sumss,ss_item_sk] ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #11 InputAdapter BroadcastExchange #14 - WholeStageCodegen (23) + WholeStageCodegen (29) Project [item_sk,web_sales,store_sales,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (22) - ColumnarToRow + WholeStageCodegen (28) + Sort [item_sk,d_date] InputAdapter - CometSort [item_sk,d_date,web_sales,store_sales] - ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 + ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt index 59b4666bf..01a5d7c56 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/explain.txt @@ -11,9 +11,9 @@ TakeOrderedAndProject (45) : : +- Window (24) : : +- * ColumnarToRow (23) : : +- CometSort (22) - : : +- CometColumnarExchange (21) + : : +- CometExchange (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -135,18 +135,18 @@ Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#10, d_moy#11, cc_n Keys [5]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -(19) CometColumnarExchange +(19) CometExchange Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (20) CometHashAggregate Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#14] Keys [5]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -(21) CometColumnarExchange +(21) CometExchange Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#15, _w0#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (22) CometSort Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#15, _w0#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt index ed9e697f3..539839af7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q57/simplified.txt @@ -16,9 +16,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m ColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,cc_name] #1 + CometExchange [i_category,i_brand,cc_name] #1 CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(cs_sales_price))] - CometColumnarExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum,cs_sales_price] CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt index 3f4892277..02eab7ed1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/explain.txt @@ -1,97 +1,87 @@ == Physical Plan == -* ColumnarToRow (93) -+- CometTakeOrderedAndProject (92) - +- CometHashAggregate (91) - +- CometColumnarExchange (90) - +- RowToColumnar (89) - +- * HashAggregate (88) - +- Union (87) - :- * HashAggregate (70) - : +- * ColumnarToRow (69) - : +- CometColumnarExchange (68) - : +- RowToColumnar (67) - : +- * HashAggregate (66) - : +- Union (65) - : :- * HashAggregate (22) - : : +- * ColumnarToRow (21) - : : +- CometColumnarExchange (20) - : : +- CometHashAggregate (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometUnion (7) - : : : : :- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.store_returns (4) - : : : +- CometBroadcastExchange (11) - : : : +- CometProject (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometScan parquet spark_catalog.default.store (14) - : :- * HashAggregate (41) - : : +- * ColumnarToRow (40) - : : +- CometColumnarExchange (39) - : : +- CometHashAggregate (38) - : : +- CometProject (37) - : : +- CometBroadcastHashJoin (36) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometUnion (29) - : : : : :- CometProject (25) - : : : : : +- CometFilter (24) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) - : : : : +- CometProject (28) - : : : : +- CometFilter (27) - : : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) - : : : +- ReusedExchange (30) - : : +- CometBroadcastExchange (35) - : : +- CometFilter (34) - : : +- CometScan parquet spark_catalog.default.catalog_page (33) - : +- * HashAggregate (64) - : +- * ColumnarToRow (63) - : +- CometColumnarExchange (62) - : +- CometHashAggregate (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometUnion (52) - : : : :- CometProject (44) - : : : : +- CometFilter (43) - : : : : +- CometScan parquet spark_catalog.default.web_sales (42) - : : : +- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometBroadcastExchange (46) - : : : : +- CometScan parquet spark_catalog.default.web_returns (45) - : : : +- CometProject (49) - : : : +- CometFilter (48) - : : : +- CometScan parquet spark_catalog.default.web_sales (47) - : : +- ReusedExchange (53) - : +- CometBroadcastExchange (58) - : +- CometFilter (57) - : +- CometScan parquet spark_catalog.default.web_site (56) - :- * HashAggregate (78) - : +- * ColumnarToRow (77) - : +- CometColumnarExchange (76) - : +- RowToColumnar (75) - : +- * HashAggregate (74) - : +- * HashAggregate (73) - : +- * ColumnarToRow (72) - : +- ReusedExchange (71) - +- * HashAggregate (86) - +- * ColumnarToRow (85) - +- CometColumnarExchange (84) - +- RowToColumnar (83) - +- * HashAggregate (82) - +- * HashAggregate (81) - +- * ColumnarToRow (80) - +- ReusedExchange (79) +TakeOrderedAndProject (83) ++- * HashAggregate (82) + +- Exchange (81) + +- * HashAggregate (80) + +- Union (79) + :- * HashAggregate (68) + : +- Exchange (67) + : +- * HashAggregate (66) + : +- Union (65) + : :- * HashAggregate (22) + : : +- * ColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometUnion (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan parquet spark_catalog.default.store (14) + : :- * HashAggregate (41) + : : +- * ColumnarToRow (40) + : : +- CometExchange (39) + : : +- CometHashAggregate (38) + : : +- CometProject (37) + : : +- CometBroadcastHashJoin (36) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometUnion (29) + : : : : :- CometProject (25) + : : : : : +- CometFilter (24) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) + : : : : +- CometProject (28) + : : : : +- CometFilter (27) + : : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) + : : : +- ReusedExchange (30) + : : +- CometBroadcastExchange (35) + : : +- CometFilter (34) + : : +- CometScan parquet spark_catalog.default.catalog_page (33) + : +- * HashAggregate (64) + : +- * ColumnarToRow (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometUnion (52) + : : : :- CometProject (44) + : : : : +- CometFilter (43) + : : : : +- CometScan parquet spark_catalog.default.web_sales (42) + : : : +- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastExchange (46) + : : : : +- CometScan parquet spark_catalog.default.web_returns (45) + : : : +- CometProject (49) + : : : +- CometFilter (48) + : : : +- CometScan parquet spark_catalog.default.web_sales (47) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometScan parquet spark_catalog.default.web_site (56) + :- * HashAggregate (73) + : +- Exchange (72) + : +- * HashAggregate (71) + : +- * HashAggregate (70) + : +- ReusedExchange (69) + +- * HashAggregate (78) + +- Exchange (77) + +- * HashAggregate (76) + +- * HashAggregate (75) + +- ReusedExchange (74) (1) Scan parquet spark_catalog.default.store_sales @@ -187,9 +177,9 @@ Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#25] Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] -(20) CometColumnarExchange +(20) CometExchange Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] -Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) ColumnarToRow [codegen id : 1] Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] @@ -278,9 +268,9 @@ Input [5]: [sales_price#46, profit#47, return_amt#48, net_loss#49, cp_catalog_pa Keys [1]: [cp_catalog_page_id#62] Functions [4]: [partial_sum(UnscaledValue(sales_price#46)), partial_sum(UnscaledValue(return_amt#48)), partial_sum(UnscaledValue(profit#47)), partial_sum(UnscaledValue(net_loss#49))] -(39) CometColumnarExchange +(39) CometExchange Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] -Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) ColumnarToRow [codegen id : 2] Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] @@ -388,9 +378,9 @@ Input [5]: [sales_price#83, profit#84, return_amt#85, net_loss#86, web_site_id#1 Keys [1]: [web_site_id#104] Functions [4]: [partial_sum(UnscaledValue(sales_price#83)), partial_sum(UnscaledValue(return_amt#85)), partial_sum(UnscaledValue(profit#84)), partial_sum(UnscaledValue(net_loss#86))] -(62) CometColumnarExchange +(62) CometExchange Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] -Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (63) ColumnarToRow [codegen id : 3] Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] @@ -411,154 +401,126 @@ Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(prof Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] Results [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -(67) RowToColumnar +(67) Exchange Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(68) CometColumnarExchange -Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(69) ColumnarToRow [codegen id : 5] -Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] - -(70) HashAggregate [codegen id : 5] +(68) HashAggregate [codegen id : 5] Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] Keys [2]: [channel#34, id#35] Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] Results [5]: [channel#34, id#35, cast(sum(sales#36)#130 as decimal(37,2)) AS sales#133, cast(sum(returns#37)#131 as decimal(37,2)) AS returns#134, cast(sum(profit#38)#132 as decimal(38,2)) AS profit#135] -(71) ReusedExchange [Reuses operator id: 68] +(69) ReusedExchange [Reuses operator id: 67] Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -(72) ColumnarToRow [codegen id : 10] -Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] - -(73) HashAggregate [codegen id : 10] +(70) HashAggregate [codegen id : 10] Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] Keys [2]: [channel#34, id#35] Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] Results [4]: [channel#34, sum(sales#36)#130 AS sales#136, sum(returns#37)#131 AS returns#137, sum(profit#38)#132 AS profit#138] -(74) HashAggregate [codegen id : 10] +(71) HashAggregate [codegen id : 10] Input [4]: [channel#34, sales#136, returns#137, profit#138] Keys [1]: [channel#34] Functions [3]: [partial_sum(sales#136), partial_sum(returns#137), partial_sum(profit#138)] Aggregate Attributes [6]: [sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] Results [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] -(75) RowToColumnar +(72) Exchange Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(76) CometColumnarExchange -Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] -Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(77) ColumnarToRow [codegen id : 11] -Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] - -(78) HashAggregate [codegen id : 11] +(73) HashAggregate [codegen id : 11] Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] Keys [1]: [channel#34] Functions [3]: [sum(sales#136), sum(returns#137), sum(profit#138)] Aggregate Attributes [3]: [sum(sales#136)#151, sum(returns#137)#152, sum(profit#138)#153] Results [5]: [channel#34, null AS id#154, sum(sales#136)#151 AS sum(sales)#155, sum(returns#137)#152 AS sum(returns)#156, sum(profit#138)#153 AS sum(profit)#157] -(79) ReusedExchange [Reuses operator id: 68] +(74) ReusedExchange [Reuses operator id: 67] Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -(80) ColumnarToRow [codegen id : 16] -Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] - -(81) HashAggregate [codegen id : 16] +(75) HashAggregate [codegen id : 16] Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] Keys [2]: [channel#34, id#35] Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] Results [3]: [sum(sales#36)#130 AS sales#158, sum(returns#37)#131 AS returns#159, sum(profit#38)#132 AS profit#160] -(82) HashAggregate [codegen id : 16] +(76) HashAggregate [codegen id : 16] Input [3]: [sales#158, returns#159, profit#160] Keys: [] Functions [3]: [partial_sum(sales#158), partial_sum(returns#159), partial_sum(profit#160)] Aggregate Attributes [6]: [sum#161, isEmpty#162, sum#163, isEmpty#164, sum#165, isEmpty#166] Results [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] -(83) RowToColumnar -Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] - -(84) CometColumnarExchange +(77) Exchange Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(85) ColumnarToRow [codegen id : 17] -Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] - -(86) HashAggregate [codegen id : 17] +(78) HashAggregate [codegen id : 17] Input [6]: [sum#167, isEmpty#168, sum#169, isEmpty#170, sum#171, isEmpty#172] Keys: [] Functions [3]: [sum(sales#158), sum(returns#159), sum(profit#160)] Aggregate Attributes [3]: [sum(sales#158)#173, sum(returns#159)#174, sum(profit#160)#175] Results [5]: [null AS channel#176, null AS id#177, sum(sales#158)#173 AS sum(sales)#178, sum(returns#159)#174 AS sum(returns)#179, sum(profit#160)#175 AS sum(profit)#180] -(87) Union +(79) Union -(88) HashAggregate [codegen id : 18] +(80) HashAggregate [codegen id : 18] Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] Aggregate Attributes: [] Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] -(89) RowToColumnar +(81) Exchange Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(90) CometColumnarExchange -Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] -Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(91) CometHashAggregate +(82) HashAggregate [codegen id : 19] Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] -(92) CometTakeOrderedAndProject -Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#34 ASC NULLS FIRST,id#35 ASC NULLS FIRST], output=[channel#34,id#35,sales#133,returns#134,profit#135]), [channel#34, id#35, sales#133, returns#134, profit#135], 100, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] - -(93) ColumnarToRow [codegen id : 19] +(83) TakeOrderedAndProject Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: 100, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (98) -+- * ColumnarToRow (97) - +- CometProject (96) - +- CometFilter (95) - +- CometScan parquet spark_catalog.default.date_dim (94) +BroadcastExchange (88) ++- * ColumnarToRow (87) + +- CometProject (86) + +- CometFilter (85) + +- CometScan parquet spark_catalog.default.date_dim (84) -(94) Scan parquet spark_catalog.default.date_dim +(84) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] ReadSchema: struct -(95) CometFilter +(85) CometFilter Input [2]: [d_date_sk#22, d_date#23] Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-08-18)) AND isnotnull(d_date_sk#22)) -(96) CometProject +(86) CometProject Input [2]: [d_date_sk#22, d_date#23] Arguments: [d_date_sk#22], [d_date_sk#22] -(97) ColumnarToRow [codegen id : 1] +(87) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#22] -(98) BroadcastExchange +(88) BroadcastExchange Input [1]: [d_date_sk#22] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/simplified.txt index 00dd3c480..751b3dc24 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q5a/simplified.txt @@ -1,128 +1,118 @@ -WholeStageCodegen (19) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,id,sales,returns,profit] - CometHashAggregate [channel,id,sales,returns,profit] - CometColumnarExchange [channel,id,sales,returns,profit] #1 - RowToColumnar - WholeStageCodegen (18) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id] #2 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_id] #3 - CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [cp_catalog_page_id] #7 - CometHashAggregate [cp_catalog_page_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [page_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #8 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [web_site_id] #9 - CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #10 - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [web_site_sk,web_site_id] #11 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - WholeStageCodegen (11) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel] #12 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #13 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (19) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Exchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #7 + CometHashAggregate [cp_catalog_page_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #8 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [web_site_id] #9 + CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #10 + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #11 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel] #12 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange #13 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/explain.txt index b662f22fb..358d4995e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (37) +- CometFilter (36) +- CometHashAggregate (35) - +- CometColumnarExchange (34) + +- CometExchange (34) +- CometHashAggregate (33) +- CometProject (32) +- CometBroadcastHashJoin (31) @@ -33,7 +33,7 @@ +- CometBroadcastExchange (27) +- CometFilter (26) +- CometHashAggregate (25) - +- CometColumnarExchange (24) + +- CometExchange (24) +- CometHashAggregate (23) +- CometFilter (22) +- CometScan parquet spark_catalog.default.item (21) @@ -154,9 +154,9 @@ Input [2]: [i_current_price#16, i_category#17] Keys [1]: [i_category#17] Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] -(24) CometColumnarExchange +(24) CometExchange Input [3]: [i_category#17, sum#18, count#19] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (25) CometHashAggregate Input [3]: [i_category#17, sum#18, count#19] @@ -198,9 +198,9 @@ Input [1]: [ca_state#2] Keys [1]: [ca_state#2] Functions [1]: [partial_count(1)] -(34) CometColumnarExchange +(34) CometExchange Input [2]: [ca_state#2, count#21] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (35) CometHashAggregate Input [2]: [ca_state#2, count#21] @@ -253,7 +253,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 40 Hosting Expression = Subquery scalar-subquery#11, [id=#12] * ColumnarToRow (50) +- CometHashAggregate (49) - +- CometColumnarExchange (48) + +- CometExchange (48) +- CometHashAggregate (47) +- CometProject (46) +- CometFilter (45) @@ -280,9 +280,9 @@ Input [1]: [d_month_seq#24] Keys [1]: [d_month_seq#24] Functions: [] -(48) CometColumnarExchange +(48) CometExchange Input [1]: [d_month_seq#24] -Arguments: hashpartitioning(d_month_seq#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(d_month_seq#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (49) CometHashAggregate Input [1]: [d_month_seq#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt index b95744087..580f668ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q6/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [state,cnt,ca_state] CometFilter [state,cnt,ca_state] CometHashAggregate [state,cnt,ca_state,count,count(1)] - CometColumnarExchange [ca_state] #1 + CometExchange [ca_state] #1 CometHashAggregate [ca_state,count] CometProject [ca_state] CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] @@ -34,7 +34,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [d_month_seq] - CometColumnarExchange [d_month_seq] #5 + CometExchange [d_month_seq] #5 CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] @@ -53,7 +53,7 @@ WholeStageCodegen (1) CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] - CometColumnarExchange [i_category] #9 + CometExchange [i_category] #9 CometHashAggregate [i_category,sum,count,i_current_price] CometFilter [i_current_price,i_category] CometScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt index 174590641..bd75cbb40 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/explain.txt @@ -7,7 +7,7 @@ +- * SortMergeJoin Inner (171) :- * ColumnarToRow (101) : +- CometSort (100) - : +- CometColumnarExchange (99) + : +- CometExchange (99) : +- CometHashAggregate (98) : +- CometHashAggregate (97) : +- CometProject (96) @@ -43,7 +43,7 @@ : : : : : : : : : : : : : : : :- CometProject (30) : : : : : : : : : : : : : : : : +- CometSortMergeJoin (29) : : : : : : : : : : : : : : : : :- CometSort (10) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (9) + : : : : : : : : : : : : : : : : : +- CometExchange (9) : : : : : : : : : : : : : : : : : +- CometProject (8) : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) @@ -56,17 +56,17 @@ : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) : : : : : : : : : : : : : : : : +- CometHashAggregate (25) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (24) + : : : : : : : : : : : : : : : : +- CometExchange (24) : : : : : : : : : : : : : : : : +- CometHashAggregate (23) : : : : : : : : : : : : : : : : +- CometProject (22) : : : : : : : : : : : : : : : : +- CometSortMergeJoin (21) : : : : : : : : : : : : : : : : :- CometSort (15) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (14) + : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (19) + : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (16) @@ -108,7 +108,7 @@ : +- CometScan parquet spark_catalog.default.item (91) +- * ColumnarToRow (170) +- CometSort (169) - +- CometColumnarExchange (168) + +- CometExchange (168) +- CometHashAggregate (167) +- CometHashAggregate (166) +- CometProject (165) @@ -144,7 +144,7 @@ : : : : : : : : : : : : : : :- CometProject (118) : : : : : : : : : : : : : : : +- CometSortMergeJoin (117) : : : : : : : : : : : : : : : :- CometSort (111) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (110) + : : : : : : : : : : : : : : : : +- CometExchange (110) : : : : : : : : : : : : : : : : +- CometProject (109) : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (108) : : : : : : : : : : : : : : : : :- CometBroadcastExchange (104) @@ -217,9 +217,9 @@ Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number# Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(9) CometColumnarExchange +(9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -240,9 +240,9 @@ Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -(14) CometColumnarExchange +(14) CometExchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (15) CometSort Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] @@ -263,9 +263,9 @@ Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -(19) CometColumnarExchange +(19) CometExchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] @@ -285,9 +285,9 @@ Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reverse Keys [1]: [cs_item_sk#17] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] -(24) CometColumnarExchange +(24) CometExchange Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometHashAggregate Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] @@ -629,9 +629,9 @@ Input [19]: [i_product_name#71, i_item_sk#68, s_store_name#35, s_zip#36, ca_stre Keys [15]: [i_product_name#71, i_item_sk#68, s_store_name#35, s_zip#36, ca_street_number#57, ca_street_name#58, ca_city#59, ca_zip#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, d_year#33, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -(99) CometColumnarExchange +(99) CometExchange Input [17]: [product_name#76, item_sk#77, store_name#78, store_zip#79, b_street_number#80, b_streen_name#81, b_city#82, b_zip#83, c_street_number#84, c_street_name#85, c_city#86, c_zip#87, syear#88, cnt#89, s1#90, s2#91, s3#92] -Arguments: hashpartitioning(item_sk#77, store_name#78, store_zip#79, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(item_sk#77, store_name#78, store_zip#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (100) CometSort Input [17]: [product_name#76, item_sk#77, store_name#78, store_zip#79, b_street_number#80, b_streen_name#81, b_city#82, b_zip#83, c_street_number#84, c_street_name#85, c_city#86, c_zip#87, syear#88, cnt#89, s1#90, s2#91, s3#92] @@ -680,9 +680,9 @@ Arguments: [ss_item_sk#93, ss_ticket_number#100], [sr_item_sk#106, sr_ticket_num Input [14]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_ticket_number#100, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104, sr_item_sk#106, sr_ticket_number#107] Arguments: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104], [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] -(110) CometColumnarExchange +(110) CometExchange Input [11]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] -Arguments: hashpartitioning(ss_item_sk#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ss_item_sk#93, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (111) CometSort Input [11]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] @@ -919,9 +919,9 @@ Input [19]: [i_product_name#154, i_item_sk#153, s_store_name#120, s_zip#121, ca_ Keys [15]: [i_product_name#154, i_item_sk#153, s_store_name#120, s_zip#121, ca_street_number#142, ca_street_name#143, ca_city#144, ca_zip#145, ca_street_number#147, ca_street_name#148, ca_city#149, ca_zip#150, d_year#118, d_year#129, d_year#131] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#101)), sum(UnscaledValue(ss_list_price#102)), sum(UnscaledValue(ss_coupon_amt#103))] -(168) CometColumnarExchange +(168) CometExchange Input [8]: [item_sk#158, store_name#159, store_zip#160, syear#161, cnt#162, s1#163, s2#164, s3#165] -Arguments: hashpartitioning(item_sk#158, store_name#159, store_zip#160, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(item_sk#158, store_name#159, store_zip#160, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (169) CometSort Input [8]: [item_sk#158, store_name#159, store_zip#160, syear#161, cnt#162, s1#163, s2#164, s3#165] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt index 70d0e66f8..911f7fcbd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q64/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (4) ColumnarToRow InputAdapter CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #2 + CometExchange [item_sk,store_name,store_zip] #2 CometHashAggregate [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -48,7 +48,7 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #3 + CometExchange [ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 @@ -68,17 +68,17 @@ WholeStageCodegen (4) CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - CometColumnarExchange [cs_item_sk] #6 + CometExchange [cs_item_sk] #6 CometHashAggregate [cs_item_sk,sum,sum,isEmpty,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometColumnarExchange [cs_item_sk,cs_order_number] #7 + CometExchange [cs_item_sk,cs_order_number] #7 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometColumnarExchange [cr_item_sk,cr_order_number] #8 + CometExchange [cr_item_sk,cr_order_number] #8 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] @@ -123,7 +123,7 @@ WholeStageCodegen (4) ColumnarToRow InputAdapter CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #19 + CometExchange [item_sk,store_name,store_zip] #19 CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -159,7 +159,7 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #20 + CometExchange [ss_item_sk] #20 CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #21 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt index f21890665..5ae73a0f7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/explain.txt @@ -1,77 +1,75 @@ == Physical Plan == -TakeOrderedAndProject (73) -+- * Filter (72) - +- Window (71) - +- WindowGroupLimit (70) - +- * ColumnarToRow (69) - +- CometSort (68) - +- CometColumnarExchange (67) - +- RowToColumnar (66) - +- WindowGroupLimit (65) - +- * ColumnarToRow (64) - +- CometSort (63) - +- CometUnion (62) - :- CometHashAggregate (21) - : +- CometColumnarExchange (20) - : +- CometHashAggregate (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.item (14) - :- CometHashAggregate (26) - : +- CometColumnarExchange (25) - : +- CometHashAggregate (24) - : +- CometHashAggregate (23) - : +- ReusedExchange (22) - :- CometHashAggregate (31) - : +- CometColumnarExchange (30) - : +- CometHashAggregate (29) - : +- CometHashAggregate (28) - : +- ReusedExchange (27) - :- CometHashAggregate (36) - : +- CometColumnarExchange (35) - : +- CometHashAggregate (34) - : +- CometHashAggregate (33) - : +- ReusedExchange (32) - :- CometHashAggregate (41) - : +- CometColumnarExchange (40) - : +- CometHashAggregate (39) - : +- CometHashAggregate (38) - : +- ReusedExchange (37) - :- CometHashAggregate (46) - : +- CometColumnarExchange (45) - : +- CometHashAggregate (44) - : +- CometHashAggregate (43) - : +- ReusedExchange (42) - :- CometHashAggregate (51) - : +- CometColumnarExchange (50) - : +- CometHashAggregate (49) - : +- CometHashAggregate (48) - : +- ReusedExchange (47) - :- CometHashAggregate (56) - : +- CometColumnarExchange (55) - : +- CometHashAggregate (54) - : +- CometHashAggregate (53) - : +- ReusedExchange (52) - +- CometHashAggregate (61) - +- CometColumnarExchange (60) - +- CometHashAggregate (59) - +- CometHashAggregate (58) - +- ReusedExchange (57) +TakeOrderedAndProject (71) ++- * Filter (70) + +- Window (69) + +- WindowGroupLimit (68) + +- * Sort (67) + +- Exchange (66) + +- WindowGroupLimit (65) + +- * ColumnarToRow (64) + +- CometSort (63) + +- CometUnion (62) + :- CometHashAggregate (21) + : +- CometExchange (20) + : +- CometHashAggregate (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan parquet spark_catalog.default.item (14) + :- CometHashAggregate (26) + : +- CometExchange (25) + : +- CometHashAggregate (24) + : +- CometHashAggregate (23) + : +- ReusedExchange (22) + :- CometHashAggregate (31) + : +- CometExchange (30) + : +- CometHashAggregate (29) + : +- CometHashAggregate (28) + : +- ReusedExchange (27) + :- CometHashAggregate (36) + : +- CometExchange (35) + : +- CometHashAggregate (34) + : +- CometHashAggregate (33) + : +- ReusedExchange (32) + :- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometHashAggregate (38) + : +- ReusedExchange (37) + :- CometHashAggregate (46) + : +- CometExchange (45) + : +- CometHashAggregate (44) + : +- CometHashAggregate (43) + : +- ReusedExchange (42) + :- CometHashAggregate (51) + : +- CometExchange (50) + : +- CometHashAggregate (49) + : +- CometHashAggregate (48) + : +- ReusedExchange (47) + :- CometHashAggregate (56) + : +- CometExchange (55) + : +- CometHashAggregate (54) + : +- CometHashAggregate (53) + : +- ReusedExchange (52) + +- CometHashAggregate (61) + +- CometExchange (60) + +- CometHashAggregate (59) + +- CometHashAggregate (58) + +- ReusedExchange (57) (1) Scan parquet spark_catalog.default.store_sales @@ -167,9 +165,9 @@ Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_st Keys [8]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(20) CometColumnarExchange +(20) CometExchange Input [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sum#19, isEmpty#20] -Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sum#19, isEmpty#20] @@ -189,9 +187,9 @@ Input [8]: [i_category#21, i_class#22, i_brand#23, i_product_name#24, d_year#25, Keys [7]: [i_category#21, i_class#22, i_brand#23, i_product_name#24, d_year#25, d_qoy#26, d_moy#27] Functions [1]: [partial_sum(sumsales#33)] -(25) CometColumnarExchange +(25) CometExchange Input [9]: [i_category#21, i_class#22, i_brand#23, i_product_name#24, d_year#25, d_qoy#26, d_moy#27, sum#34, isEmpty#35] -Arguments: hashpartitioning(i_category#21, i_class#22, i_brand#23, i_product_name#24, d_year#25, d_qoy#26, d_moy#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#21, i_class#22, i_brand#23, i_product_name#24, d_year#25, d_qoy#26, d_moy#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (26) CometHashAggregate Input [9]: [i_category#21, i_class#22, i_brand#23, i_product_name#24, d_year#25, d_qoy#26, d_moy#27, sum#34, isEmpty#35] @@ -211,9 +209,9 @@ Input [7]: [i_category#36, i_class#37, i_brand#38, i_product_name#39, d_year#40, Keys [6]: [i_category#36, i_class#37, i_brand#38, i_product_name#39, d_year#40, d_qoy#41] Functions [1]: [partial_sum(sumsales#48)] -(30) CometColumnarExchange +(30) CometExchange Input [8]: [i_category#36, i_class#37, i_brand#38, i_product_name#39, d_year#40, d_qoy#41, sum#49, isEmpty#50] -Arguments: hashpartitioning(i_category#36, i_class#37, i_brand#38, i_product_name#39, d_year#40, d_qoy#41, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_category#36, i_class#37, i_brand#38, i_product_name#39, d_year#40, d_qoy#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (31) CometHashAggregate Input [8]: [i_category#36, i_class#37, i_brand#38, i_product_name#39, d_year#40, d_qoy#41, sum#49, isEmpty#50] @@ -233,9 +231,9 @@ Input [6]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, Keys [5]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55] Functions [1]: [partial_sum(sumsales#63)] -(35) CometColumnarExchange +(35) CometExchange Input [7]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, sum#64, isEmpty#65] -Arguments: hashpartitioning(i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (36) CometHashAggregate Input [7]: [i_category#51, i_class#52, i_brand#53, i_product_name#54, d_year#55, sum#64, isEmpty#65] @@ -255,9 +253,9 @@ Input [5]: [i_category#66, i_class#67, i_brand#68, i_product_name#69, sumsales#7 Keys [4]: [i_category#66, i_class#67, i_brand#68, i_product_name#69] Functions [1]: [partial_sum(sumsales#78)] -(40) CometColumnarExchange +(40) CometExchange Input [6]: [i_category#66, i_class#67, i_brand#68, i_product_name#69, sum#79, isEmpty#80] -Arguments: hashpartitioning(i_category#66, i_class#67, i_brand#68, i_product_name#69, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(i_category#66, i_class#67, i_brand#68, i_product_name#69, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (41) CometHashAggregate Input [6]: [i_category#66, i_class#67, i_brand#68, i_product_name#69, sum#79, isEmpty#80] @@ -277,9 +275,9 @@ Input [4]: [i_category#81, i_class#82, i_brand#83, sumsales#93] Keys [3]: [i_category#81, i_class#82, i_brand#83] Functions [1]: [partial_sum(sumsales#93)] -(45) CometColumnarExchange +(45) CometExchange Input [5]: [i_category#81, i_class#82, i_brand#83, sum#94, isEmpty#95] -Arguments: hashpartitioning(i_category#81, i_class#82, i_brand#83, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(i_category#81, i_class#82, i_brand#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (46) CometHashAggregate Input [5]: [i_category#81, i_class#82, i_brand#83, sum#94, isEmpty#95] @@ -299,9 +297,9 @@ Input [3]: [i_category#96, i_class#97, sumsales#108] Keys [2]: [i_category#96, i_class#97] Functions [1]: [partial_sum(sumsales#108)] -(50) CometColumnarExchange +(50) CometExchange Input [4]: [i_category#96, i_class#97, sum#109, isEmpty#110] -Arguments: hashpartitioning(i_category#96, i_class#97, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(i_category#96, i_class#97, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (51) CometHashAggregate Input [4]: [i_category#96, i_class#97, sum#109, isEmpty#110] @@ -321,9 +319,9 @@ Input [2]: [i_category#111, sumsales#123] Keys [1]: [i_category#111] Functions [1]: [partial_sum(sumsales#123)] -(55) CometColumnarExchange +(55) CometExchange Input [3]: [i_category#111, sum#124, isEmpty#125] -Arguments: hashpartitioning(i_category#111, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(i_category#111, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (56) CometHashAggregate Input [3]: [i_category#111, sum#124, isEmpty#125] @@ -343,9 +341,9 @@ Input [1]: [sumsales#138] Keys: [] Functions [1]: [partial_sum(sumsales#138)] -(60) CometColumnarExchange +(60) CometExchange Input [2]: [sum#139, isEmpty#140] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (61) CometHashAggregate Input [2]: [sum#139, isEmpty#140] @@ -374,65 +372,59 @@ Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141] Arguments: [i_category#17], [sumsales#141 DESC NULLS LAST], rank(sumsales#141), 100, Partial -(66) RowToColumnar +(66) Exchange Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141] +Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(67) CometColumnarExchange +(67) Sort [codegen id : 2] Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: [i_category#17 ASC NULLS FIRST, sumsales#141 DESC NULLS LAST], false, 0 -(68) CometSort -Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141] -Arguments: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141], [i_category#17 ASC NULLS FIRST, sumsales#141 DESC NULLS LAST] - -(69) ColumnarToRow [codegen id : 2] -Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141] - -(70) WindowGroupLimit +(68) WindowGroupLimit Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141] Arguments: [i_category#17], [sumsales#141 DESC NULLS LAST], rank(sumsales#141), 100, Final -(71) Window +(69) Window Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141] Arguments: [rank(sumsales#141) windowspecdefinition(i_category#17, sumsales#141 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#186], [i_category#17], [sumsales#141 DESC NULLS LAST] -(72) Filter [codegen id : 3] +(70) Filter [codegen id : 3] Input [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141, rk#186] Condition : (rk#186 <= 100) -(73) TakeOrderedAndProject +(71) TakeOrderedAndProject Input [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141, rk#186] Arguments: 100, [i_category#17 ASC NULLS FIRST, i_class#16 ASC NULLS FIRST, i_brand#15 ASC NULLS FIRST, i_product_name#18 ASC NULLS FIRST, d_year#9 ASC NULLS FIRST, d_qoy#11 ASC NULLS FIRST, d_moy#10 ASC NULLS FIRST, s_store_id#13 ASC NULLS FIRST, sumsales#141 ASC NULLS FIRST, rk#186 ASC NULLS FIRST], [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#141, rk#186] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (78) -+- * ColumnarToRow (77) - +- CometProject (76) - +- CometFilter (75) - +- CometScan parquet spark_catalog.default.date_dim (74) +BroadcastExchange (76) ++- * ColumnarToRow (75) + +- CometProject (74) + +- CometFilter (73) + +- CometScan parquet spark_catalog.default.date_dim (72) -(74) Scan parquet spark_catalog.default.date_dim +(72) Scan parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(75) CometFilter +(73) CometFilter Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1212)) AND (d_month_seq#8 <= 1223)) AND isnotnull(d_date_sk#7)) -(76) CometProject +(74) CometProject Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -(77) ColumnarToRow [codegen id : 1] +(75) ColumnarToRow [codegen id : 1] Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -(78) BroadcastExchange +(76) BroadcastExchange Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/simplified.txt index 2298c66db..4ff3d4215 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q67a/simplified.txt @@ -5,83 +5,81 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ Window [sumsales,i_category] WindowGroupLimit [i_category,sumsales] WholeStageCodegen (2) - ColumnarToRow + Sort [i_category,sumsales] InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometColumnarExchange [i_category] #1 - RowToColumnar - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,ss_sales_price,ss_quantity] - CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #5 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #7 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #8 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year] #9 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name] #10 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,i_product_name,sumsales,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class,i_brand] #11 - CometHashAggregate [i_category,i_class,i_brand,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,i_brand,sumsales,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class] #12 - CometHashAggregate [i_category,i_class,sum,isEmpty,sumsales] - CometHashAggregate [i_category,i_class,sumsales,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category] #13 - CometHashAggregate [i_category,sum,isEmpty,sumsales] - CometHashAggregate [i_category,sumsales,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange #14 - CometHashAggregate [sum,isEmpty,sumsales] - CometHashAggregate [sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + Exchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,ss_sales_price,ss_quantity] + CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #7 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty,sumsales] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #8 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty,sumsales] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #9 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty,sumsales] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] + CometExchange [i_category,i_class,i_brand,i_product_name] #10 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty,sumsales] + CometHashAggregate [i_category,i_class,i_brand,i_product_name,sumsales,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] + CometExchange [i_category,i_class,i_brand] #11 + CometHashAggregate [i_category,i_class,i_brand,sum,isEmpty,sumsales] + CometHashAggregate [i_category,i_class,i_brand,sumsales,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] + CometExchange [i_category,i_class] #12 + CometHashAggregate [i_category,i_class,sum,isEmpty,sumsales] + CometHashAggregate [i_category,i_class,sumsales,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] + CometExchange [i_category] #13 + CometHashAggregate [i_category,sum,isEmpty,sumsales] + CometHashAggregate [i_category,sumsales,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] + CometExchange #14 + CometHashAggregate [sum,isEmpty,sumsales] + CometHashAggregate [sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/explain.txt index 632d097a8..a77ebb126 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/explain.txt @@ -1,72 +1,62 @@ == Physical Plan == -TakeOrderedAndProject (68) -+- * Project (67) - +- Window (66) - +- * ColumnarToRow (65) - +- CometSort (64) - +- CometColumnarExchange (63) - +- CometHashAggregate (62) - +- CometColumnarExchange (61) - +- RowToColumnar (60) - +- * HashAggregate (59) - +- Union (58) - :- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometColumnarExchange (39) - : +- RowToColumnar (38) - : +- * HashAggregate (37) - : +- * Project (36) - : +- * BroadcastHashJoin Inner BuildRight (35) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- BroadcastExchange (34) - : +- * BroadcastHashJoin LeftSemi BuildRight (33) - : :- * ColumnarToRow (12) - : : +- CometFilter (11) - : : +- CometScan parquet spark_catalog.default.store (10) - : +- BroadcastExchange (32) - : +- * Project (31) - : +- * Filter (30) - : +- Window (29) - : +- WindowGroupLimit (28) - : +- * ColumnarToRow (27) - : +- CometSort (26) - : +- CometHashAggregate (25) - : +- CometColumnarExchange (24) - : +- CometHashAggregate (23) - : +- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store_sales (13) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.store (15) - : +- ReusedExchange (20) - :- * HashAggregate (49) - : +- * ColumnarToRow (48) - : +- CometColumnarExchange (47) - : +- RowToColumnar (46) - : +- * HashAggregate (45) - : +- * HashAggregate (44) - : +- * ColumnarToRow (43) - : +- ReusedExchange (42) - +- * HashAggregate (57) - +- * ColumnarToRow (56) - +- CometColumnarExchange (55) - +- RowToColumnar (54) - +- * HashAggregate (53) - +- * HashAggregate (52) - +- * ColumnarToRow (51) - +- ReusedExchange (50) +TakeOrderedAndProject (58) ++- * Project (57) + +- Window (56) + +- * Sort (55) + +- Exchange (54) + +- * HashAggregate (53) + +- Exchange (52) + +- * HashAggregate (51) + +- Union (50) + :- * HashAggregate (39) + : +- Exchange (38) + : +- * HashAggregate (37) + : +- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * ColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- BroadcastExchange (34) + : +- * BroadcastHashJoin LeftSemi BuildRight (33) + : :- * ColumnarToRow (12) + : : +- CometFilter (11) + : : +- CometScan parquet spark_catalog.default.store (10) + : +- BroadcastExchange (32) + : +- * Project (31) + : +- * Filter (30) + : +- Window (29) + : +- WindowGroupLimit (28) + : +- * ColumnarToRow (27) + : +- CometSort (26) + : +- CometHashAggregate (25) + : +- CometExchange (24) + : +- CometHashAggregate (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store_sales (13) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometScan parquet spark_catalog.default.store (15) + : +- ReusedExchange (20) + :- * HashAggregate (44) + : +- Exchange (43) + : +- * HashAggregate (42) + : +- * HashAggregate (41) + : +- ReusedExchange (40) + +- * HashAggregate (49) + +- Exchange (48) + +- * HashAggregate (47) + +- * HashAggregate (46) + +- ReusedExchange (45) (1) Scan parquet spark_catalog.default.store_sales @@ -179,9 +169,9 @@ Input [2]: [ss_net_profit#11, s_state#15] Keys [1]: [s_state#15] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] -(24) CometColumnarExchange +(24) CometExchange Input [2]: [s_state#15, sum#17] -Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (25) CometHashAggregate Input [2]: [s_state#15, sum#17] @@ -242,170 +232,142 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum#20] Results [3]: [s_state#9, s_county#8, sum#21] -(38) RowToColumnar +(38) Exchange Input [3]: [s_state#9, s_county#8, sum#21] +Arguments: hashpartitioning(s_state#9, s_county#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(39) CometColumnarExchange -Input [3]: [s_state#9, s_county#8, sum#21] -Arguments: hashpartitioning(s_state#9, s_county#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(40) ColumnarToRow [codegen id : 5] -Input [3]: [s_state#9, s_county#8, sum#21] - -(41) HashAggregate [codegen id : 5] +(39) HashAggregate [codegen id : 5] Input [3]: [s_state#9, s_county#8, sum#21] Keys [2]: [s_state#9, s_county#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#9, s_county#8, 0 AS g_state#24, 0 AS g_county#25, 0 AS lochierarchy#26] -(42) ReusedExchange [Reuses operator id: 39] +(40) ReusedExchange [Reuses operator id: 38] Output [3]: [s_state#27, s_county#28, sum#29] -(43) ColumnarToRow [codegen id : 10] -Input [3]: [s_state#27, s_county#28, sum#29] - -(44) HashAggregate [codegen id : 10] +(41) HashAggregate [codegen id : 10] Input [3]: [s_state#27, s_county#28, sum#29] Keys [2]: [s_state#27, s_county#28] Functions [1]: [sum(UnscaledValue(ss_net_profit#30))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#30))#22] Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#30))#22,17,2) AS total_sum#31, s_state#27] -(45) HashAggregate [codegen id : 10] +(42) HashAggregate [codegen id : 10] Input [2]: [total_sum#31, s_state#27] Keys [1]: [s_state#27] Functions [1]: [partial_sum(total_sum#31)] Aggregate Attributes [2]: [sum#32, isEmpty#33] Results [3]: [s_state#27, sum#34, isEmpty#35] -(46) RowToColumnar +(43) Exchange Input [3]: [s_state#27, sum#34, isEmpty#35] +Arguments: hashpartitioning(s_state#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(47) CometColumnarExchange -Input [3]: [s_state#27, sum#34, isEmpty#35] -Arguments: hashpartitioning(s_state#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(48) ColumnarToRow [codegen id : 11] -Input [3]: [s_state#27, sum#34, isEmpty#35] - -(49) HashAggregate [codegen id : 11] +(44) HashAggregate [codegen id : 11] Input [3]: [s_state#27, sum#34, isEmpty#35] Keys [1]: [s_state#27] Functions [1]: [sum(total_sum#31)] Aggregate Attributes [1]: [sum(total_sum#31)#36] Results [6]: [sum(total_sum#31)#36 AS total_sum#37, s_state#27, null AS s_county#38, 0 AS g_state#39, 1 AS g_county#40, 1 AS lochierarchy#41] -(50) ReusedExchange [Reuses operator id: 39] +(45) ReusedExchange [Reuses operator id: 38] Output [3]: [s_state#42, s_county#43, sum#44] -(51) ColumnarToRow [codegen id : 16] -Input [3]: [s_state#42, s_county#43, sum#44] - -(52) HashAggregate [codegen id : 16] +(46) HashAggregate [codegen id : 16] Input [3]: [s_state#42, s_county#43, sum#44] Keys [2]: [s_state#42, s_county#43] Functions [1]: [sum(UnscaledValue(ss_net_profit#45))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#45))#22] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#45))#22,17,2) AS total_sum#46] -(53) HashAggregate [codegen id : 16] +(47) HashAggregate [codegen id : 16] Input [1]: [total_sum#46] Keys: [] Functions [1]: [partial_sum(total_sum#46)] Aggregate Attributes [2]: [sum#47, isEmpty#48] Results [2]: [sum#49, isEmpty#50] -(54) RowToColumnar -Input [2]: [sum#49, isEmpty#50] - -(55) CometColumnarExchange -Input [2]: [sum#49, isEmpty#50] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(56) ColumnarToRow [codegen id : 17] +(48) Exchange Input [2]: [sum#49, isEmpty#50] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(57) HashAggregate [codegen id : 17] +(49) HashAggregate [codegen id : 17] Input [2]: [sum#49, isEmpty#50] Keys: [] Functions [1]: [sum(total_sum#46)] Aggregate Attributes [1]: [sum(total_sum#46)#51] Results [6]: [sum(total_sum#46)#51 AS total_sum#52, null AS s_state#53, null AS s_county#54, 1 AS g_state#55, 1 AS g_county#56, 2 AS lochierarchy#57] -(58) Union +(50) Union -(59) HashAggregate [codegen id : 18] +(51) HashAggregate [codegen id : 18] Input [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] Keys [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] Functions: [] Aggregate Attributes: [] Results [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] -(60) RowToColumnar +(52) Exchange Input [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] +Arguments: hashpartitioning(total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(61) CometColumnarExchange -Input [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] -Arguments: hashpartitioning(total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(62) CometHashAggregate +(53) HashAggregate [codegen id : 19] Input [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] Keys [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] Functions: [] +Aggregate Attributes: [] +Results [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, CASE WHEN (g_county#25 = 0) THEN s_state#9 END AS _w0#58] -(63) CometColumnarExchange -Input [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#58] -Arguments: hashpartitioning(lochierarchy#26, _w0#58, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(64) CometSort +(54) Exchange Input [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#58] -Arguments: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#58], [lochierarchy#26 ASC NULLS FIRST, _w0#58 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST] +Arguments: hashpartitioning(lochierarchy#26, _w0#58, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(65) ColumnarToRow [codegen id : 19] +(55) Sort [codegen id : 20] Input [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#58] +Arguments: [lochierarchy#26 ASC NULLS FIRST, _w0#58 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST], false, 0 -(66) Window +(56) Window Input [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#58] Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#26, _w0#58, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#59], [lochierarchy#26, _w0#58], [total_sum#23 DESC NULLS LAST] -(67) Project [codegen id : 20] +(57) Project [codegen id : 21] Output [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, rank_within_parent#59] Input [6]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#58, rank_within_parent#59] -(68) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, rank_within_parent#59] Arguments: 100, [lochierarchy#26 DESC NULLS LAST, CASE WHEN (lochierarchy#26 = 0) THEN s_state#9 END ASC NULLS FIRST, rank_within_parent#59 ASC NULLS FIRST], [total_sum#23, s_state#9, s_county#8, lochierarchy#26, rank_within_parent#59] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (73) -+- * ColumnarToRow (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan parquet spark_catalog.default.date_dim (69) +BroadcastExchange (63) ++- * ColumnarToRow (62) + +- CometProject (61) + +- CometFilter (60) + +- CometScan parquet spark_catalog.default.date_dim (59) -(69) Scan parquet spark_catalog.default.date_dim +(59) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(70) CometFilter +(60) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) -(71) CometProject +(61) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(72) ColumnarToRow [codegen id : 1] +(62) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(73) BroadcastExchange +(63) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt index 5c57b1764..cb772b800 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q70a/simplified.txt @@ -1,103 +1,95 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (20) + WholeStageCodegen (21) Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] InputAdapter Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (19) - ColumnarToRow + WholeStageCodegen (20) + Sort [lochierarchy,_w0,total_sum] InputAdapter - CometSort [total_sum,s_state,s_county,lochierarchy,_w0] - CometColumnarExchange [lochierarchy,_w0] #1 - CometHashAggregate [total_sum,s_state,s_county,lochierarchy,_w0,g_state,g_county] - CometColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 - RowToColumnar + Exchange [lochierarchy,_w0] #1 + WholeStageCodegen (19) + HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0] + InputAdapter + Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 WholeStageCodegen (18) HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] InputAdapter Union WholeStageCodegen (5) HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_state,s_county] #3 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_county,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (2) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [s_state,_w0] - CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] - CometColumnarExchange [s_state] #8 - CometHashAggregate [s_state,sum,ss_net_profit] - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #9 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #5 + InputAdapter + Exchange [s_state,s_county] #3 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] + Project [ss_net_profit,s_county,s_state] + BroadcastHashJoin [ss_store_sk,s_store_sk] + ColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + BroadcastHashJoin [s_state,s_state] + ColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [s_state,_w0] + CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #8 + CometHashAggregate [s_state,sum,ss_net_profit] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [s_store_sk,s_state] #9 + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #5 WholeStageCodegen (11) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_state] #10 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 + InputAdapter + Exchange [s_state] #10 + WholeStageCodegen (10) + HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 WholeStageCodegen (17) HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #11 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 + InputAdapter + Exchange #11 + WholeStageCodegen (16) + HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt index 53e1bb5a6..276e36240 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/explain.txt @@ -1,72 +1,71 @@ == Physical Plan == -* ColumnarToRow (68) -+- CometTakeOrderedAndProject (67) - +- CometHashAggregate (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometProject (63) - +- CometSortMergeJoin (62) - :- CometSort (56) - : +- CometColumnarExchange (55) - : +- RowToColumnar (54) - : +- * Project (53) - : +- * BroadcastHashJoin LeftOuter BuildRight (52) - : :- * Project (47) - : : +- * BroadcastHashJoin Inner BuildRight (46) - : : :- * ColumnarToRow (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometProject (35) - : : : : +- CometBroadcastHashJoin (34) - : : : : :- CometProject (29) - : : : : : +- CometBroadcastHashJoin (28) - : : : : : :- CometProject (23) - : : : : : : +- CometBroadcastHashJoin (22) - : : : : : : :- CometProject (17) - : : : : : : : +- CometBroadcastHashJoin (16) - : : : : : : : :- CometProject (12) - : : : : : : : : +- CometBroadcastHashJoin (11) - : : : : : : : : :- CometProject (7) - : : : : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- CometBroadcastExchange (5) - : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) - : : : : : : : : +- CometBroadcastExchange (10) - : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) - : : : : : : : +- CometBroadcastExchange (15) - : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan parquet spark_catalog.default.item (13) - : : : : : : +- CometBroadcastExchange (21) - : : : : : : +- CometProject (20) - : : : : : : +- CometFilter (19) - : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) - : : : : : +- CometBroadcastExchange (27) - : : : : : +- CometProject (26) - : : : : : +- CometFilter (25) - : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) - : : : : +- CometBroadcastExchange (33) - : : : : +- CometProject (32) - : : : : +- CometFilter (31) - : : : : +- CometScan parquet spark_catalog.default.date_dim (30) - : : : +- CometBroadcastExchange (38) - : : : +- CometFilter (37) - : : : +- CometScan parquet spark_catalog.default.date_dim (36) - : : +- BroadcastExchange (45) - : : +- * ColumnarToRow (44) - : : +- CometFilter (43) - : : +- CometScan parquet spark_catalog.default.date_dim (42) - : +- BroadcastExchange (51) - : +- * ColumnarToRow (50) - : +- CometFilter (49) - : +- CometScan parquet spark_catalog.default.promotion (48) - +- CometSort (61) - +- CometColumnarExchange (60) - +- CometProject (59) - +- CometFilter (58) - +- CometScan parquet spark_catalog.default.catalog_returns (57) +TakeOrderedAndProject (67) ++- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * SortMergeJoin LeftOuter (62) + :- * Sort (55) + : +- Exchange (54) + : +- * Project (53) + : +- * BroadcastHashJoin LeftOuter BuildRight (52) + : :- * Project (47) + : : +- * BroadcastHashJoin Inner BuildRight (46) + : : :- * ColumnarToRow (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometScan parquet spark_catalog.default.date_dim (36) + : : +- BroadcastExchange (45) + : : +- * ColumnarToRow (44) + : : +- CometFilter (43) + : : +- CometScan parquet spark_catalog.default.date_dim (42) + : +- BroadcastExchange (51) + : +- * ColumnarToRow (50) + : +- CometFilter (49) + : +- CometScan parquet spark_catalog.default.promotion (48) + +- * ColumnarToRow (61) + +- CometSort (60) + +- CometExchange (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.catalog_returns (56) (1) Scan parquet spark_catalog.default.catalog_sales @@ -321,99 +320,101 @@ Join condition: None Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -(54) RowToColumnar +(54) Exchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(55) CometColumnarExchange +(55) Sort [codegen id : 4] Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 -(56) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] - -(57) Scan parquet spark_catalog.default.catalog_returns +(56) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(58) CometFilter +(57) CometFilter Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) -(59) CometProject +(58) CometProject Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] -(60) CometColumnarExchange +(59) CometExchange Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(61) CometSort +(60) CometSort Input [2]: [cr_item_sk#31, cr_order_number#32] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] -(62) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +(61) ColumnarToRow [codegen id : 5] +Input [2]: [cr_item_sk#31, cr_order_number#32] -(63) CometProject +(62) SortMergeJoin [codegen id : 6] +Left keys [2]: [cs_item_sk#4, cs_order_number#6] +Right keys [2]: [cr_item_sk#31, cr_order_number#32] +Join type: LeftOuter +Join condition: None + +(63) Project [codegen id : 6] +Output [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(64) CometHashAggregate +(64) HashAggregate [codegen id : 6] Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#34] +Results [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] -(65) CometColumnarExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +(65) Exchange +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] +Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(66) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] +(66) HashAggregate [codegen id : 7] +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#36] +Results [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count(1)#36 AS no_promo#37, count(1)#36 AS promo#38, count(1)#36 AS total_cnt#39] -(67) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -(68) ColumnarToRow [codegen id : 4] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +(67) TakeOrderedAndProject +Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] +Arguments: 100, [total_cnt#39 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (73) -+- * ColumnarToRow (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan parquet spark_catalog.default.date_dim (69) +BroadcastExchange (72) ++- * ColumnarToRow (71) + +- CometProject (70) + +- CometFilter (69) + +- CometScan parquet spark_catalog.default.date_dim (68) -(69) Scan parquet spark_catalog.default.date_dim +(68) Scan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(70) CometFilter +(69) CometFilter Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) -(71) CometProject +(70) CometProject Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] -(72) ColumnarToRow [codegen id : 1] +(71) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -(73) BroadcastExchange +(72) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt index 1022e1a90..1efc2e5d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q72/simplified.txt @@ -1,86 +1,92 @@ -WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] - CometColumnarExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] - CometProject [w_warehouse_name,i_item_desc,d_week_seq] - CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometColumnarExchange [cs_item_sk,cs_order_number] #2 - RowToColumnar - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - ColumnarToRow - InputAdapter - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 - CometFilter [d_date_sk,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] +TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] + WholeStageCodegen (7) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] + InputAdapter + Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + WholeStageCodegen (6) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] + Project [w_warehouse_name,i_item_desc,d_week_seq] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter + WholeStageCodegen (4) + Sort [cs_item_sk,cs_order_number] + InputAdapter + Exchange [cs_item_sk,cs_order_number] #2 + WholeStageCodegen (3) + Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_promo_sk,p_promo_sk] + Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] + ColumnarToRow + InputAdapter + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #7 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #8 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometFilter [d_date_sk,d_week_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #11 - WholeStageCodegen (1) + BroadcastExchange #12 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] - CometSort [cr_item_sk,cr_order_number] - CometColumnarExchange [cr_item_sk,cr_order_number] #13 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometFilter [p_promo_sk] + CometScan parquet spark_catalog.default.promotion [p_promo_sk] + InputAdapter + WholeStageCodegen (5) + ColumnarToRow + InputAdapter + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #13 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt index e92f554c5..4072b2277 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/explain.txt @@ -8,7 +8,7 @@ : :- CometBroadcastHashJoin (33) : : :- CometFilter (16) : : : +- CometHashAggregate (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometHashAggregate (13) : : : +- CometProject (12) : : : +- CometBroadcastHashJoin (11) @@ -24,7 +24,7 @@ : : : +- CometScan parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometProject (28) : : +- CometBroadcastHashJoin (27) @@ -41,7 +41,7 @@ : +- CometBroadcastExchange (48) : +- CometFilter (47) : +- CometHashAggregate (46) - : +- CometColumnarExchange (45) + : +- CometExchange (45) : +- CometHashAggregate (44) : +- CometProject (43) : +- CometBroadcastHashJoin (42) @@ -55,7 +55,7 @@ : +- ReusedExchange (41) +- CometBroadcastExchange (64) +- CometHashAggregate (63) - +- CometColumnarExchange (62) + +- CometExchange (62) +- CometHashAggregate (61) +- CometProject (60) +- CometBroadcastHashJoin (59) @@ -134,9 +134,9 @@ Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_yea Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -(14) CometColumnarExchange +(14) CometExchange Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#11] @@ -212,9 +212,9 @@ Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, ss_net_paid#19, d Keys [4]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#19))] -(30) CometColumnarExchange +(30) CometExchange Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, sum#24] -Arguments: hashpartitioning(c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, sum#24] @@ -283,9 +283,9 @@ Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ws_net_paid#34, d Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#34))] -(45) CometColumnarExchange +(45) CometExchange Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, sum#39] -Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometHashAggregate Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, sum#39] @@ -362,9 +362,9 @@ Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, ws_net_paid#47, d Keys [4]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#47))] -(62) CometColumnarExchange +(62) CometExchange Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, sum#52] -Arguments: hashpartitioning(c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (63) CometHashAggregate Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, sum#52] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt index 0b288df94..631a82f1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q74/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -32,7 +32,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -56,7 +56,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -71,7 +71,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt index 2aeff8de1..68deab557 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/explain.txt @@ -4,19 +4,19 @@ TakeOrderedAndProject (125) +- * SortMergeJoin Inner (123) :- * ColumnarToRow (67) : +- CometSort (66) - : +- CometColumnarExchange (65) + : +- CometExchange (65) : +- CometFilter (64) : +- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometHashAggregate (60) - : +- CometColumnarExchange (59) + : +- CometExchange (59) : +- CometHashAggregate (58) : +- CometUnion (57) : :- CometProject (22) : : +- CometSortMergeJoin (21) : : :- CometSort (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometProject (13) : : : +- CometBroadcastHashJoin (12) : : : :- CometProject (8) @@ -31,14 +31,14 @@ TakeOrderedAndProject (125) : : : +- CometFilter (10) : : : +- CometScan parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) : : +- CometScan parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) - : : : +- CometColumnarExchange (31) + : : : +- CometExchange (31) : : : +- CometProject (30) : : : +- CometBroadcastHashJoin (29) : : : :- CometProject (27) @@ -48,14 +48,14 @@ TakeOrderedAndProject (125) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) - : : +- CometColumnarExchange (36) + : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) : : +- CometScan parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) - : : +- CometColumnarExchange (48) + : : +- CometExchange (48) : : +- CometProject (47) : : +- CometBroadcastHashJoin (46) : : :- CometProject (44) @@ -65,25 +65,25 @@ TakeOrderedAndProject (125) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) - : +- CometColumnarExchange (53) + : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) : +- CometScan parquet spark_catalog.default.web_returns (50) +- * ColumnarToRow (122) +- CometSort (121) - +- CometColumnarExchange (120) + +- CometExchange (120) +- CometFilter (119) +- CometHashAggregate (118) - +- CometColumnarExchange (117) + +- CometExchange (117) +- CometHashAggregate (116) +- CometHashAggregate (115) - +- CometColumnarExchange (114) + +- CometExchange (114) +- CometHashAggregate (113) +- CometUnion (112) :- CometProject (83) : +- CometSortMergeJoin (82) : :- CometSort (79) - : : +- CometColumnarExchange (78) + : : +- CometExchange (78) : : +- CometProject (77) : : +- CometBroadcastHashJoin (76) : : :- CometProject (72) @@ -99,7 +99,7 @@ TakeOrderedAndProject (125) :- CometProject (97) : +- CometSortMergeJoin (96) : :- CometSort (93) - : : +- CometColumnarExchange (92) + : : +- CometExchange (92) : : +- CometProject (91) : : +- CometBroadcastHashJoin (90) : : :- CometProject (88) @@ -113,7 +113,7 @@ TakeOrderedAndProject (125) +- CometProject (111) +- CometSortMergeJoin (110) :- CometSort (107) - : +- CometColumnarExchange (106) + : +- CometExchange (106) : +- CometProject (105) : +- CometBroadcastHashJoin (104) : :- CometProject (102) @@ -190,9 +190,9 @@ Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -(14) CometColumnarExchange +(14) CometExchange Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] @@ -213,9 +213,9 @@ Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -(19) CometColumnarExchange +(19) CometExchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -266,9 +266,9 @@ Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -(31) CometColumnarExchange +(31) CometExchange Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] @@ -289,9 +289,9 @@ Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -(36) CometColumnarExchange +(36) CometExchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -342,9 +342,9 @@ Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -(48) CometColumnarExchange +(48) CometExchange Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] @@ -365,9 +365,9 @@ Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -(53) CometColumnarExchange +(53) CometExchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -392,9 +392,9 @@ Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_ Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] Functions: [] -(59) CometColumnarExchange +(59) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] @@ -406,9 +406,9 @@ Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_ Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -(62) CometColumnarExchange +(62) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (63) CometHashAggregate Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] @@ -419,9 +419,9 @@ Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] Condition : isnotnull(sales_cnt#64) -(65) CometColumnarExchange +(65) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (66) CometSort Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] @@ -478,9 +478,9 @@ Arguments: [cs_sold_date_sk#70], [d_date_sk#77], Inner, BuildRight Input [11]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -(78) CometColumnarExchange +(78) CometExchange Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (79) CometSort Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] @@ -538,9 +538,9 @@ Arguments: [ss_sold_date_sk#87], [d_date_sk#94], Inner, BuildRight Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_date_sk#94, d_year#95] Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -(92) CometColumnarExchange +(92) CometExchange Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (93) CometSort Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] @@ -598,9 +598,9 @@ Arguments: [ws_sold_date_sk#104], [d_date_sk#111], Inner, BuildRight Input [11]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -(106) CometColumnarExchange +(106) CometExchange Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (107) CometSort Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] @@ -632,9 +632,9 @@ Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufac Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] Functions: [] -(114) CometColumnarExchange +(114) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (115) CometHashAggregate Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] @@ -646,9 +646,9 @@ Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufac Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -(117) CometColumnarExchange +(117) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] +Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (118) CometHashAggregate Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] @@ -659,9 +659,9 @@ Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] Condition : isnotnull(sales_cnt#118) -(120) CometColumnarExchange +(120) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] +Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (121) CometSort Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/simplified.txt index 9b187bca1..60df69dea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q75/simplified.txt @@ -7,19 +7,19 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #4 + CometExchange [cs_order_number,cs_item_sk] #4 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -41,14 +41,14 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometColumnarExchange [cr_order_number,cr_item_sk] #8 + CometExchange [cr_order_number,cr_item_sk] #8 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #9 + CometExchange [ss_ticket_number,ss_item_sk] #9 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -59,14 +59,14 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 ReusedExchange [d_date_sk,d_year] #7 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #10 + CometExchange [sr_ticket_number,sr_item_sk] #10 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #11 + CometExchange [ws_order_number,ws_item_sk] #11 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -77,7 +77,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 ReusedExchange [d_date_sk,d_year] #7 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometColumnarExchange [wr_order_number,wr_item_sk] #12 + CometExchange [wr_order_number,wr_item_sk] #12 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] @@ -86,19 +86,19 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #16 + CometExchange [cs_order_number,cs_item_sk] #16 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -121,7 +121,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #19 + CometExchange [ss_ticket_number,ss_item_sk] #19 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -136,7 +136,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #20 + CometExchange [ws_order_number,ws_item_sk] #20 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/explain.txt index a2adcb0b5..96b3db104 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/explain.txt @@ -1,111 +1,101 @@ == Physical Plan == -* ColumnarToRow (107) -+- CometTakeOrderedAndProject (106) - +- CometHashAggregate (105) - +- CometColumnarExchange (104) - +- RowToColumnar (103) - +- * HashAggregate (102) - +- Union (101) - :- * HashAggregate (84) - : +- * ColumnarToRow (83) - : +- CometColumnarExchange (82) - : +- RowToColumnar (81) - : +- * HashAggregate (80) - : +- Union (79) - : :- * ColumnarToRow (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometHashAggregate (16) - : : : +- CometColumnarExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.store (9) - : : +- CometBroadcastExchange (28) - : : +- CometHashAggregate (27) - : : +- CometColumnarExchange (26) - : : +- CometHashAggregate (25) - : : +- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometScan parquet spark_catalog.default.store_returns (17) - : : : +- ReusedExchange (19) - : : +- ReusedExchange (22) - : :- * Project (50) - : : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) - : : :- BroadcastExchange (40) - : : : +- * ColumnarToRow (39) - : : : +- CometHashAggregate (38) - : : : +- CometColumnarExchange (37) - : : : +- CometHashAggregate (36) - : : : +- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometScan parquet spark_catalog.default.catalog_sales (32) - : : : +- ReusedExchange (33) - : : +- * ColumnarToRow (48) - : : +- CometHashAggregate (47) - : : +- CometColumnarExchange (46) - : : +- CometHashAggregate (45) - : : +- CometProject (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometScan parquet spark_catalog.default.catalog_returns (41) - : : +- ReusedExchange (42) - : +- * ColumnarToRow (78) - : +- CometProject (77) - : +- CometBroadcastHashJoin (76) - : :- CometHashAggregate (63) - : : +- CometColumnarExchange (62) - : : +- CometHashAggregate (61) - : : +- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometFilter (52) - : : : : +- CometScan parquet spark_catalog.default.web_sales (51) - : : : +- ReusedExchange (53) - : : +- CometBroadcastExchange (58) - : : +- CometFilter (57) - : : +- CometScan parquet spark_catalog.default.web_page (56) - : +- CometBroadcastExchange (75) - : +- CometHashAggregate (74) - : +- CometColumnarExchange (73) - : +- CometHashAggregate (72) - : +- CometProject (71) - : +- CometBroadcastHashJoin (70) - : :- CometProject (68) - : : +- CometBroadcastHashJoin (67) - : : :- CometFilter (65) - : : : +- CometScan parquet spark_catalog.default.web_returns (64) - : : +- ReusedExchange (66) - : +- ReusedExchange (69) - :- * HashAggregate (92) - : +- * ColumnarToRow (91) - : +- CometColumnarExchange (90) - : +- RowToColumnar (89) - : +- * HashAggregate (88) - : +- * HashAggregate (87) - : +- * ColumnarToRow (86) - : +- ReusedExchange (85) - +- * HashAggregate (100) - +- * ColumnarToRow (99) - +- CometColumnarExchange (98) - +- RowToColumnar (97) - +- * HashAggregate (96) - +- * HashAggregate (95) - +- * ColumnarToRow (94) - +- ReusedExchange (93) +TakeOrderedAndProject (97) ++- * HashAggregate (96) + +- Exchange (95) + +- * HashAggregate (94) + +- Union (93) + :- * HashAggregate (82) + : +- Exchange (81) + : +- * HashAggregate (80) + : +- Union (79) + : :- * ColumnarToRow (31) + : : +- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan parquet spark_catalog.default.store (9) + : : +- CometBroadcastExchange (28) + : : +- CometHashAggregate (27) + : : +- CometExchange (26) + : : +- CometHashAggregate (25) + : : +- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (21) + : : : +- CometBroadcastHashJoin (20) + : : : :- CometFilter (18) + : : : : +- CometScan parquet spark_catalog.default.store_returns (17) + : : : +- ReusedExchange (19) + : : +- ReusedExchange (22) + : :- * Project (50) + : : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : : :- BroadcastExchange (40) + : : : +- * ColumnarToRow (39) + : : : +- CometHashAggregate (38) + : : : +- CometExchange (37) + : : : +- CometHashAggregate (36) + : : : +- CometProject (35) + : : : +- CometBroadcastHashJoin (34) + : : : :- CometScan parquet spark_catalog.default.catalog_sales (32) + : : : +- ReusedExchange (33) + : : +- * ColumnarToRow (48) + : : +- CometHashAggregate (47) + : : +- CometExchange (46) + : : +- CometHashAggregate (45) + : : +- CometProject (44) + : : +- CometBroadcastHashJoin (43) + : : :- CometScan parquet spark_catalog.default.catalog_returns (41) + : : +- ReusedExchange (42) + : +- * ColumnarToRow (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (55) + : : : +- CometBroadcastHashJoin (54) + : : : :- CometFilter (52) + : : : : +- CometScan parquet spark_catalog.default.web_sales (51) + : : : +- ReusedExchange (53) + : : +- CometBroadcastExchange (58) + : : +- CometFilter (57) + : : +- CometScan parquet spark_catalog.default.web_page (56) + : +- CometBroadcastExchange (75) + : +- CometHashAggregate (74) + : +- CometExchange (73) + : +- CometHashAggregate (72) + : +- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometProject (68) + : : +- CometBroadcastHashJoin (67) + : : :- CometFilter (65) + : : : +- CometScan parquet spark_catalog.default.web_returns (64) + : : +- ReusedExchange (66) + : +- ReusedExchange (69) + :- * HashAggregate (87) + : +- Exchange (86) + : +- * HashAggregate (85) + : +- * HashAggregate (84) + : +- ReusedExchange (83) + +- * HashAggregate (92) + +- Exchange (91) + +- * HashAggregate (90) + +- * HashAggregate (89) + +- ReusedExchange (88) (1) Scan parquet spark_catalog.default.store_sales @@ -177,9 +167,9 @@ Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] Keys [1]: [s_store_sk#8] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [s_store_sk#8, sum#9, sum#10] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [s_store_sk#8, sum#9, sum#10] @@ -227,9 +217,9 @@ Input [3]: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] Keys [1]: [s_store_sk#17] Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#12)), partial_sum(UnscaledValue(sr_net_loss#13))] -(26) CometColumnarExchange +(26) CometExchange Input [3]: [s_store_sk#17, sum#18, sum#19] -Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate Input [3]: [s_store_sk#17, sum#18, sum#19] @@ -276,9 +266,9 @@ Input [3]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] Keys [1]: [cs_call_center_sk#28] Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#29)), partial_sum(UnscaledValue(cs_net_profit#30))] -(37) CometColumnarExchange +(37) CometExchange Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometHashAggregate Input [3]: [cs_call_center_sk#28, sum#34, sum#35] @@ -316,9 +306,9 @@ Input [2]: [cr_return_amount#38, cr_net_loss#39] Keys: [] Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#38)), partial_sum(UnscaledValue(cr_net_loss#39))] -(46) CometColumnarExchange +(46) CometExchange Input [2]: [sum#43, sum#44] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (47) CometHashAggregate Input [2]: [sum#43, sum#44] @@ -389,9 +379,9 @@ Input [3]: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] Keys [1]: [wp_web_page_sk#56] Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#51)), partial_sum(UnscaledValue(ws_net_profit#52))] -(62) CometColumnarExchange +(62) CometExchange Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (63) CometHashAggregate Input [3]: [wp_web_page_sk#56, sum#57, sum#58] @@ -439,9 +429,9 @@ Input [3]: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] Keys [1]: [wp_web_page_sk#65] Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#60)), partial_sum(UnscaledValue(wr_net_loss#61))] -(73) CometColumnarExchange +(73) CometExchange Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (74) CometHashAggregate Input [3]: [wp_web_page_sk#65, sum#66, sum#67] @@ -473,154 +463,126 @@ Functions [3]: [partial_sum(sales#22), partial_sum(returns#26), partial_sum(prof Aggregate Attributes [6]: [sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] Results [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -(81) RowToColumnar +(81) Exchange Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Arguments: hashpartitioning(channel#24, id#25, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(82) CometColumnarExchange -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Arguments: hashpartitioning(channel#24, id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(83) ColumnarToRow [codegen id : 6] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] - -(84) HashAggregate [codegen id : 6] +(82) HashAggregate [codegen id : 6] Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] Keys [2]: [channel#24, id#25] Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] Results [5]: [channel#24, id#25, cast(sum(sales#22)#88 as decimal(37,2)) AS sales#91, cast(sum(returns#26)#89 as decimal(37,2)) AS returns#92, cast(sum(profit#27)#90 as decimal(38,2)) AS profit#93] -(85) ReusedExchange [Reuses operator id: 82] +(83) ReusedExchange [Reuses operator id: 81] Output [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -(86) ColumnarToRow [codegen id : 12] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] - -(87) HashAggregate [codegen id : 12] +(84) HashAggregate [codegen id : 12] Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] Keys [2]: [channel#24, id#25] Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] Results [4]: [channel#24, sum(sales#22)#88 AS sales#94, sum(returns#26)#89 AS returns#95, sum(profit#27)#90 AS profit#96] -(88) HashAggregate [codegen id : 12] +(85) HashAggregate [codegen id : 12] Input [4]: [channel#24, sales#94, returns#95, profit#96] Keys [1]: [channel#24] Functions [3]: [partial_sum(sales#94), partial_sum(returns#95), partial_sum(profit#96)] Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] Results [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -(89) RowToColumnar +(86) Exchange Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(channel#24, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(90) CometColumnarExchange -Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Arguments: hashpartitioning(channel#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(91) ColumnarToRow [codegen id : 13] -Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(92) HashAggregate [codegen id : 13] +(87) HashAggregate [codegen id : 13] Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] Keys [1]: [channel#24] Functions [3]: [sum(sales#94), sum(returns#95), sum(profit#96)] Aggregate Attributes [3]: [sum(sales#94)#109, sum(returns#95)#110, sum(profit#96)#111] Results [5]: [channel#24, null AS id#112, sum(sales#94)#109 AS sales#113, sum(returns#95)#110 AS returns#114, sum(profit#96)#111 AS profit#115] -(93) ReusedExchange [Reuses operator id: 82] +(88) ReusedExchange [Reuses operator id: 81] Output [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -(94) ColumnarToRow [codegen id : 19] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] - -(95) HashAggregate [codegen id : 19] +(89) HashAggregate [codegen id : 19] Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] Keys [2]: [channel#24, id#25] Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] Results [3]: [sum(sales#22)#88 AS sales#116, sum(returns#26)#89 AS returns#117, sum(profit#27)#90 AS profit#118] -(96) HashAggregate [codegen id : 19] +(90) HashAggregate [codegen id : 19] Input [3]: [sales#116, returns#117, profit#118] Keys: [] Functions [3]: [partial_sum(sales#116), partial_sum(returns#117), partial_sum(profit#118)] Aggregate Attributes [6]: [sum#119, isEmpty#120, sum#121, isEmpty#122, sum#123, isEmpty#124] Results [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130] -(97) RowToColumnar -Input [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130] - -(98) CometColumnarExchange +(91) Exchange Input [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] -(99) ColumnarToRow [codegen id : 20] -Input [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130] - -(100) HashAggregate [codegen id : 20] +(92) HashAggregate [codegen id : 20] Input [6]: [sum#125, isEmpty#126, sum#127, isEmpty#128, sum#129, isEmpty#130] Keys: [] Functions [3]: [sum(sales#116), sum(returns#117), sum(profit#118)] Aggregate Attributes [3]: [sum(sales#116)#131, sum(returns#117)#132, sum(profit#118)#133] Results [5]: [null AS channel#134, null AS id#135, sum(sales#116)#131 AS sales#136, sum(returns#117)#132 AS returns#137, sum(profit#118)#133 AS profit#138] -(101) Union +(93) Union -(102) HashAggregate [codegen id : 21] +(94) HashAggregate [codegen id : 21] Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] Keys [5]: [channel#24, id#25, sales#91, returns#92, profit#93] Functions: [] Aggregate Attributes: [] Results [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -(103) RowToColumnar +(95) Exchange Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] +Arguments: hashpartitioning(channel#24, id#25, sales#91, returns#92, profit#93, 5), ENSURE_REQUIREMENTS, [plan_id=11] -(104) CometColumnarExchange -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Arguments: hashpartitioning(channel#24, id#25, sales#91, returns#92, profit#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] - -(105) CometHashAggregate +(96) HashAggregate [codegen id : 22] Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] Keys [5]: [channel#24, id#25, sales#91, returns#92, profit#93] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -(106) CometTakeOrderedAndProject -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#24 ASC NULLS FIRST,id#25 ASC NULLS FIRST], output=[channel#24,id#25,sales#91,returns#92,profit#93]), [channel#24, id#25, sales#91, returns#92, profit#93], 100, [channel#24 ASC NULLS FIRST, id#25 ASC NULLS FIRST], [channel#24, id#25, sales#91, returns#92, profit#93] - -(107) ColumnarToRow [codegen id : 22] +(97) TakeOrderedAndProject Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] +Arguments: 100, [channel#24 ASC NULLS FIRST, id#25 ASC NULLS FIRST], [channel#24, id#25, sales#91, returns#92, profit#93] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (112) -+- * ColumnarToRow (111) - +- CometProject (110) - +- CometFilter (109) - +- CometScan parquet spark_catalog.default.date_dim (108) +BroadcastExchange (102) ++- * ColumnarToRow (101) + +- CometProject (100) + +- CometFilter (99) + +- CometScan parquet spark_catalog.default.date_dim (98) -(108) Scan parquet spark_catalog.default.date_dim +(98) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_date#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(109) CometFilter +(99) CometFilter Input [2]: [d_date_sk#6, d_date#7] Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6)) -(110) CometProject +(100) CometProject Input [2]: [d_date_sk#6, d_date#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(111) ColumnarToRow [codegen id : 1] +(101) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(112) BroadcastExchange +(102) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/simplified.txt index 6e5506d68..01cd2f085 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q77a/simplified.txt @@ -1,145 +1,135 @@ -WholeStageCodegen (22) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,id,sales,returns,profit] - CometHashAggregate [channel,id,sales,returns,profit] - CometColumnarExchange [channel,id,sales,returns,profit] #1 - RowToColumnar - WholeStageCodegen (21) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id] #2 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (22) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Exchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (21) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (6) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel,id] #2 + WholeStageCodegen (5) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + ColumnarToRow InputAdapter - Union - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [s_store_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] - CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] - CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] - CometColumnarExchange [s_store_sk] #3 - CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #6 - CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #7 - CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - CometColumnarExchange [s_store_sk] #8 - CometHashAggregate [s_store_sk,sum,sum,sr_return_amt,sr_net_loss] - CometProject [sr_return_amt,sr_net_loss,s_store_sk] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] - CometProject [sr_store_sk,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [s_store_sk] #6 - WholeStageCodegen (3) - Project [cs_call_center_sk,sales,returns,profit,profit_loss] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometColumnarExchange [cs_call_center_sk] #10 - CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ColumnarToRow - InputAdapter - CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometColumnarExchange #11 - CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometProject [s_store_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] + CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] + CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_sk] #3 + CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #6 + CometFilter [s_store_sk] + CometScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [s_store_sk,returns,profit_loss] #7 + CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] + CometExchange [s_store_sk] #8 + CometHashAggregate [s_store_sk,sum,sum,sr_return_amt,sr_net_loss] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] - CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] - CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - CometColumnarExchange [wp_web_page_sk] #12 - CometHashAggregate [wp_web_page_sk,sum,sum,ws_ext_sales_price,ws_net_profit] - CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] - CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [wp_web_page_sk] #13 - CometFilter [wp_web_page_sk] - CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #14 - CometHashAggregate [wp_web_page_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] - CometColumnarExchange [wp_web_page_sk] #15 - CometHashAggregate [wp_web_page_sk,sum,sum,wr_return_amt,wr_net_loss] - CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] - CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] - CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [wp_web_page_sk] #13 - WholeStageCodegen (13) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel] #16 - RowToColumnar - WholeStageCodegen (12) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (20) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #17 - RowToColumnar - WholeStageCodegen (19) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + ReusedExchange [d_date_sk] #5 + ReusedExchange [s_store_sk] #6 + WholeStageCodegen (3) + Project [cs_call_center_sk,sales,returns,profit,profit_loss] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] + CometExchange [cs_call_center_sk] #10 + CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 ColumnarToRow InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] + CometExchange #11 + CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] + CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] + CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] + CometExchange [wp_web_page_sk] #12 + CometHashAggregate [wp_web_page_sk,sum,sum,ws_ext_sales_price,ws_net_profit] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [wp_web_page_sk] #13 + CometFilter [wp_web_page_sk] + CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #14 + CometHashAggregate [wp_web_page_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] + CometExchange [wp_web_page_sk] #15 + CometHashAggregate [wp_web_page_sk,sum,sum,wr_return_amt,wr_net_loss] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + ReusedExchange [wp_web_page_sk] #13 + WholeStageCodegen (13) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel] #16 + WholeStageCodegen (12) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (20) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange #17 + WholeStageCodegen (19) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/explain.txt index 9304cb85f..ac26f8d51 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q78/explain.txt @@ -7,7 +7,7 @@ TakeOrderedAndProject (67) : +- CometSortMergeJoin (42) : :- CometSort (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -15,11 +15,11 @@ TakeOrderedAndProject (67) : : : +- CometFilter (11) : : : +- CometSortMergeJoin (10) : : : :- CometSort (4) - : : : : +- CometColumnarExchange (3) + : : : : +- CometExchange (3) : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- CometSort (9) - : : : +- CometColumnarExchange (8) + : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) : : : +- CometScan parquet spark_catalog.default.store_returns (5) @@ -29,7 +29,7 @@ TakeOrderedAndProject (67) : +- CometSort (41) : +- CometFilter (40) : +- CometHashAggregate (39) - : +- CometColumnarExchange (38) + : +- CometExchange (38) : +- CometHashAggregate (37) : +- CometProject (36) : +- CometBroadcastHashJoin (35) @@ -37,11 +37,11 @@ TakeOrderedAndProject (67) : : +- CometFilter (32) : : +- CometSortMergeJoin (31) : : :- CometSort (25) - : : : +- CometColumnarExchange (24) + : : : +- CometExchange (24) : : : +- CometFilter (23) : : : +- CometScan parquet spark_catalog.default.web_sales (22) : : +- CometSort (30) - : : +- CometColumnarExchange (29) + : : +- CometExchange (29) : : +- CometProject (28) : : +- CometFilter (27) : : +- CometScan parquet spark_catalog.default.web_returns (26) @@ -49,7 +49,7 @@ TakeOrderedAndProject (67) +- CometSort (63) +- CometFilter (62) +- CometHashAggregate (61) - +- CometColumnarExchange (60) + +- CometExchange (60) +- CometHashAggregate (59) +- CometProject (58) +- CometBroadcastHashJoin (57) @@ -57,11 +57,11 @@ TakeOrderedAndProject (67) : +- CometFilter (54) : +- CometSortMergeJoin (53) : :- CometSort (47) - : : +- CometColumnarExchange (46) + : : +- CometExchange (46) : : +- CometFilter (45) : : +- CometScan parquet spark_catalog.default.catalog_sales (44) : +- CometSort (52) - : +- CometColumnarExchange (51) + : +- CometExchange (51) : +- CometProject (50) : +- CometFilter (49) : +- CometScan parquet spark_catalog.default.catalog_returns (48) @@ -80,9 +80,9 @@ ReadSchema: struct -(127) CometFilter +(117) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1998-08-04)) AND (d_date#15 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) -(128) CometProject +(118) CometProject Input [2]: [d_date_sk#14, d_date#15] Arguments: [d_date_sk#14], [d_date_sk#14] -(129) ColumnarToRow [codegen id : 1] +(119) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(130) BroadcastExchange +(120) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/simplified.txt index 210c9b6d2..aa61bbd57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q80a/simplified.txt @@ -1,157 +1,147 @@ -WholeStageCodegen (19) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,id,sales,returns,profit] - CometHashAggregate [channel,id,sales,returns,profit] - CometColumnarExchange [channel,id,sales,returns,profit] #1 - RowToColumnar - WholeStageCodegen (18) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id] #2 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_id] #3 - CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk,ss_ticket_number] #4 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometColumnarExchange [sr_item_sk,sr_ticket_number] #6 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #7 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #8 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #9 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #10 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [cp_catalog_page_id] #11 - CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometColumnarExchange [cs_item_sk,cs_order_number] #12 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometColumnarExchange [cr_item_sk,cr_order_number] #13 - CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #14 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [web_site_id] #15 - CometHashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] - CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] - CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] - CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometColumnarExchange [ws_item_sk,ws_order_number] #16 - CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometColumnarExchange [wr_item_sk,wr_order_number] #17 - CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [web_site_sk,web_site_id] #18 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 - WholeStageCodegen (11) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel] #19 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #20 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (19) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Exchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #4 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #6 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #8 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #9 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #10 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #11 + CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #12 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #13 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #7 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #14 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #9 + ReusedExchange [p_promo_sk] #10 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [web_site_id] #15 + CometHashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #16 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #17 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #7 + CometBroadcastExchange [web_site_sk,web_site_id] #18 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #9 + ReusedExchange [p_promo_sk] #10 + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel] #19 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange #20 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/explain.txt index c2c6f352b..6f4890bd3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/explain.txt @@ -4,13 +4,13 @@ TakeOrderedAndProject (36) +- Window (34) +- * ColumnarToRow (33) +- CometSort (32) - +- CometColumnarExchange (31) + +- CometExchange (31) +- CometHashAggregate (30) - +- CometColumnarExchange (29) + +- CometExchange (29) +- CometHashAggregate (28) +- CometUnion (27) :- CometHashAggregate (16) - : +- CometColumnarExchange (15) + : +- CometExchange (15) : +- CometHashAggregate (14) : +- CometProject (13) : +- CometBroadcastHashJoin (12) @@ -26,12 +26,12 @@ TakeOrderedAndProject (36) : +- CometFilter (10) : +- CometScan parquet spark_catalog.default.item (9) :- CometHashAggregate (21) - : +- CometColumnarExchange (20) + : +- CometExchange (20) : +- CometHashAggregate (19) : +- CometHashAggregate (18) : +- ReusedExchange (17) +- CometHashAggregate (26) - +- CometColumnarExchange (25) + +- CometExchange (25) +- CometHashAggregate (24) +- CometHashAggregate (23) +- ReusedExchange (22) @@ -106,9 +106,9 @@ Input [3]: [ws_net_paid#2, i_class#8, i_category#9] Keys [2]: [i_category#9, i_class#8] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [i_category#9, i_class#8, sum#10] -Arguments: hashpartitioning(i_category#9, i_class#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#9, i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [i_category#9, i_class#8, sum#10] @@ -128,9 +128,9 @@ Input [2]: [total_sum#15, i_category#11] Keys [1]: [i_category#11] Functions [1]: [partial_sum(total_sum#15)] -(20) CometColumnarExchange +(20) CometExchange Input [3]: [i_category#11, sum#16, isEmpty#17] -Arguments: hashpartitioning(i_category#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (21) CometHashAggregate Input [3]: [i_category#11, sum#16, isEmpty#17] @@ -150,9 +150,9 @@ Input [1]: [total_sum#22] Keys: [] Functions [1]: [partial_sum(total_sum#22)] -(25) CometColumnarExchange +(25) CometExchange Input [2]: [sum#23, isEmpty#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (26) CometHashAggregate Input [2]: [sum#23, isEmpty#24] @@ -169,18 +169,18 @@ Input [6]: [total_sum#25, i_category#9, i_class#8, g_category#26, g_class#27, lo Keys [6]: [total_sum#25, i_category#9, i_class#8, g_category#26, g_class#27, lochierarchy#28] Functions: [] -(29) CometColumnarExchange +(29) CometExchange Input [6]: [total_sum#25, i_category#9, i_class#8, g_category#26, g_class#27, lochierarchy#28] -Arguments: hashpartitioning(total_sum#25, i_category#9, i_class#8, g_category#26, g_class#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(total_sum#25, i_category#9, i_class#8, g_category#26, g_class#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (30) CometHashAggregate Input [6]: [total_sum#25, i_category#9, i_class#8, g_category#26, g_class#27, lochierarchy#28] Keys [6]: [total_sum#25, i_category#9, i_class#8, g_category#26, g_class#27, lochierarchy#28] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [5]: [total_sum#25, i_category#9, i_class#8, lochierarchy#28, _w0#40] -Arguments: hashpartitioning(lochierarchy#28, _w0#40, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(lochierarchy#28, _w0#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (32) CometSort Input [5]: [total_sum#25, i_category#9, i_class#8, lochierarchy#28, _w0#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt index 255e4acc3..ae161e5ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q86a/simplified.txt @@ -7,13 +7,13 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl ColumnarToRow InputAdapter CometSort [total_sum,i_category,i_class,lochierarchy,_w0] - CometColumnarExchange [lochierarchy,_w0] #1 + CometExchange [lochierarchy,_w0] #1 CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,g_category,g_class] - CometColumnarExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 + CometExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] CometUnion [total_sum,i_category,i_class,g_category,g_class,lochierarchy] CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [i_category,i_class] #3 + CometExchange [i_category,i_class] #3 CometHashAggregate [i_category,i_class,sum,ws_net_paid] CometProject [ws_net_paid,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] @@ -37,12 +37,12 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometFilter [i_item_sk,i_class,i_category] CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] - CometColumnarExchange [i_category] #7 + CometExchange [i_category] #7 CometHashAggregate [i_category,sum,isEmpty,total_sum] CometHashAggregate [total_sum,i_category,i_class,sum,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] - CometColumnarExchange #8 + CometExchange #8 CometHashAggregate [sum,isEmpty,total_sum] CometHashAggregate [total_sum,i_category,i_class,sum,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt index 8b0facb13..e85baa1ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/explain.txt @@ -7,9 +7,9 @@ +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -95,18 +95,18 @@ Input [6]: [ss_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt index cbc817472..989ade02c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark3_5/q98/simplified.txt @@ -12,9 +12,9 @@ WholeStageCodegen (3) ColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometColumnarExchange [i_class] #2 + CometExchange [i_class] #2 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt index f2de73f90..fc449ff52 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (39) +- CometTakeOrderedAndProject (38) +- CometHashAggregate (37) - +- CometColumnarExchange (36) + +- CometExchange (36) +- CometHashAggregate (35) +- CometProject (34) +- CometBroadcastHashJoin (33) @@ -207,9 +207,9 @@ Input [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purch Keys [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Functions [1]: [partial_count(1)] -(36) CometColumnarExchange +(36) CometExchange Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#31] -Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (37) CometHashAggregate Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt index 243bdbb91..2e8f2022c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q10a/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6] CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6,count,count(1)] - CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] CometProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometBroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt index 16a25285c..2b4fa79cd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/explain.txt @@ -1,83 +1,75 @@ == Physical Plan == -TakeOrderedAndProject (79) -+- * Project (78) - +- * BroadcastHashJoin Inner BuildRight (77) - :- * Project (59) - : +- * BroadcastHashJoin Inner BuildRight (58) - : :- * BroadcastHashJoin Inner BuildRight (39) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- * ColumnarToRow (17) - : : : +- CometColumnarExchange (16) - : : : +- RowToColumnar (15) - : : : +- * HashAggregate (14) - : : : +- * ColumnarToRow (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- BroadcastExchange (38) - : : +- * HashAggregate (37) - : : +- * ColumnarToRow (36) - : : +- CometColumnarExchange (35) - : : +- RowToColumnar (34) - : : +- * HashAggregate (33) - : : +- * ColumnarToRow (32) - : : +- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (26) - : : : +- CometBroadcastHashJoin (25) - : : : :- CometFilter (21) - : : : : +- CometScan parquet spark_catalog.default.customer (20) - : : : +- CometBroadcastExchange (24) - : : : +- CometFilter (23) - : : : +- CometScan parquet spark_catalog.default.store_sales (22) - : : +- CometBroadcastExchange (29) - : : +- CometFilter (28) - : : +- CometScan parquet spark_catalog.default.date_dim (27) - : +- BroadcastExchange (57) - : +- * Filter (56) - : +- * HashAggregate (55) - : +- * ColumnarToRow (54) - : +- CometColumnarExchange (53) - : +- RowToColumnar (52) - : +- * HashAggregate (51) - : +- * ColumnarToRow (50) - : +- CometProject (49) - : +- CometBroadcastHashJoin (48) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometFilter (41) - : : : +- CometScan parquet spark_catalog.default.customer (40) - : : +- CometBroadcastExchange (44) - : : +- CometFilter (43) - : : +- CometScan parquet spark_catalog.default.web_sales (42) - : +- ReusedExchange (47) - +- BroadcastExchange (76) - +- * HashAggregate (75) - +- * ColumnarToRow (74) - +- CometColumnarExchange (73) - +- RowToColumnar (72) - +- * HashAggregate (71) - +- * ColumnarToRow (70) - +- CometProject (69) - +- CometBroadcastHashJoin (68) - :- CometProject (66) - : +- CometBroadcastHashJoin (65) - : :- CometFilter (61) - : : +- CometScan parquet spark_catalog.default.customer (60) - : +- CometBroadcastExchange (64) - : +- CometFilter (63) - : +- CometScan parquet spark_catalog.default.web_sales (62) - +- ReusedExchange (67) +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (53) + : +- * BroadcastHashJoin Inner BuildRight (52) + : :- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (17) + : : : +- * HashAggregate (16) + : : : +- Exchange (15) + : : : +- * HashAggregate (14) + : : : +- * ColumnarToRow (13) + : : : +- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- Exchange (32) + : : +- * HashAggregate (31) + : : +- * ColumnarToRow (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometFilter (19) + : : : : +- CometScan parquet spark_catalog.default.customer (18) + : : : +- CometBroadcastExchange (22) + : : : +- CometFilter (21) + : : : +- CometScan parquet spark_catalog.default.store_sales (20) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (51) + : +- * Filter (50) + : +- * HashAggregate (49) + : +- Exchange (48) + : +- * HashAggregate (47) + : +- * ColumnarToRow (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (42) + : : +- CometBroadcastHashJoin (41) + : : :- CometFilter (37) + : : : +- CometScan parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (40) + : : +- CometFilter (39) + : : +- CometScan parquet spark_catalog.default.web_sales (38) + : +- ReusedExchange (43) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- Exchange (66) + +- * HashAggregate (65) + +- * ColumnarToRow (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometFilter (55) + : : +- CometScan parquet spark_catalog.default.customer (54) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometScan parquet spark_catalog.default.web_sales (56) + +- ReusedExchange (61) (1) Scan parquet spark_catalog.default.customer @@ -150,39 +142,33 @@ Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discoun Aggregate Attributes [1]: [sum#16] Results [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#17] -(15) RowToColumnar +(15) Exchange Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#17] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(16) CometColumnarExchange -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#17] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(17) ColumnarToRow [codegen id : 8] -Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#17] - -(18) HashAggregate [codegen id : 8] +(16) HashAggregate [codegen id : 8] Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#17] Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#18] Results [2]: [c_customer_id#2 AS customer_id#19, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))#18,18,2) AS year_total#20] -(19) Filter [codegen id : 8] +(17) Filter [codegen id : 8] Input [2]: [customer_id#19, year_total#20] Condition : (isnotnull(year_total#20) AND (year_total#20 > 0.00)) -(20) Scan parquet spark_catalog.default.customer +(18) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(21) CometFilter +(19) CometFilter Input [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] Condition : (isnotnull(c_customer_sk#21) AND isnotnull(c_customer_id#22)) -(22) Scan parquet spark_catalog.default.store_sales +(20) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Batched: true Location: InMemoryFileIndex [] @@ -190,96 +176,90 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#32), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(23) CometFilter +(21) CometFilter Input [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Condition : isnotnull(ss_customer_sk#29) -(24) CometBroadcastExchange +(22) CometBroadcastExchange Input [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Arguments: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] -(25) CometBroadcastHashJoin +(23) CometBroadcastHashJoin Left output [8]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] Right output [4]: [ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Arguments: [c_customer_sk#21], [ss_customer_sk#29], Inner, BuildRight -(26) CometProject +(24) CometProject Input [12]: [c_customer_sk#21, c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_customer_sk#29, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32], [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] -(27) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#34, d_year#35] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [2]: [d_date_sk#34, d_year#35] Condition : ((isnotnull(d_year#35) AND (d_year#35 = 2002)) AND isnotnull(d_date_sk#34)) -(29) CometBroadcastExchange +(27) CometBroadcastExchange Input [2]: [d_date_sk#34, d_year#35] Arguments: [d_date_sk#34, d_year#35] -(30) CometBroadcastHashJoin +(28) CometBroadcastHashJoin Left output [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32] Right output [2]: [d_date_sk#34, d_year#35] Arguments: [ss_sold_date_sk#32], [d_date_sk#34], Inner, BuildRight -(31) CometProject +(29) CometProject Input [12]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, ss_sold_date_sk#32, d_date_sk#34, d_year#35] Arguments: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, d_year#35], [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, d_year#35] -(32) ColumnarToRow [codegen id : 2] +(30) ColumnarToRow [codegen id : 2] Input [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, d_year#35] -(33) HashAggregate [codegen id : 2] +(31) HashAggregate [codegen id : 2] Input [10]: [c_customer_id#22, c_first_name#23, c_last_name#24, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, ss_ext_discount_amt#30, ss_ext_list_price#31, d_year#35] Keys [8]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#31 - ss_ext_discount_amt#30)))] Aggregate Attributes [1]: [sum#36] Results [9]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, sum#37] -(34) RowToColumnar -Input [9]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, sum#37] - -(35) CometColumnarExchange -Input [9]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, sum#37] -Arguments: hashpartitioning(c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(36) ColumnarToRow [codegen id : 3] +(32) Exchange Input [9]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, sum#37] +Arguments: hashpartitioning(c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(37) HashAggregate [codegen id : 3] +(33) HashAggregate [codegen id : 3] Input [9]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28, sum#37] Keys [8]: [c_customer_id#22, c_first_name#23, c_last_name#24, d_year#35, c_preferred_cust_flag#25, c_birth_country#26, c_login#27, c_email_address#28] Functions [1]: [sum(UnscaledValue((ss_ext_list_price#31 - ss_ext_discount_amt#30)))] Aggregate Attributes [1]: [sum(UnscaledValue((ss_ext_list_price#31 - ss_ext_discount_amt#30)))#18] Results [5]: [c_customer_id#22 AS customer_id#38, c_first_name#23 AS customer_first_name#39, c_last_name#24 AS customer_last_name#40, c_email_address#28 AS customer_email_address#41, MakeDecimal(sum(UnscaledValue((ss_ext_list_price#31 - ss_ext_discount_amt#30)))#18,18,2) AS year_total#42] -(38) BroadcastExchange +(34) BroadcastExchange Input [5]: [customer_id#38, customer_first_name#39, customer_last_name#40, customer_email_address#41, year_total#42] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(39) BroadcastHashJoin [codegen id : 8] +(35) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#19] Right keys [1]: [customer_id#38] Join type: Inner Join condition: None -(40) Scan parquet spark_catalog.default.customer +(36) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#43, c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(41) CometFilter +(37) CometFilter Input [8]: [c_customer_sk#43, c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50] Condition : (isnotnull(c_customer_sk#43) AND isnotnull(c_customer_id#44)) -(42) Scan parquet spark_catalog.default.web_sales +(38) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] Batched: true Location: InMemoryFileIndex [] @@ -287,92 +267,86 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#54), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(43) CometFilter +(39) CometFilter Input [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] Condition : isnotnull(ws_bill_customer_sk#51) -(44) CometBroadcastExchange +(40) CometBroadcastExchange Input [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] Arguments: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] -(45) CometBroadcastHashJoin +(41) CometBroadcastHashJoin Left output [8]: [c_customer_sk#43, c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50] Right output [4]: [ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] Arguments: [c_customer_sk#43], [ws_bill_customer_sk#51], Inner, BuildRight -(46) CometProject +(42) CometProject Input [12]: [c_customer_sk#43, c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, ws_bill_customer_sk#51, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] Arguments: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54], [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] -(47) ReusedExchange [Reuses operator id: 10] +(43) ReusedExchange [Reuses operator id: 10] Output [2]: [d_date_sk#56, d_year#57] -(48) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [10]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54] Right output [2]: [d_date_sk#56, d_year#57] Arguments: [ws_sold_date_sk#54], [d_date_sk#56], Inner, BuildRight -(49) CometProject +(45) CometProject Input [12]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, ws_ext_discount_amt#52, ws_ext_list_price#53, ws_sold_date_sk#54, d_date_sk#56, d_year#57] Arguments: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, ws_ext_discount_amt#52, ws_ext_list_price#53, d_year#57], [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, ws_ext_discount_amt#52, ws_ext_list_price#53, d_year#57] -(50) ColumnarToRow [codegen id : 4] +(46) ColumnarToRow [codegen id : 4] Input [10]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, ws_ext_discount_amt#52, ws_ext_list_price#53, d_year#57] -(51) HashAggregate [codegen id : 4] +(47) HashAggregate [codegen id : 4] Input [10]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, ws_ext_discount_amt#52, ws_ext_list_price#53, d_year#57] Keys [8]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, d_year#57] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#53 - ws_ext_discount_amt#52)))] Aggregate Attributes [1]: [sum#58] Results [9]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, d_year#57, sum#59] -(52) RowToColumnar -Input [9]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, d_year#57, sum#59] - -(53) CometColumnarExchange -Input [9]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, d_year#57, sum#59] -Arguments: hashpartitioning(c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, d_year#57, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(54) ColumnarToRow [codegen id : 5] +(48) Exchange Input [9]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, d_year#57, sum#59] +Arguments: hashpartitioning(c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, d_year#57, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(55) HashAggregate [codegen id : 5] +(49) HashAggregate [codegen id : 5] Input [9]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, d_year#57, sum#59] Keys [8]: [c_customer_id#44, c_first_name#45, c_last_name#46, c_preferred_cust_flag#47, c_birth_country#48, c_login#49, c_email_address#50, d_year#57] Functions [1]: [sum(UnscaledValue((ws_ext_list_price#53 - ws_ext_discount_amt#52)))] Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#53 - ws_ext_discount_amt#52)))#60] Results [2]: [c_customer_id#44 AS customer_id#61, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#53 - ws_ext_discount_amt#52)))#60,18,2) AS year_total#62] -(56) Filter [codegen id : 5] +(50) Filter [codegen id : 5] Input [2]: [customer_id#61, year_total#62] Condition : (isnotnull(year_total#62) AND (year_total#62 > 0.00)) -(57) BroadcastExchange +(51) BroadcastExchange Input [2]: [customer_id#61, year_total#62] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(58) BroadcastHashJoin [codegen id : 8] +(52) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#19] Right keys [1]: [customer_id#61] Join type: Inner Join condition: None -(59) Project [codegen id : 8] +(53) Project [codegen id : 8] Output [8]: [customer_id#19, year_total#20, customer_id#38, customer_first_name#39, customer_last_name#40, customer_email_address#41, year_total#42, year_total#62] Input [9]: [customer_id#19, year_total#20, customer_id#38, customer_first_name#39, customer_last_name#40, customer_email_address#41, year_total#42, customer_id#61, year_total#62] -(60) Scan parquet spark_catalog.default.customer +(54) Scan parquet spark_catalog.default.customer Output [8]: [c_customer_sk#63, c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(61) CometFilter +(55) CometFilter Input [8]: [c_customer_sk#63, c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70] Condition : (isnotnull(c_customer_sk#63) AND isnotnull(c_customer_id#64)) -(62) Scan parquet spark_catalog.default.web_sales +(56) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_bill_customer_sk#71, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] Batched: true Location: InMemoryFileIndex [] @@ -380,134 +354,128 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#74), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(63) CometFilter +(57) CometFilter Input [4]: [ws_bill_customer_sk#71, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] Condition : isnotnull(ws_bill_customer_sk#71) -(64) CometBroadcastExchange +(58) CometBroadcastExchange Input [4]: [ws_bill_customer_sk#71, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] Arguments: [ws_bill_customer_sk#71, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] -(65) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [8]: [c_customer_sk#63, c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70] Right output [4]: [ws_bill_customer_sk#71, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] Arguments: [c_customer_sk#63], [ws_bill_customer_sk#71], Inner, BuildRight -(66) CometProject +(60) CometProject Input [12]: [c_customer_sk#63, c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, ws_bill_customer_sk#71, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] Arguments: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74], [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] -(67) ReusedExchange [Reuses operator id: 29] +(61) ReusedExchange [Reuses operator id: 27] Output [2]: [d_date_sk#76, d_year#77] -(68) CometBroadcastHashJoin +(62) CometBroadcastHashJoin Left output [10]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74] Right output [2]: [d_date_sk#76, d_year#77] Arguments: [ws_sold_date_sk#74], [d_date_sk#76], Inner, BuildRight -(69) CometProject +(63) CometProject Input [12]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, ws_ext_discount_amt#72, ws_ext_list_price#73, ws_sold_date_sk#74, d_date_sk#76, d_year#77] Arguments: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, ws_ext_discount_amt#72, ws_ext_list_price#73, d_year#77], [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, ws_ext_discount_amt#72, ws_ext_list_price#73, d_year#77] -(70) ColumnarToRow [codegen id : 6] +(64) ColumnarToRow [codegen id : 6] Input [10]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, ws_ext_discount_amt#72, ws_ext_list_price#73, d_year#77] -(71) HashAggregate [codegen id : 6] +(65) HashAggregate [codegen id : 6] Input [10]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, ws_ext_discount_amt#72, ws_ext_list_price#73, d_year#77] Keys [8]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, d_year#77] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#73 - ws_ext_discount_amt#72)))] Aggregate Attributes [1]: [sum#78] Results [9]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, d_year#77, sum#79] -(72) RowToColumnar -Input [9]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, d_year#77, sum#79] - -(73) CometColumnarExchange -Input [9]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, d_year#77, sum#79] -Arguments: hashpartitioning(c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, d_year#77, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(74) ColumnarToRow [codegen id : 7] +(66) Exchange Input [9]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, d_year#77, sum#79] +Arguments: hashpartitioning(c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, d_year#77, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(75) HashAggregate [codegen id : 7] +(67) HashAggregate [codegen id : 7] Input [9]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, d_year#77, sum#79] Keys [8]: [c_customer_id#64, c_first_name#65, c_last_name#66, c_preferred_cust_flag#67, c_birth_country#68, c_login#69, c_email_address#70, d_year#77] Functions [1]: [sum(UnscaledValue((ws_ext_list_price#73 - ws_ext_discount_amt#72)))] Aggregate Attributes [1]: [sum(UnscaledValue((ws_ext_list_price#73 - ws_ext_discount_amt#72)))#60] Results [2]: [c_customer_id#64 AS customer_id#80, MakeDecimal(sum(UnscaledValue((ws_ext_list_price#73 - ws_ext_discount_amt#72)))#60,18,2) AS year_total#81] -(76) BroadcastExchange +(68) BroadcastExchange Input [2]: [customer_id#80, year_total#81] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(77) BroadcastHashJoin [codegen id : 8] +(69) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#19] Right keys [1]: [customer_id#80] Join type: Inner Join condition: (CASE WHEN (year_total#62 > 0.00) THEN (year_total#81 / year_total#62) ELSE 0E-20 END > CASE WHEN (year_total#20 > 0.00) THEN (year_total#42 / year_total#20) ELSE 0E-20 END) -(78) Project [codegen id : 8] +(70) Project [codegen id : 8] Output [4]: [customer_id#38, customer_first_name#39, customer_last_name#40, customer_email_address#41] Input [10]: [customer_id#19, year_total#20, customer_id#38, customer_first_name#39, customer_last_name#40, customer_email_address#41, year_total#42, year_total#62, customer_id#80, year_total#81] -(79) TakeOrderedAndProject +(71) TakeOrderedAndProject Input [4]: [customer_id#38, customer_first_name#39, customer_last_name#40, customer_email_address#41] Arguments: 100, [customer_id#38 ASC NULLS FIRST, customer_first_name#39 ASC NULLS FIRST, customer_last_name#40 ASC NULLS FIRST, customer_email_address#41 ASC NULLS FIRST], [customer_id#38, customer_first_name#39, customer_last_name#40, customer_email_address#41] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (83) -+- * ColumnarToRow (82) - +- CometFilter (81) - +- CometScan parquet spark_catalog.default.date_dim (80) +BroadcastExchange (75) ++- * ColumnarToRow (74) + +- CometFilter (73) + +- CometScan parquet spark_catalog.default.date_dim (72) -(80) Scan parquet spark_catalog.default.date_dim +(72) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_year#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(81) CometFilter +(73) CometFilter Input [2]: [d_date_sk#14, d_year#15] Condition : ((isnotnull(d_year#15) AND (d_year#15 = 2001)) AND isnotnull(d_date_sk#14)) -(82) ColumnarToRow [codegen id : 1] +(74) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#14, d_year#15] -(83) BroadcastExchange +(75) BroadcastExchange Input [2]: [d_date_sk#14, d_year#15] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#32 IN dynamicpruning#33 -BroadcastExchange (87) -+- * ColumnarToRow (86) - +- CometFilter (85) - +- CometScan parquet spark_catalog.default.date_dim (84) +Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#32 IN dynamicpruning#33 +BroadcastExchange (79) ++- * ColumnarToRow (78) + +- CometFilter (77) + +- CometScan parquet spark_catalog.default.date_dim (76) -(84) Scan parquet spark_catalog.default.date_dim +(76) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#34, d_year#35] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(85) CometFilter +(77) CometFilter Input [2]: [d_date_sk#34, d_year#35] Condition : ((isnotnull(d_year#35) AND (d_year#35 = 2002)) AND isnotnull(d_date_sk#34)) -(86) ColumnarToRow [codegen id : 1] +(78) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#34, d_year#35] -(87) BroadcastExchange +(79) BroadcastExchange Input [2]: [d_date_sk#34, d_year#35] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] -Subquery:3 Hosting operator id = 42 Hosting Expression = ws_sold_date_sk#54 IN dynamicpruning#13 +Subquery:3 Hosting operator id = 38 Hosting Expression = ws_sold_date_sk#54 IN dynamicpruning#13 -Subquery:4 Hosting operator id = 62 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#33 +Subquery:4 Hosting operator id = 56 Hosting Expression = ws_sold_date_sk#74 IN dynamicpruning#33 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt index 916aff7c8..d7c6ef3b2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q11/simplified.txt @@ -7,11 +7,38 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom BroadcastHashJoin [customer_id,customer_id] Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 - RowToColumnar - WholeStageCodegen (1) + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,sum] + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + WholeStageCodegen (2) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] ColumnarToRow InputAdapter @@ -21,94 +48,59 @@ TakeOrderedAndProject [customer_id,customer_first_name,customer_last_name,custom CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #2 + CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #8 WholeStageCodegen (1) ColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum] [sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt))),customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_list_price,ss_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #10 WholeStageCodegen (5) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + WholeStageCodegen (4) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #12 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_year] #4 InputAdapter BroadcastExchange #13 WholeStageCodegen (7) HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum] [sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt))),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] - CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + WholeStageCodegen (6) + HashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,ws_ext_list_price,ws_ext_discount_amt] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] + CometBroadcastExchange [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] #15 + CometFilter [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt index 6144d0adc..9160db4e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/explain.txt @@ -1,30 +1,26 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * Project (25) - +- Window (24) - +- * ColumnarToRow (23) - +- CometSort (22) - +- CometColumnarExchange (21) - +- RowToColumnar (20) - +- * HashAggregate (19) - +- * ColumnarToRow (18) - +- CometColumnarExchange (17) - +- RowToColumnar (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.web_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) +TakeOrderedAndProject (22) ++- * Project (21) + +- Window (20) + +- * Sort (19) + +- Exchange (18) + +- * HashAggregate (17) + +- Exchange (16) + +- * HashAggregate (15) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometBroadcastHashJoin (12) + :- CometProject (7) + : +- CometBroadcastHashJoin (6) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.web_sales (1) + : +- CometBroadcastExchange (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (11) + +- CometProject (10) + +- CometFilter (9) + +- CometScan parquet spark_catalog.default.date_dim (8) (1) Scan parquet spark_catalog.default.web_sales @@ -101,78 +97,66 @@ Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] Aggregate Attributes [1]: [sum#13] Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -(16) RowToColumnar +(16) Exchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 2] -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] - -(19) HashAggregate [codegen id : 2] +(17) HashAggregate [codegen id : 2] Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_ext_sales_price#2))#15] Results [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#15,17,2) AS itemrevenue#16, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#2))#15,17,2) AS _w0#17] -(20) RowToColumnar -Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] - -(21) CometColumnarExchange -Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(22) CometSort +(18) Exchange Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] -Arguments: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17], [i_class#9 ASC NULLS FIRST] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(23) ColumnarToRow [codegen id : 3] +(19) Sort [codegen id : 3] Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] +Arguments: [i_class#9 ASC NULLS FIRST], false, 0 -(24) Window +(20) Window Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] Arguments: [sum(_w0#17) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#9] -(25) Project [codegen id : 4] +(21) Project [codegen id : 4] Output [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, _we0#18] -(26) TakeOrderedAndProject +(22) TakeOrderedAndProject Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * ColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.date_dim (27) +BroadcastExchange (27) ++- * ColumnarToRow (26) + +- CometProject (25) + +- CometFilter (24) + +- CometScan parquet spark_catalog.default.date_dim (23) -(27) Scan parquet spark_catalog.default.date_dim +(23) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(24) CometFilter Input [2]: [d_date_sk#11, d_date#12] Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 1999-02-22)) AND (d_date#12 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(29) CometProject +(25) CometProject Input [2]: [d_date_sk#11, d_date#12] Arguments: [d_date_sk#11], [d_date_sk#11] -(30) ColumnarToRow [codegen id : 1] +(26) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#11] -(31) BroadcastExchange +(27) BroadcastExchange Input [1]: [d_date_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt index 663689cf7..07c91b94d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q12/simplified.txt @@ -4,39 +4,35 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (3) - ColumnarToRow + Sort [i_class] InputAdapter - CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometColumnarExchange [i_class] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + Exchange [i_class] #1 + WholeStageCodegen (2) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ws_ext_sales_price)),itemrevenue,_w0,sum] + InputAdapter + Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ws_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_ext_sales_price,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt index 2479fafef..5c0b28c77 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/explain.txt @@ -1,94 +1,90 @@ == Physical Plan == -TakeOrderedAndProject (90) -+- * BroadcastHashJoin Inner BuildRight (89) - :- * Filter (67) - : +- * HashAggregate (66) - : +- * ColumnarToRow (65) - : +- CometColumnarExchange (64) - : +- RowToColumnar (63) - : +- * HashAggregate (62) - : +- * ColumnarToRow (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (54) - : : +- CometBroadcastHashJoin (53) - : : :- CometBroadcastHashJoin (47) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (46) - : : : +- CometProject (45) - : : : +- CometBroadcastHashJoin (44) - : : : :- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.item (3) - : : : +- CometBroadcastExchange (43) - : : : +- CometBroadcastHashJoin (42) - : : : :- CometHashAggregate (32) - : : : : +- CometColumnarExchange (31) - : : : : +- CometHashAggregate (30) - : : : : +- CometProject (29) - : : : : +- CometBroadcastHashJoin (28) - : : : : :- CometProject (26) - : : : : : +- CometBroadcastHashJoin (25) - : : : : : :- CometFilter (6) - : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) - : : : : : +- CometBroadcastExchange (24) - : : : : : +- CometBroadcastHashJoin (23) - : : : : : :- CometFilter (8) - : : : : : : +- CometScan parquet spark_catalog.default.item (7) - : : : : : +- CometBroadcastExchange (22) - : : : : : +- CometProject (21) - : : : : : +- CometBroadcastHashJoin (20) - : : : : : :- CometProject (15) - : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : :- CometFilter (10) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) - : : : : : : +- CometBroadcastExchange (13) - : : : : : : +- CometFilter (12) - : : : : : : +- CometScan parquet spark_catalog.default.item (11) - : : : : : +- CometBroadcastExchange (19) - : : : : : +- CometProject (18) - : : : : : +- CometFilter (17) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) - : : : : +- ReusedExchange (27) - : : : +- CometBroadcastExchange (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometProject (37) - : : : : +- CometBroadcastHashJoin (36) - : : : : :- CometFilter (34) - : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) - : : : : +- ReusedExchange (35) - : : : +- ReusedExchange (38) - : : +- CometBroadcastExchange (52) - : : +- CometBroadcastHashJoin (51) - : : :- CometFilter (49) - : : : +- CometScan parquet spark_catalog.default.item (48) - : : +- ReusedExchange (50) - : +- CometBroadcastExchange (58) - : +- CometProject (57) - : +- CometFilter (56) - : +- CometScan parquet spark_catalog.default.date_dim (55) - +- BroadcastExchange (88) - +- * Filter (87) - +- * HashAggregate (86) - +- * ColumnarToRow (85) - +- CometColumnarExchange (84) - +- RowToColumnar (83) - +- * HashAggregate (82) - +- * ColumnarToRow (81) - +- CometProject (80) - +- CometBroadcastHashJoin (79) - :- CometProject (74) - : +- CometBroadcastHashJoin (73) - : :- CometBroadcastHashJoin (71) - : : :- CometFilter (69) - : : : +- CometScan parquet spark_catalog.default.store_sales (68) - : : +- ReusedExchange (70) - : +- ReusedExchange (72) - +- CometBroadcastExchange (78) - +- CometProject (77) - +- CometFilter (76) - +- CometScan parquet spark_catalog.default.date_dim (75) +TakeOrderedAndProject (86) ++- * BroadcastHashJoin Inner BuildRight (85) + :- * Filter (65) + : +- * HashAggregate (64) + : +- Exchange (63) + : +- * HashAggregate (62) + : +- * ColumnarToRow (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (54) + : : +- CometBroadcastHashJoin (53) + : : :- CometBroadcastHashJoin (47) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (46) + : : : +- CometProject (45) + : : : +- CometBroadcastHashJoin (44) + : : : :- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (43) + : : : +- CometBroadcastHashJoin (42) + : : : :- CometHashAggregate (32) + : : : : +- CometExchange (31) + : : : : +- CometHashAggregate (30) + : : : : +- CometProject (29) + : : : : +- CometBroadcastHashJoin (28) + : : : : :- CometProject (26) + : : : : : +- CometBroadcastHashJoin (25) + : : : : : :- CometFilter (6) + : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) + : : : : : +- CometBroadcastExchange (24) + : : : : : +- CometBroadcastHashJoin (23) + : : : : : :- CometFilter (8) + : : : : : : +- CometScan parquet spark_catalog.default.item (7) + : : : : : +- CometBroadcastExchange (22) + : : : : : +- CometProject (21) + : : : : : +- CometBroadcastHashJoin (20) + : : : : : :- CometProject (15) + : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : :- CometFilter (10) + : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : +- CometBroadcastExchange (13) + : : : : : : +- CometFilter (12) + : : : : : : +- CometScan parquet spark_catalog.default.item (11) + : : : : : +- CometBroadcastExchange (19) + : : : : : +- CometProject (18) + : : : : : +- CometFilter (17) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) + : : : : +- ReusedExchange (27) + : : : +- CometBroadcastExchange (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (37) + : : : : +- CometBroadcastHashJoin (36) + : : : : :- CometFilter (34) + : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) + : : : : +- ReusedExchange (35) + : : : +- ReusedExchange (38) + : : +- CometBroadcastExchange (52) + : : +- CometBroadcastHashJoin (51) + : : :- CometFilter (49) + : : : +- CometScan parquet spark_catalog.default.item (48) + : : +- ReusedExchange (50) + : +- CometBroadcastExchange (58) + : +- CometProject (57) + : +- CometFilter (56) + : +- CometScan parquet spark_catalog.default.date_dim (55) + +- BroadcastExchange (84) + +- * Filter (83) + +- * HashAggregate (82) + +- Exchange (81) + +- * HashAggregate (80) + +- * ColumnarToRow (79) + +- CometProject (78) + +- CometBroadcastHashJoin (77) + :- CometProject (72) + : +- CometBroadcastHashJoin (71) + : :- CometBroadcastHashJoin (69) + : : :- CometFilter (67) + : : : +- CometScan parquet spark_catalog.default.store_sales (66) + : : +- ReusedExchange (68) + : +- ReusedExchange (70) + +- CometBroadcastExchange (76) + +- CometProject (75) + +- CometFilter (74) + +- CometScan parquet spark_catalog.default.date_dim (73) (1) Scan parquet spark_catalog.default.store_sales @@ -240,9 +236,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -386,28 +382,22 @@ Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_pric Aggregate Attributes [3]: [sum#47, isEmpty#48, count#49] Results [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#50, isEmpty#51, count#52] -(63) RowToColumnar +(63) Exchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#50, isEmpty#51, count#52] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(64) CometColumnarExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#50, isEmpty#51, count#52] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(65) ColumnarToRow [codegen id : 4] -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#50, isEmpty#51, count#52] - -(66) HashAggregate [codegen id : 4] +(64) HashAggregate [codegen id : 4] Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#50, isEmpty#51, count#52] Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#53, count(1)#54] Results [6]: [store AS channel#55, i_brand_id#40, i_class_id#41, i_category_id#42, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#53 AS sales#56, count(1)#54 AS number_sales#57] -(67) Filter [codegen id : 4] +(65) Filter [codegen id : 4] Input [6]: [channel#55, i_brand_id#40, i_class_id#41, i_category_id#42, sales#56, number_sales#57] Condition : (isnotnull(sales#56) AND (cast(sales#56 as decimal(32,6)) > cast(Subquery scalar-subquery#58, [id=#59] as decimal(32,6)))) -(68) Scan parquet spark_catalog.default.store_sales +(66) Scan parquet spark_catalog.default.store_sales Output [4]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63] Batched: true Location: InMemoryFileIndex [] @@ -415,306 +405,292 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#63), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(69) CometFilter +(67) CometFilter Input [4]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63] Condition : isnotnull(ss_item_sk#60) -(70) ReusedExchange [Reuses operator id: 46] +(68) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#65] -(71) CometBroadcastHashJoin +(69) CometBroadcastHashJoin Left output [4]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63] Right output [1]: [ss_item_sk#65] Arguments: [ss_item_sk#60], [ss_item_sk#65], LeftSemi, BuildRight -(72) ReusedExchange [Reuses operator id: 52] +(70) ReusedExchange [Reuses operator id: 52] Output [4]: [i_item_sk#66, i_brand_id#67, i_class_id#68, i_category_id#69] -(73) CometBroadcastHashJoin +(71) CometBroadcastHashJoin Left output [4]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63] Right output [4]: [i_item_sk#66, i_brand_id#67, i_class_id#68, i_category_id#69] Arguments: [ss_item_sk#60], [i_item_sk#66], Inner, BuildRight -(74) CometProject +(72) CometProject Input [8]: [ss_item_sk#60, ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_item_sk#66, i_brand_id#67, i_class_id#68, i_category_id#69] Arguments: [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69], [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69] -(75) Scan parquet spark_catalog.default.date_dim +(73) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#70, d_week_seq#71] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#72), IsNotNull(d_date_sk)] ReadSchema: struct -(76) CometFilter +(74) CometFilter Input [2]: [d_date_sk#70, d_week_seq#71] Condition : ((isnotnull(d_week_seq#71) AND (d_week_seq#71 = ReusedSubquery Subquery scalar-subquery#72, [id=#73])) AND isnotnull(d_date_sk#70)) -(77) CometProject +(75) CometProject Input [2]: [d_date_sk#70, d_week_seq#71] Arguments: [d_date_sk#70], [d_date_sk#70] -(78) CometBroadcastExchange +(76) CometBroadcastExchange Input [1]: [d_date_sk#70] Arguments: [d_date_sk#70] -(79) CometBroadcastHashJoin +(77) CometBroadcastHashJoin Left output [6]: [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69] Right output [1]: [d_date_sk#70] Arguments: [ss_sold_date_sk#63], [d_date_sk#70], Inner, BuildRight -(80) CometProject +(78) CometProject Input [7]: [ss_quantity#61, ss_list_price#62, ss_sold_date_sk#63, i_brand_id#67, i_class_id#68, i_category_id#69, d_date_sk#70] Arguments: [ss_quantity#61, ss_list_price#62, i_brand_id#67, i_class_id#68, i_category_id#69], [ss_quantity#61, ss_list_price#62, i_brand_id#67, i_class_id#68, i_category_id#69] -(81) ColumnarToRow [codegen id : 2] +(79) ColumnarToRow [codegen id : 2] Input [5]: [ss_quantity#61, ss_list_price#62, i_brand_id#67, i_class_id#68, i_category_id#69] -(82) HashAggregate [codegen id : 2] +(80) HashAggregate [codegen id : 2] Input [5]: [ss_quantity#61, ss_list_price#62, i_brand_id#67, i_class_id#68, i_category_id#69] Keys [3]: [i_brand_id#67, i_class_id#68, i_category_id#69] Functions [2]: [partial_sum((cast(ss_quantity#61 as decimal(10,0)) * ss_list_price#62)), partial_count(1)] Aggregate Attributes [3]: [sum#74, isEmpty#75, count#76] Results [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#77, isEmpty#78, count#79] -(83) RowToColumnar +(81) Exchange Input [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#77, isEmpty#78, count#79] +Arguments: hashpartitioning(i_brand_id#67, i_class_id#68, i_category_id#69, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(84) CometColumnarExchange -Input [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#77, isEmpty#78, count#79] -Arguments: hashpartitioning(i_brand_id#67, i_class_id#68, i_category_id#69, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(85) ColumnarToRow [codegen id : 3] -Input [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#77, isEmpty#78, count#79] - -(86) HashAggregate [codegen id : 3] +(82) HashAggregate [codegen id : 3] Input [6]: [i_brand_id#67, i_class_id#68, i_category_id#69, sum#77, isEmpty#78, count#79] Keys [3]: [i_brand_id#67, i_class_id#68, i_category_id#69] Functions [2]: [sum((cast(ss_quantity#61 as decimal(10,0)) * ss_list_price#62)), count(1)] Aggregate Attributes [2]: [sum((cast(ss_quantity#61 as decimal(10,0)) * ss_list_price#62))#80, count(1)#81] Results [6]: [store AS channel#82, i_brand_id#67, i_class_id#68, i_category_id#69, sum((cast(ss_quantity#61 as decimal(10,0)) * ss_list_price#62))#80 AS sales#83, count(1)#81 AS number_sales#84] -(87) Filter [codegen id : 3] +(83) Filter [codegen id : 3] Input [6]: [channel#82, i_brand_id#67, i_class_id#68, i_category_id#69, sales#83, number_sales#84] Condition : (isnotnull(sales#83) AND (cast(sales#83 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#58, [id=#59] as decimal(32,6)))) -(88) BroadcastExchange +(84) BroadcastExchange Input [6]: [channel#82, i_brand_id#67, i_class_id#68, i_category_id#69, sales#83, number_sales#84] Arguments: HashedRelationBroadcastMode(List(input[1, int, true], input[2, int, true], input[3, int, true]),false), [plan_id=4] -(89) BroadcastHashJoin [codegen id : 4] +(85) BroadcastHashJoin [codegen id : 4] Left keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Right keys [3]: [i_brand_id#67, i_class_id#68, i_category_id#69] Join type: Inner Join condition: None -(90) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [12]: [channel#55, i_brand_id#40, i_class_id#41, i_category_id#42, sales#56, number_sales#57, channel#82, i_brand_id#67, i_class_id#68, i_category_id#69, sales#83, number_sales#84] Arguments: 100, [i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#55, i_brand_id#40, i_class_id#41, i_category_id#42, sales#56, number_sales#57, channel#82, i_brand_id#67, i_class_id#68, i_category_id#69, sales#83, number_sales#84] ===== Subqueries ===== -Subquery:1 Hosting operator id = 67 Hosting Expression = Subquery scalar-subquery#58, [id=#59] -* HashAggregate (109) -+- * ColumnarToRow (108) - +- CometColumnarExchange (107) - +- RowToColumnar (106) - +- * HashAggregate (105) - +- * ColumnarToRow (104) - +- CometUnion (103) - :- CometProject (94) - : +- CometBroadcastHashJoin (93) - : :- CometScan parquet spark_catalog.default.store_sales (91) - : +- ReusedExchange (92) - :- CometProject (98) - : +- CometBroadcastHashJoin (97) - : :- CometScan parquet spark_catalog.default.catalog_sales (95) - : +- ReusedExchange (96) - +- CometProject (102) - +- CometBroadcastHashJoin (101) - :- CometScan parquet spark_catalog.default.web_sales (99) - +- ReusedExchange (100) - - -(91) Scan parquet spark_catalog.default.store_sales +Subquery:1 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#58, [id=#59] +* HashAggregate (103) ++- Exchange (102) + +- * HashAggregate (101) + +- * ColumnarToRow (100) + +- CometUnion (99) + :- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometScan parquet spark_catalog.default.store_sales (87) + : +- ReusedExchange (88) + :- CometProject (94) + : +- CometBroadcastHashJoin (93) + : :- CometScan parquet spark_catalog.default.catalog_sales (91) + : +- ReusedExchange (92) + +- CometProject (98) + +- CometBroadcastHashJoin (97) + :- CometScan parquet spark_catalog.default.web_sales (95) + +- ReusedExchange (96) + + +(87) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#85, ss_list_price#86, ss_sold_date_sk#87] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#87), dynamicpruningexpression(ss_sold_date_sk#87 IN dynamicpruning#88)] ReadSchema: struct -(92) ReusedExchange [Reuses operator id: 19] +(88) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#89] -(93) CometBroadcastHashJoin +(89) CometBroadcastHashJoin Left output [3]: [ss_quantity#85, ss_list_price#86, ss_sold_date_sk#87] Right output [1]: [d_date_sk#89] Arguments: [ss_sold_date_sk#87], [d_date_sk#89], Inner, BuildRight -(94) CometProject +(90) CometProject Input [4]: [ss_quantity#85, ss_list_price#86, ss_sold_date_sk#87, d_date_sk#89] Arguments: [quantity#90, list_price#91], [ss_quantity#85 AS quantity#90, ss_list_price#86 AS list_price#91] -(95) Scan parquet spark_catalog.default.catalog_sales +(91) Scan parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#92, cs_list_price#93, cs_sold_date_sk#94] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#94), dynamicpruningexpression(cs_sold_date_sk#94 IN dynamicpruning#95)] ReadSchema: struct -(96) ReusedExchange [Reuses operator id: 19] +(92) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#96] -(97) CometBroadcastHashJoin +(93) CometBroadcastHashJoin Left output [3]: [cs_quantity#92, cs_list_price#93, cs_sold_date_sk#94] Right output [1]: [d_date_sk#96] Arguments: [cs_sold_date_sk#94], [d_date_sk#96], Inner, BuildRight -(98) CometProject +(94) CometProject Input [4]: [cs_quantity#92, cs_list_price#93, cs_sold_date_sk#94, d_date_sk#96] Arguments: [quantity#97, list_price#98], [cs_quantity#92 AS quantity#97, cs_list_price#93 AS list_price#98] -(99) Scan parquet spark_catalog.default.web_sales +(95) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#99, ws_list_price#100, ws_sold_date_sk#101] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#101), dynamicpruningexpression(ws_sold_date_sk#101 IN dynamicpruning#102)] ReadSchema: struct -(100) ReusedExchange [Reuses operator id: 19] +(96) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#103] -(101) CometBroadcastHashJoin +(97) CometBroadcastHashJoin Left output [3]: [ws_quantity#99, ws_list_price#100, ws_sold_date_sk#101] Right output [1]: [d_date_sk#103] Arguments: [ws_sold_date_sk#101], [d_date_sk#103], Inner, BuildRight -(102) CometProject +(98) CometProject Input [4]: [ws_quantity#99, ws_list_price#100, ws_sold_date_sk#101, d_date_sk#103] Arguments: [quantity#104, list_price#105], [ws_quantity#99 AS quantity#104, ws_list_price#100 AS list_price#105] -(103) CometUnion +(99) CometUnion Child 0 Input [2]: [quantity#90, list_price#91] Child 1 Input [2]: [quantity#97, list_price#98] Child 2 Input [2]: [quantity#104, list_price#105] -(104) ColumnarToRow [codegen id : 1] +(100) ColumnarToRow [codegen id : 1] Input [2]: [quantity#90, list_price#91] -(105) HashAggregate [codegen id : 1] +(101) HashAggregate [codegen id : 1] Input [2]: [quantity#90, list_price#91] Keys: [] Functions [1]: [partial_avg((cast(quantity#90 as decimal(10,0)) * list_price#91))] Aggregate Attributes [2]: [sum#106, count#107] Results [2]: [sum#108, count#109] -(106) RowToColumnar -Input [2]: [sum#108, count#109] - -(107) CometColumnarExchange -Input [2]: [sum#108, count#109] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(108) ColumnarToRow [codegen id : 2] +(102) Exchange Input [2]: [sum#108, count#109] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] -(109) HashAggregate [codegen id : 2] +(103) HashAggregate [codegen id : 2] Input [2]: [sum#108, count#109] Keys: [] Functions [1]: [avg((cast(quantity#90 as decimal(10,0)) * list_price#91))] Aggregate Attributes [1]: [avg((cast(quantity#90 as decimal(10,0)) * list_price#91))#110] Results [1]: [avg((cast(quantity#90 as decimal(10,0)) * list_price#91))#110 AS average_sales#111] -Subquery:2 Hosting operator id = 91 Hosting Expression = ss_sold_date_sk#87 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 87 Hosting Expression = ss_sold_date_sk#87 IN dynamicpruning#12 -Subquery:3 Hosting operator id = 95 Hosting Expression = cs_sold_date_sk#94 IN dynamicpruning#12 +Subquery:3 Hosting operator id = 91 Hosting Expression = cs_sold_date_sk#94 IN dynamicpruning#12 -Subquery:4 Hosting operator id = 99 Hosting Expression = ws_sold_date_sk#101 IN dynamicpruning#12 +Subquery:4 Hosting operator id = 95 Hosting Expression = ws_sold_date_sk#101 IN dynamicpruning#12 Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (114) -+- * ColumnarToRow (113) - +- CometProject (112) - +- CometFilter (111) - +- CometScan parquet spark_catalog.default.date_dim (110) +BroadcastExchange (108) ++- * ColumnarToRow (107) + +- CometProject (106) + +- CometFilter (105) + +- CometScan parquet spark_catalog.default.date_dim (104) -(110) Scan parquet spark_catalog.default.date_dim +(104) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#43, d_week_seq#44] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#45), IsNotNull(d_date_sk)] ReadSchema: struct -(111) CometFilter +(105) CometFilter Input [2]: [d_date_sk#43, d_week_seq#44] Condition : ((isnotnull(d_week_seq#44) AND (d_week_seq#44 = ReusedSubquery Subquery scalar-subquery#45, [id=#46])) AND isnotnull(d_date_sk#43)) -(112) CometProject +(106) CometProject Input [2]: [d_date_sk#43, d_week_seq#44] Arguments: [d_date_sk#43], [d_date_sk#43] -(113) ColumnarToRow [codegen id : 1] +(107) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#43] -(114) BroadcastExchange +(108) BroadcastExchange Input [1]: [d_date_sk#43] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -Subquery:6 Hosting operator id = 111 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] +Subquery:6 Hosting operator id = 105 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] -Subquery:7 Hosting operator id = 110 Hosting Expression = Subquery scalar-subquery#45, [id=#46] -* ColumnarToRow (118) -+- CometProject (117) - +- CometFilter (116) - +- CometScan parquet spark_catalog.default.date_dim (115) +Subquery:7 Hosting operator id = 104 Hosting Expression = Subquery scalar-subquery#45, [id=#46] +* ColumnarToRow (112) ++- CometProject (111) + +- CometFilter (110) + +- CometScan parquet spark_catalog.default.date_dim (109) -(115) Scan parquet spark_catalog.default.date_dim +(109) Scan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#112, d_year#113, d_moy#114, d_dom#115] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1999), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(116) CometFilter +(110) CometFilter Input [4]: [d_week_seq#112, d_year#113, d_moy#114, d_dom#115] Condition : (((((isnotnull(d_year#113) AND isnotnull(d_moy#114)) AND isnotnull(d_dom#115)) AND (d_year#113 = 1999)) AND (d_moy#114 = 12)) AND (d_dom#115 = 16)) -(117) CometProject +(111) CometProject Input [4]: [d_week_seq#112, d_year#113, d_moy#114, d_dom#115] Arguments: [d_week_seq#112], [d_week_seq#112] -(118) ColumnarToRow [codegen id : 1] +(112) ColumnarToRow [codegen id : 1] Input [1]: [d_week_seq#112] Subquery:8 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (123) -+- * ColumnarToRow (122) - +- CometProject (121) - +- CometFilter (120) - +- CometScan parquet spark_catalog.default.date_dim (119) +BroadcastExchange (117) ++- * ColumnarToRow (116) + +- CometProject (115) + +- CometFilter (114) + +- CometScan parquet spark_catalog.default.date_dim (113) -(119) Scan parquet spark_catalog.default.date_dim +(113) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#116] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(120) CometFilter +(114) CometFilter Input [2]: [d_date_sk#26, d_year#116] Condition : (((isnotnull(d_year#116) AND (d_year#116 >= 1998)) AND (d_year#116 <= 2000)) AND isnotnull(d_date_sk#26)) -(121) CometProject +(115) CometProject Input [2]: [d_date_sk#26, d_year#116] Arguments: [d_date_sk#26], [d_date_sk#26] -(122) ColumnarToRow [codegen id : 1] +(116) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#26] -(123) BroadcastExchange +(117) BroadcastExchange Input [1]: [d_date_sk#26] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=7] @@ -726,67 +702,67 @@ Subquery:11 Hosting operator id = 56 Hosting Expression = ReusedSubquery Subquer Subquery:12 Hosting operator id = 55 Hosting Expression = ReusedSubquery Subquery scalar-subquery#45, [id=#46] -Subquery:13 Hosting operator id = 87 Hosting Expression = ReusedSubquery Subquery scalar-subquery#58, [id=#59] +Subquery:13 Hosting operator id = 83 Hosting Expression = ReusedSubquery Subquery scalar-subquery#58, [id=#59] -Subquery:14 Hosting operator id = 68 Hosting Expression = ss_sold_date_sk#63 IN dynamicpruning#64 -BroadcastExchange (128) -+- * ColumnarToRow (127) - +- CometProject (126) - +- CometFilter (125) - +- CometScan parquet spark_catalog.default.date_dim (124) +Subquery:14 Hosting operator id = 66 Hosting Expression = ss_sold_date_sk#63 IN dynamicpruning#64 +BroadcastExchange (122) ++- * ColumnarToRow (121) + +- CometProject (120) + +- CometFilter (119) + +- CometScan parquet spark_catalog.default.date_dim (118) -(124) Scan parquet spark_catalog.default.date_dim +(118) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#70, d_week_seq#71] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_week_seq), EqualTo(d_week_seq,ScalarSubquery#72), IsNotNull(d_date_sk)] ReadSchema: struct -(125) CometFilter +(119) CometFilter Input [2]: [d_date_sk#70, d_week_seq#71] Condition : ((isnotnull(d_week_seq#71) AND (d_week_seq#71 = ReusedSubquery Subquery scalar-subquery#72, [id=#73])) AND isnotnull(d_date_sk#70)) -(126) CometProject +(120) CometProject Input [2]: [d_date_sk#70, d_week_seq#71] Arguments: [d_date_sk#70], [d_date_sk#70] -(127) ColumnarToRow [codegen id : 1] +(121) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#70] -(128) BroadcastExchange +(122) BroadcastExchange Input [1]: [d_date_sk#70] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:15 Hosting operator id = 125 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] +Subquery:15 Hosting operator id = 119 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] -Subquery:16 Hosting operator id = 124 Hosting Expression = Subquery scalar-subquery#72, [id=#73] -* ColumnarToRow (132) -+- CometProject (131) - +- CometFilter (130) - +- CometScan parquet spark_catalog.default.date_dim (129) +Subquery:16 Hosting operator id = 118 Hosting Expression = Subquery scalar-subquery#72, [id=#73] +* ColumnarToRow (126) ++- CometProject (125) + +- CometFilter (124) + +- CometScan parquet spark_catalog.default.date_dim (123) -(129) Scan parquet spark_catalog.default.date_dim +(123) Scan parquet spark_catalog.default.date_dim Output [4]: [d_week_seq#117, d_year#118, d_moy#119, d_dom#120] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), IsNotNull(d_dom), EqualTo(d_year,1998), EqualTo(d_moy,12), EqualTo(d_dom,16)] ReadSchema: struct -(130) CometFilter +(124) CometFilter Input [4]: [d_week_seq#117, d_year#118, d_moy#119, d_dom#120] Condition : (((((isnotnull(d_year#118) AND isnotnull(d_moy#119)) AND isnotnull(d_dom#120)) AND (d_year#118 = 1998)) AND (d_moy#119 = 12)) AND (d_dom#120 = 16)) -(131) CometProject +(125) CometProject Input [4]: [d_week_seq#117, d_year#118, d_moy#119, d_dom#120] Arguments: [d_week_seq#117], [d_week_seq#117] -(132) ColumnarToRow [codegen id : 1] +(126) ColumnarToRow [codegen id : 1] Input [1]: [d_week_seq#117] -Subquery:17 Hosting operator id = 76 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] +Subquery:17 Hosting operator id = 74 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] -Subquery:18 Hosting operator id = 75 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] +Subquery:18 Hosting operator id = 73 Hosting Expression = ReusedSubquery Subquery scalar-subquery#72, [id=#73] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt index 1210748c5..1fecc0fca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14/simplified.txt @@ -5,169 +5,163 @@ TakeOrderedAndProject [i_brand_id,i_class_id,i_category_id,channel,sales,number_ Subquery #4 WholeStageCodegen (2) HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #14 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [quantity,list_price] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometUnion [quantity,list_price] - CometProject [ss_quantity,ss_list_price] [quantity,list_price] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 - CometProject [cs_quantity,cs_list_price] [quantity,list_price] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 - CometProject [ws_quantity,ws_list_price] [quantity,list_price] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [d_date_sk] #10 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #1 - RowToColumnar + InputAdapter + Exchange #14 WholeStageCodegen (1) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] + HashAggregate [quantity,list_price] [sum,count,sum,count] ColumnarToRow InputAdapter - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - CometBroadcastExchange [ss_item_sk] #3 - CometProject [i_item_sk] [ss_item_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #4 - CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #5 - CometHashAggregate [brand_id,class_id,category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] - CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #3 - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #10 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - ReusedExchange [d_date_sk] #10 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + ReusedExchange [d_date_sk] #10 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + ReusedExchange [d_date_sk] #10 + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + ReusedExchange [d_date_sk] #10 + HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] + InputAdapter + Exchange [i_brand_id,i_class_id,i_category_id] #1 + WholeStageCodegen (1) + HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #2 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + Subquery #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + CometBroadcastExchange [ss_item_sk] #3 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #3 - CometBroadcastExchange [d_date_sk] #13 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #2 + CometBroadcastExchange [brand_id,class_id,category_id] #4 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #5 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #3 + BroadcastExchange #6 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #7 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #8 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [d_date_sk] #10 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #11 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + ReusedExchange [d_date_sk] #10 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #3 + CometBroadcastExchange [d_date_sk] #13 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #2 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #2 InputAdapter BroadcastExchange #15 WholeStageCodegen (3) Filter [sales] ReusedSubquery [average_sales] #4 HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #16 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #5 - BroadcastExchange #17 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - Subquery #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_week_seq] - CometFilter [d_week_seq,d_year,d_moy,d_dom] - CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] - ReusedExchange [ss_item_sk] #3 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 - CometBroadcastExchange [d_date_sk] #18 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] - ReusedSubquery [d_week_seq] #6 + InputAdapter + Exchange [i_brand_id,i_class_id,i_category_id] #16 + WholeStageCodegen (2) + HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #5 + BroadcastExchange #17 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #6 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + Subquery #6 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_week_seq] + CometFilter [d_week_seq,d_year,d_moy,d_dom] + CometScan parquet spark_catalog.default.date_dim [d_week_seq,d_year,d_moy,d_dom] + ReusedExchange [ss_item_sk] #3 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #12 + CometBroadcastExchange [d_date_sk] #18 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #6 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + ReusedSubquery [d_week_seq] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt index 44ac98d43..cd6293c6f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/explain.txt @@ -1,150 +1,128 @@ == Physical Plan == -* ColumnarToRow (146) -+- CometTakeOrderedAndProject (145) - +- CometHashAggregate (144) - +- CometColumnarExchange (143) - +- RowToColumnar (142) - +- * HashAggregate (141) - +- Union (140) - :- * HashAggregate (107) - : +- * ColumnarToRow (106) - : +- CometColumnarExchange (105) - : +- RowToColumnar (104) - : +- * HashAggregate (103) - : +- Union (102) - : :- * Filter (67) - : : +- * HashAggregate (66) - : : +- * ColumnarToRow (65) - : : +- CometColumnarExchange (64) - : : +- RowToColumnar (63) - : : +- * HashAggregate (62) - : : +- * ColumnarToRow (61) - : : +- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (54) - : : : +- CometBroadcastHashJoin (53) - : : : :- CometBroadcastHashJoin (47) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometBroadcastExchange (46) - : : : : +- CometProject (45) - : : : : +- CometBroadcastHashJoin (44) - : : : : :- CometFilter (4) - : : : : : +- CometScan parquet spark_catalog.default.item (3) - : : : : +- CometBroadcastExchange (43) - : : : : +- CometBroadcastHashJoin (42) - : : : : :- CometHashAggregate (32) - : : : : : +- CometColumnarExchange (31) - : : : : : +- CometHashAggregate (30) - : : : : : +- CometProject (29) - : : : : : +- CometBroadcastHashJoin (28) - : : : : : :- CometProject (26) - : : : : : : +- CometBroadcastHashJoin (25) - : : : : : : :- CometFilter (6) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) - : : : : : : +- CometBroadcastExchange (24) - : : : : : : +- CometBroadcastHashJoin (23) - : : : : : : :- CometFilter (8) - : : : : : : : +- CometScan parquet spark_catalog.default.item (7) - : : : : : : +- CometBroadcastExchange (22) - : : : : : : +- CometProject (21) - : : : : : : +- CometBroadcastHashJoin (20) - : : : : : : :- CometProject (15) - : : : : : : : +- CometBroadcastHashJoin (14) - : : : : : : : :- CometFilter (10) - : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) - : : : : : : : +- CometBroadcastExchange (13) - : : : : : : : +- CometFilter (12) - : : : : : : : +- CometScan parquet spark_catalog.default.item (11) - : : : : : : +- CometBroadcastExchange (19) - : : : : : : +- CometProject (18) - : : : : : : +- CometFilter (17) - : : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) - : : : : : +- ReusedExchange (27) - : : : : +- CometBroadcastExchange (41) - : : : : +- CometProject (40) - : : : : +- CometBroadcastHashJoin (39) - : : : : :- CometProject (37) - : : : : : +- CometBroadcastHashJoin (36) - : : : : : :- CometFilter (34) - : : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) - : : : : : +- ReusedExchange (35) - : : : : +- ReusedExchange (38) - : : : +- CometBroadcastExchange (52) - : : : +- CometBroadcastHashJoin (51) - : : : :- CometFilter (49) - : : : : +- CometScan parquet spark_catalog.default.item (48) - : : : +- ReusedExchange (50) - : : +- CometBroadcastExchange (58) - : : +- CometProject (57) - : : +- CometFilter (56) - : : +- CometScan parquet spark_catalog.default.date_dim (55) - : :- * Filter (84) - : : +- * HashAggregate (83) - : : +- * ColumnarToRow (82) - : : +- CometColumnarExchange (81) - : : +- RowToColumnar (80) - : : +- * HashAggregate (79) - : : +- * ColumnarToRow (78) - : : +- CometProject (77) - : : +- CometBroadcastHashJoin (76) - : : :- CometProject (74) - : : : +- CometBroadcastHashJoin (73) - : : : :- CometBroadcastHashJoin (71) - : : : : :- CometFilter (69) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (68) - : : : : +- ReusedExchange (70) - : : : +- ReusedExchange (72) - : : +- ReusedExchange (75) - : +- * Filter (101) - : +- * HashAggregate (100) - : +- * ColumnarToRow (99) - : +- CometColumnarExchange (98) - : +- RowToColumnar (97) - : +- * HashAggregate (96) - : +- * ColumnarToRow (95) - : +- CometProject (94) - : +- CometBroadcastHashJoin (93) - : :- CometProject (91) - : : +- CometBroadcastHashJoin (90) - : : :- CometBroadcastHashJoin (88) - : : : :- CometFilter (86) - : : : : +- CometScan parquet spark_catalog.default.web_sales (85) - : : : +- ReusedExchange (87) - : : +- ReusedExchange (89) - : +- ReusedExchange (92) - :- * HashAggregate (115) - : +- * ColumnarToRow (114) - : +- CometColumnarExchange (113) - : +- RowToColumnar (112) - : +- * HashAggregate (111) - : +- * HashAggregate (110) - : +- * ColumnarToRow (109) - : +- ReusedExchange (108) - :- * HashAggregate (123) - : +- * ColumnarToRow (122) - : +- CometColumnarExchange (121) - : +- RowToColumnar (120) - : +- * HashAggregate (119) - : +- * HashAggregate (118) - : +- * ColumnarToRow (117) - : +- ReusedExchange (116) - :- * HashAggregate (131) - : +- * ColumnarToRow (130) - : +- CometColumnarExchange (129) - : +- RowToColumnar (128) - : +- * HashAggregate (127) - : +- * HashAggregate (126) - : +- * ColumnarToRow (125) - : +- ReusedExchange (124) - +- * HashAggregate (139) - +- * ColumnarToRow (138) - +- CometColumnarExchange (137) - +- RowToColumnar (136) - +- * HashAggregate (135) - +- * HashAggregate (134) - +- * ColumnarToRow (133) - +- ReusedExchange (132) +TakeOrderedAndProject (124) ++- * HashAggregate (123) + +- Exchange (122) + +- * HashAggregate (121) + +- Union (120) + :- * HashAggregate (99) + : +- Exchange (98) + : +- * HashAggregate (97) + : +- Union (96) + : :- * Filter (65) + : : +- * HashAggregate (64) + : : +- Exchange (63) + : : +- * HashAggregate (62) + : : +- * ColumnarToRow (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometBroadcastHashJoin (47) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (46) + : : : : +- CometProject (45) + : : : : +- CometBroadcastHashJoin (44) + : : : : :- CometFilter (4) + : : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : : +- CometBroadcastExchange (43) + : : : : +- CometBroadcastHashJoin (42) + : : : : :- CometHashAggregate (32) + : : : : : +- CometExchange (31) + : : : : : +- CometHashAggregate (30) + : : : : : +- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (26) + : : : : : : +- CometBroadcastHashJoin (25) + : : : : : : :- CometFilter (6) + : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (5) + : : : : : : +- CometBroadcastExchange (24) + : : : : : : +- CometBroadcastHashJoin (23) + : : : : : : :- CometFilter (8) + : : : : : : : +- CometScan parquet spark_catalog.default.item (7) + : : : : : : +- CometBroadcastExchange (22) + : : : : : : +- CometProject (21) + : : : : : : +- CometBroadcastHashJoin (20) + : : : : : : :- CometProject (15) + : : : : : : : +- CometBroadcastHashJoin (14) + : : : : : : : :- CometFilter (10) + : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (9) + : : : : : : : +- CometBroadcastExchange (13) + : : : : : : : +- CometFilter (12) + : : : : : : : +- CometScan parquet spark_catalog.default.item (11) + : : : : : : +- CometBroadcastExchange (19) + : : : : : : +- CometProject (18) + : : : : : : +- CometFilter (17) + : : : : : : +- CometScan parquet spark_catalog.default.date_dim (16) + : : : : : +- ReusedExchange (27) + : : : : +- CometBroadcastExchange (41) + : : : : +- CometProject (40) + : : : : +- CometBroadcastHashJoin (39) + : : : : :- CometProject (37) + : : : : : +- CometBroadcastHashJoin (36) + : : : : : :- CometFilter (34) + : : : : : : +- CometScan parquet spark_catalog.default.web_sales (33) + : : : : : +- ReusedExchange (35) + : : : : +- ReusedExchange (38) + : : : +- CometBroadcastExchange (52) + : : : +- CometBroadcastHashJoin (51) + : : : :- CometFilter (49) + : : : : +- CometScan parquet spark_catalog.default.item (48) + : : : +- ReusedExchange (50) + : : +- CometBroadcastExchange (58) + : : +- CometProject (57) + : : +- CometFilter (56) + : : +- CometScan parquet spark_catalog.default.date_dim (55) + : :- * Filter (80) + : : +- * HashAggregate (79) + : : +- Exchange (78) + : : +- * HashAggregate (77) + : : +- * ColumnarToRow (76) + : : +- CometProject (75) + : : +- CometBroadcastHashJoin (74) + : : :- CometProject (72) + : : : +- CometBroadcastHashJoin (71) + : : : :- CometBroadcastHashJoin (69) + : : : : :- CometFilter (67) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (66) + : : : : +- ReusedExchange (68) + : : : +- ReusedExchange (70) + : : +- ReusedExchange (73) + : +- * Filter (95) + : +- * HashAggregate (94) + : +- Exchange (93) + : +- * HashAggregate (92) + : +- * ColumnarToRow (91) + : +- CometProject (90) + : +- CometBroadcastHashJoin (89) + : :- CometProject (87) + : : +- CometBroadcastHashJoin (86) + : : :- CometBroadcastHashJoin (84) + : : : :- CometFilter (82) + : : : : +- CometScan parquet spark_catalog.default.web_sales (81) + : : : +- ReusedExchange (83) + : : +- ReusedExchange (85) + : +- ReusedExchange (88) + :- * HashAggregate (104) + : +- Exchange (103) + : +- * HashAggregate (102) + : +- * HashAggregate (101) + : +- ReusedExchange (100) + :- * HashAggregate (109) + : +- Exchange (108) + : +- * HashAggregate (107) + : +- * HashAggregate (106) + : +- ReusedExchange (105) + :- * HashAggregate (114) + : +- Exchange (113) + : +- * HashAggregate (112) + : +- * HashAggregate (111) + : +- ReusedExchange (110) + +- * HashAggregate (119) + +- Exchange (118) + +- * HashAggregate (117) + +- * HashAggregate (116) + +- ReusedExchange (115) (1) Scan parquet spark_catalog.default.store_sales @@ -296,9 +274,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -442,28 +420,22 @@ Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_pric Aggregate Attributes [3]: [sum#46, isEmpty#47, count#48] Results [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#49, isEmpty#50, count#51] -(63) RowToColumnar +(63) Exchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#49, isEmpty#50, count#51] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(64) CometColumnarExchange -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#49, isEmpty#50, count#51] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(65) ColumnarToRow [codegen id : 2] -Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#49, isEmpty#50, count#51] - -(66) HashAggregate [codegen id : 2] +(64) HashAggregate [codegen id : 2] Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#49, isEmpty#50, count#51] Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), count(1)] Aggregate Attributes [2]: [sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#52, count(1)#53] Results [6]: [store AS channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3))#52 AS sales#55, count(1)#53 AS number_sales#56] -(67) Filter [codegen id : 2] +(65) Filter [codegen id : 2] Input [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sales#55, number_sales#56] Condition : (isnotnull(sales#55) AND (cast(sales#55 as decimal(32,6)) > cast(Subquery scalar-subquery#57, [id=#58] as decimal(32,6)))) -(68) Scan parquet spark_catalog.default.catalog_sales +(66) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_item_sk#59, cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62] Batched: true Location: InMemoryFileIndex [] @@ -471,74 +443,68 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#62), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(69) CometFilter +(67) CometFilter Input [4]: [cs_item_sk#59, cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62] Condition : isnotnull(cs_item_sk#59) -(70) ReusedExchange [Reuses operator id: 46] +(68) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#64] -(71) CometBroadcastHashJoin +(69) CometBroadcastHashJoin Left output [4]: [cs_item_sk#59, cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62] Right output [1]: [ss_item_sk#64] Arguments: [cs_item_sk#59], [ss_item_sk#64], LeftSemi, BuildRight -(72) ReusedExchange [Reuses operator id: 52] +(70) ReusedExchange [Reuses operator id: 52] Output [4]: [i_item_sk#65, i_brand_id#66, i_class_id#67, i_category_id#68] -(73) CometBroadcastHashJoin +(71) CometBroadcastHashJoin Left output [4]: [cs_item_sk#59, cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62] Right output [4]: [i_item_sk#65, i_brand_id#66, i_class_id#67, i_category_id#68] Arguments: [cs_item_sk#59], [i_item_sk#65], Inner, BuildRight -(74) CometProject +(72) CometProject Input [8]: [cs_item_sk#59, cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62, i_item_sk#65, i_brand_id#66, i_class_id#67, i_category_id#68] Arguments: [cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62, i_brand_id#66, i_class_id#67, i_category_id#68], [cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62, i_brand_id#66, i_class_id#67, i_category_id#68] -(75) ReusedExchange [Reuses operator id: 58] +(73) ReusedExchange [Reuses operator id: 58] Output [1]: [d_date_sk#69] -(76) CometBroadcastHashJoin +(74) CometBroadcastHashJoin Left output [6]: [cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62, i_brand_id#66, i_class_id#67, i_category_id#68] Right output [1]: [d_date_sk#69] Arguments: [cs_sold_date_sk#62], [d_date_sk#69], Inner, BuildRight -(77) CometProject +(75) CometProject Input [7]: [cs_quantity#60, cs_list_price#61, cs_sold_date_sk#62, i_brand_id#66, i_class_id#67, i_category_id#68, d_date_sk#69] Arguments: [cs_quantity#60, cs_list_price#61, i_brand_id#66, i_class_id#67, i_category_id#68], [cs_quantity#60, cs_list_price#61, i_brand_id#66, i_class_id#67, i_category_id#68] -(78) ColumnarToRow [codegen id : 3] +(76) ColumnarToRow [codegen id : 3] Input [5]: [cs_quantity#60, cs_list_price#61, i_brand_id#66, i_class_id#67, i_category_id#68] -(79) HashAggregate [codegen id : 3] +(77) HashAggregate [codegen id : 3] Input [5]: [cs_quantity#60, cs_list_price#61, i_brand_id#66, i_class_id#67, i_category_id#68] Keys [3]: [i_brand_id#66, i_class_id#67, i_category_id#68] Functions [2]: [partial_sum((cast(cs_quantity#60 as decimal(10,0)) * cs_list_price#61)), partial_count(1)] Aggregate Attributes [3]: [sum#70, isEmpty#71, count#72] Results [6]: [i_brand_id#66, i_class_id#67, i_category_id#68, sum#73, isEmpty#74, count#75] -(80) RowToColumnar -Input [6]: [i_brand_id#66, i_class_id#67, i_category_id#68, sum#73, isEmpty#74, count#75] - -(81) CometColumnarExchange -Input [6]: [i_brand_id#66, i_class_id#67, i_category_id#68, sum#73, isEmpty#74, count#75] -Arguments: hashpartitioning(i_brand_id#66, i_class_id#67, i_category_id#68, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(82) ColumnarToRow [codegen id : 4] +(78) Exchange Input [6]: [i_brand_id#66, i_class_id#67, i_category_id#68, sum#73, isEmpty#74, count#75] +Arguments: hashpartitioning(i_brand_id#66, i_class_id#67, i_category_id#68, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(83) HashAggregate [codegen id : 4] +(79) HashAggregate [codegen id : 4] Input [6]: [i_brand_id#66, i_class_id#67, i_category_id#68, sum#73, isEmpty#74, count#75] Keys [3]: [i_brand_id#66, i_class_id#67, i_category_id#68] Functions [2]: [sum((cast(cs_quantity#60 as decimal(10,0)) * cs_list_price#61)), count(1)] Aggregate Attributes [2]: [sum((cast(cs_quantity#60 as decimal(10,0)) * cs_list_price#61))#76, count(1)#77] Results [6]: [catalog AS channel#78, i_brand_id#66, i_class_id#67, i_category_id#68, sum((cast(cs_quantity#60 as decimal(10,0)) * cs_list_price#61))#76 AS sales#79, count(1)#77 AS number_sales#80] -(84) Filter [codegen id : 4] +(80) Filter [codegen id : 4] Input [6]: [channel#78, i_brand_id#66, i_class_id#67, i_category_id#68, sales#79, number_sales#80] Condition : (isnotnull(sales#79) AND (cast(sales#79 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#57, [id=#58] as decimal(32,6)))) -(85) Scan parquet spark_catalog.default.web_sales +(81) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#81, ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Batched: true Location: InMemoryFileIndex [] @@ -546,498 +512,438 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#84), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(86) CometFilter +(82) CometFilter Input [4]: [ws_item_sk#81, ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Condition : isnotnull(ws_item_sk#81) -(87) ReusedExchange [Reuses operator id: 46] +(83) ReusedExchange [Reuses operator id: 46] Output [1]: [ss_item_sk#86] -(88) CometBroadcastHashJoin +(84) CometBroadcastHashJoin Left output [4]: [ws_item_sk#81, ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Right output [1]: [ss_item_sk#86] Arguments: [ws_item_sk#81], [ss_item_sk#86], LeftSemi, BuildRight -(89) ReusedExchange [Reuses operator id: 52] +(85) ReusedExchange [Reuses operator id: 52] Output [4]: [i_item_sk#87, i_brand_id#88, i_class_id#89, i_category_id#90] -(90) CometBroadcastHashJoin +(86) CometBroadcastHashJoin Left output [4]: [ws_item_sk#81, ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84] Right output [4]: [i_item_sk#87, i_brand_id#88, i_class_id#89, i_category_id#90] Arguments: [ws_item_sk#81], [i_item_sk#87], Inner, BuildRight -(91) CometProject +(87) CometProject Input [8]: [ws_item_sk#81, ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, i_item_sk#87, i_brand_id#88, i_class_id#89, i_category_id#90] Arguments: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, i_brand_id#88, i_class_id#89, i_category_id#90], [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, i_brand_id#88, i_class_id#89, i_category_id#90] -(92) ReusedExchange [Reuses operator id: 58] +(88) ReusedExchange [Reuses operator id: 58] Output [1]: [d_date_sk#91] -(93) CometBroadcastHashJoin +(89) CometBroadcastHashJoin Left output [6]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, i_brand_id#88, i_class_id#89, i_category_id#90] Right output [1]: [d_date_sk#91] Arguments: [ws_sold_date_sk#84], [d_date_sk#91], Inner, BuildRight -(94) CometProject +(90) CometProject Input [7]: [ws_quantity#82, ws_list_price#83, ws_sold_date_sk#84, i_brand_id#88, i_class_id#89, i_category_id#90, d_date_sk#91] Arguments: [ws_quantity#82, ws_list_price#83, i_brand_id#88, i_class_id#89, i_category_id#90], [ws_quantity#82, ws_list_price#83, i_brand_id#88, i_class_id#89, i_category_id#90] -(95) ColumnarToRow [codegen id : 5] +(91) ColumnarToRow [codegen id : 5] Input [5]: [ws_quantity#82, ws_list_price#83, i_brand_id#88, i_class_id#89, i_category_id#90] -(96) HashAggregate [codegen id : 5] +(92) HashAggregate [codegen id : 5] Input [5]: [ws_quantity#82, ws_list_price#83, i_brand_id#88, i_class_id#89, i_category_id#90] Keys [3]: [i_brand_id#88, i_class_id#89, i_category_id#90] Functions [2]: [partial_sum((cast(ws_quantity#82 as decimal(10,0)) * ws_list_price#83)), partial_count(1)] Aggregate Attributes [3]: [sum#92, isEmpty#93, count#94] Results [6]: [i_brand_id#88, i_class_id#89, i_category_id#90, sum#95, isEmpty#96, count#97] -(97) RowToColumnar -Input [6]: [i_brand_id#88, i_class_id#89, i_category_id#90, sum#95, isEmpty#96, count#97] - -(98) CometColumnarExchange -Input [6]: [i_brand_id#88, i_class_id#89, i_category_id#90, sum#95, isEmpty#96, count#97] -Arguments: hashpartitioning(i_brand_id#88, i_class_id#89, i_category_id#90, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(99) ColumnarToRow [codegen id : 6] +(93) Exchange Input [6]: [i_brand_id#88, i_class_id#89, i_category_id#90, sum#95, isEmpty#96, count#97] +Arguments: hashpartitioning(i_brand_id#88, i_class_id#89, i_category_id#90, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(100) HashAggregate [codegen id : 6] +(94) HashAggregate [codegen id : 6] Input [6]: [i_brand_id#88, i_class_id#89, i_category_id#90, sum#95, isEmpty#96, count#97] Keys [3]: [i_brand_id#88, i_class_id#89, i_category_id#90] Functions [2]: [sum((cast(ws_quantity#82 as decimal(10,0)) * ws_list_price#83)), count(1)] Aggregate Attributes [2]: [sum((cast(ws_quantity#82 as decimal(10,0)) * ws_list_price#83))#98, count(1)#99] Results [6]: [web AS channel#100, i_brand_id#88, i_class_id#89, i_category_id#90, sum((cast(ws_quantity#82 as decimal(10,0)) * ws_list_price#83))#98 AS sales#101, count(1)#99 AS number_sales#102] -(101) Filter [codegen id : 6] +(95) Filter [codegen id : 6] Input [6]: [channel#100, i_brand_id#88, i_class_id#89, i_category_id#90, sales#101, number_sales#102] Condition : (isnotnull(sales#101) AND (cast(sales#101 as decimal(32,6)) > cast(ReusedSubquery Subquery scalar-subquery#57, [id=#58] as decimal(32,6)))) -(102) Union +(96) Union -(103) HashAggregate [codegen id : 7] +(97) HashAggregate [codegen id : 7] Input [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sales#55, number_sales#56] Keys [4]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum(sales#55), partial_sum(number_sales#56)] Aggregate Attributes [3]: [sum#103, isEmpty#104, sum#105] Results [7]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum#106, isEmpty#107, sum#108] -(104) RowToColumnar +(98) Exchange Input [7]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum#106, isEmpty#107, sum#108] +Arguments: hashpartitioning(channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(105) CometColumnarExchange -Input [7]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum#106, isEmpty#107, sum#108] -Arguments: hashpartitioning(channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(106) ColumnarToRow [codegen id : 8] -Input [7]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum#106, isEmpty#107, sum#108] - -(107) HashAggregate [codegen id : 8] +(99) HashAggregate [codegen id : 8] Input [7]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum#106, isEmpty#107, sum#108] Keys [4]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [sum(sales#55), sum(number_sales#56)] Aggregate Attributes [2]: [sum(sales#55)#109, sum(number_sales#56)#110] Results [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum(sales#55)#109 AS sum_sales#111, sum(number_sales#56)#110 AS number_sales#112] -(108) ReusedExchange [Reuses operator id: 105] +(100) ReusedExchange [Reuses operator id: 98] Output [7]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum#117, isEmpty#118, sum#119] -(109) ColumnarToRow [codegen id : 16] -Input [7]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum#117, isEmpty#118, sum#119] - -(110) HashAggregate [codegen id : 16] +(101) HashAggregate [codegen id : 16] Input [7]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116, sum#117, isEmpty#118, sum#119] Keys [4]: [channel#113, i_brand_id#114, i_class_id#115, i_category_id#116] Functions [2]: [sum(sales#120), sum(number_sales#121)] Aggregate Attributes [2]: [sum(sales#120)#109, sum(number_sales#121)#110] Results [5]: [channel#113, i_brand_id#114, i_class_id#115, sum(sales#120)#109 AS sum_sales#122, sum(number_sales#121)#110 AS number_sales#123] -(111) HashAggregate [codegen id : 16] +(102) HashAggregate [codegen id : 16] Input [5]: [channel#113, i_brand_id#114, i_class_id#115, sum_sales#122, number_sales#123] Keys [3]: [channel#113, i_brand_id#114, i_class_id#115] Functions [2]: [partial_sum(sum_sales#122), partial_sum(number_sales#123)] Aggregate Attributes [3]: [sum#124, isEmpty#125, sum#126] Results [6]: [channel#113, i_brand_id#114, i_class_id#115, sum#127, isEmpty#128, sum#129] -(112) RowToColumnar +(103) Exchange Input [6]: [channel#113, i_brand_id#114, i_class_id#115, sum#127, isEmpty#128, sum#129] +Arguments: hashpartitioning(channel#113, i_brand_id#114, i_class_id#115, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(113) CometColumnarExchange -Input [6]: [channel#113, i_brand_id#114, i_class_id#115, sum#127, isEmpty#128, sum#129] -Arguments: hashpartitioning(channel#113, i_brand_id#114, i_class_id#115, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(114) ColumnarToRow [codegen id : 17] -Input [6]: [channel#113, i_brand_id#114, i_class_id#115, sum#127, isEmpty#128, sum#129] - -(115) HashAggregate [codegen id : 17] +(104) HashAggregate [codegen id : 17] Input [6]: [channel#113, i_brand_id#114, i_class_id#115, sum#127, isEmpty#128, sum#129] Keys [3]: [channel#113, i_brand_id#114, i_class_id#115] Functions [2]: [sum(sum_sales#122), sum(number_sales#123)] Aggregate Attributes [2]: [sum(sum_sales#122)#130, sum(number_sales#123)#131] Results [6]: [channel#113, i_brand_id#114, i_class_id#115, null AS i_category_id#132, sum(sum_sales#122)#130 AS sum(sum_sales)#133, sum(number_sales#123)#131 AS sum(number_sales)#134] -(116) ReusedExchange [Reuses operator id: 105] +(105) ReusedExchange [Reuses operator id: 98] Output [7]: [channel#135, i_brand_id#136, i_class_id#137, i_category_id#138, sum#139, isEmpty#140, sum#141] -(117) ColumnarToRow [codegen id : 25] -Input [7]: [channel#135, i_brand_id#136, i_class_id#137, i_category_id#138, sum#139, isEmpty#140, sum#141] - -(118) HashAggregate [codegen id : 25] +(106) HashAggregate [codegen id : 25] Input [7]: [channel#135, i_brand_id#136, i_class_id#137, i_category_id#138, sum#139, isEmpty#140, sum#141] Keys [4]: [channel#135, i_brand_id#136, i_class_id#137, i_category_id#138] Functions [2]: [sum(sales#142), sum(number_sales#143)] Aggregate Attributes [2]: [sum(sales#142)#109, sum(number_sales#143)#110] Results [4]: [channel#135, i_brand_id#136, sum(sales#142)#109 AS sum_sales#144, sum(number_sales#143)#110 AS number_sales#145] -(119) HashAggregate [codegen id : 25] +(107) HashAggregate [codegen id : 25] Input [4]: [channel#135, i_brand_id#136, sum_sales#144, number_sales#145] Keys [2]: [channel#135, i_brand_id#136] Functions [2]: [partial_sum(sum_sales#144), partial_sum(number_sales#145)] Aggregate Attributes [3]: [sum#146, isEmpty#147, sum#148] Results [5]: [channel#135, i_brand_id#136, sum#149, isEmpty#150, sum#151] -(120) RowToColumnar -Input [5]: [channel#135, i_brand_id#136, sum#149, isEmpty#150, sum#151] - -(121) CometColumnarExchange -Input [5]: [channel#135, i_brand_id#136, sum#149, isEmpty#150, sum#151] -Arguments: hashpartitioning(channel#135, i_brand_id#136, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(122) ColumnarToRow [codegen id : 26] +(108) Exchange Input [5]: [channel#135, i_brand_id#136, sum#149, isEmpty#150, sum#151] +Arguments: hashpartitioning(channel#135, i_brand_id#136, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(123) HashAggregate [codegen id : 26] +(109) HashAggregate [codegen id : 26] Input [5]: [channel#135, i_brand_id#136, sum#149, isEmpty#150, sum#151] Keys [2]: [channel#135, i_brand_id#136] Functions [2]: [sum(sum_sales#144), sum(number_sales#145)] Aggregate Attributes [2]: [sum(sum_sales#144)#152, sum(number_sales#145)#153] Results [6]: [channel#135, i_brand_id#136, null AS i_class_id#154, null AS i_category_id#155, sum(sum_sales#144)#152 AS sum(sum_sales)#156, sum(number_sales#145)#153 AS sum(number_sales)#157] -(124) ReusedExchange [Reuses operator id: 105] +(110) ReusedExchange [Reuses operator id: 98] Output [7]: [channel#158, i_brand_id#159, i_class_id#160, i_category_id#161, sum#162, isEmpty#163, sum#164] -(125) ColumnarToRow [codegen id : 34] -Input [7]: [channel#158, i_brand_id#159, i_class_id#160, i_category_id#161, sum#162, isEmpty#163, sum#164] - -(126) HashAggregate [codegen id : 34] +(111) HashAggregate [codegen id : 34] Input [7]: [channel#158, i_brand_id#159, i_class_id#160, i_category_id#161, sum#162, isEmpty#163, sum#164] Keys [4]: [channel#158, i_brand_id#159, i_class_id#160, i_category_id#161] Functions [2]: [sum(sales#165), sum(number_sales#166)] Aggregate Attributes [2]: [sum(sales#165)#109, sum(number_sales#166)#110] Results [3]: [channel#158, sum(sales#165)#109 AS sum_sales#167, sum(number_sales#166)#110 AS number_sales#168] -(127) HashAggregate [codegen id : 34] +(112) HashAggregate [codegen id : 34] Input [3]: [channel#158, sum_sales#167, number_sales#168] Keys [1]: [channel#158] Functions [2]: [partial_sum(sum_sales#167), partial_sum(number_sales#168)] Aggregate Attributes [3]: [sum#169, isEmpty#170, sum#171] Results [4]: [channel#158, sum#172, isEmpty#173, sum#174] -(128) RowToColumnar -Input [4]: [channel#158, sum#172, isEmpty#173, sum#174] - -(129) CometColumnarExchange +(113) Exchange Input [4]: [channel#158, sum#172, isEmpty#173, sum#174] -Arguments: hashpartitioning(channel#158, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(channel#158, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(130) ColumnarToRow [codegen id : 35] -Input [4]: [channel#158, sum#172, isEmpty#173, sum#174] - -(131) HashAggregate [codegen id : 35] +(114) HashAggregate [codegen id : 35] Input [4]: [channel#158, sum#172, isEmpty#173, sum#174] Keys [1]: [channel#158] Functions [2]: [sum(sum_sales#167), sum(number_sales#168)] Aggregate Attributes [2]: [sum(sum_sales#167)#175, sum(number_sales#168)#176] Results [6]: [channel#158, null AS i_brand_id#177, null AS i_class_id#178, null AS i_category_id#179, sum(sum_sales#167)#175 AS sum(sum_sales)#180, sum(number_sales#168)#176 AS sum(number_sales)#181] -(132) ReusedExchange [Reuses operator id: 105] +(115) ReusedExchange [Reuses operator id: 98] Output [7]: [channel#182, i_brand_id#183, i_class_id#184, i_category_id#185, sum#186, isEmpty#187, sum#188] -(133) ColumnarToRow [codegen id : 43] -Input [7]: [channel#182, i_brand_id#183, i_class_id#184, i_category_id#185, sum#186, isEmpty#187, sum#188] - -(134) HashAggregate [codegen id : 43] +(116) HashAggregate [codegen id : 43] Input [7]: [channel#182, i_brand_id#183, i_class_id#184, i_category_id#185, sum#186, isEmpty#187, sum#188] Keys [4]: [channel#182, i_brand_id#183, i_class_id#184, i_category_id#185] Functions [2]: [sum(sales#189), sum(number_sales#190)] Aggregate Attributes [2]: [sum(sales#189)#109, sum(number_sales#190)#110] Results [2]: [sum(sales#189)#109 AS sum_sales#191, sum(number_sales#190)#110 AS number_sales#192] -(135) HashAggregate [codegen id : 43] +(117) HashAggregate [codegen id : 43] Input [2]: [sum_sales#191, number_sales#192] Keys: [] Functions [2]: [partial_sum(sum_sales#191), partial_sum(number_sales#192)] Aggregate Attributes [3]: [sum#193, isEmpty#194, sum#195] Results [3]: [sum#196, isEmpty#197, sum#198] -(136) RowToColumnar +(118) Exchange Input [3]: [sum#196, isEmpty#197, sum#198] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] -(137) CometColumnarExchange -Input [3]: [sum#196, isEmpty#197, sum#198] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(138) ColumnarToRow [codegen id : 44] -Input [3]: [sum#196, isEmpty#197, sum#198] - -(139) HashAggregate [codegen id : 44] +(119) HashAggregate [codegen id : 44] Input [3]: [sum#196, isEmpty#197, sum#198] Keys: [] Functions [2]: [sum(sum_sales#191), sum(number_sales#192)] Aggregate Attributes [2]: [sum(sum_sales#191)#199, sum(number_sales#192)#200] Results [6]: [null AS channel#201, null AS i_brand_id#202, null AS i_class_id#203, null AS i_category_id#204, sum(sum_sales#191)#199 AS sum(sum_sales)#205, sum(number_sales#192)#200 AS sum(number_sales)#206] -(140) Union +(120) Union -(141) HashAggregate [codegen id : 45] +(121) HashAggregate [codegen id : 45] Input [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] Keys [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] Functions: [] Aggregate Attributes: [] Results [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] -(142) RowToColumnar -Input [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] - -(143) CometColumnarExchange +(122) Exchange Input [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] -Arguments: hashpartitioning(channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(144) CometHashAggregate +(123) HashAggregate [codegen id : 46] Input [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] Keys [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] Functions: [] +Aggregate Attributes: [] +Results [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] -(145) CometTakeOrderedAndProject -Input [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#54 ASC NULLS FIRST,i_brand_id#40 ASC NULLS FIRST,i_class_id#41 ASC NULLS FIRST,i_category_id#42 ASC NULLS FIRST], output=[channel#54,i_brand_id#40,i_class_id#41,i_category_id#42,sum_sales#111,number_sales#112]), [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112], 100, [channel#54 ASC NULLS FIRST, i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] - -(146) ColumnarToRow [codegen id : 46] +(124) TakeOrderedAndProject Input [6]: [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] +Arguments: 100, [channel#54 ASC NULLS FIRST, i_brand_id#40 ASC NULLS FIRST, i_class_id#41 ASC NULLS FIRST, i_category_id#42 ASC NULLS FIRST], [channel#54, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#111, number_sales#112] ===== Subqueries ===== -Subquery:1 Hosting operator id = 67 Hosting Expression = Subquery scalar-subquery#57, [id=#58] -* HashAggregate (168) -+- * ColumnarToRow (167) - +- CometColumnarExchange (166) - +- RowToColumnar (165) - +- * HashAggregate (164) - +- * ColumnarToRow (163) - +- CometUnion (162) - :- CometProject (150) - : +- CometBroadcastHashJoin (149) - : :- CometScan parquet spark_catalog.default.store_sales (147) - : +- ReusedExchange (148) - :- CometProject (157) - : +- CometBroadcastHashJoin (156) - : :- CometScan parquet spark_catalog.default.catalog_sales (151) - : +- CometBroadcastExchange (155) - : +- CometProject (154) - : +- CometFilter (153) - : +- CometScan parquet spark_catalog.default.date_dim (152) - +- CometProject (161) - +- CometBroadcastHashJoin (160) - :- CometScan parquet spark_catalog.default.web_sales (158) - +- ReusedExchange (159) - - -(147) Scan parquet spark_catalog.default.store_sales +Subquery:1 Hosting operator id = 65 Hosting Expression = Subquery scalar-subquery#57, [id=#58] +* HashAggregate (144) ++- Exchange (143) + +- * HashAggregate (142) + +- * ColumnarToRow (141) + +- CometUnion (140) + :- CometProject (128) + : +- CometBroadcastHashJoin (127) + : :- CometScan parquet spark_catalog.default.store_sales (125) + : +- ReusedExchange (126) + :- CometProject (135) + : +- CometBroadcastHashJoin (134) + : :- CometScan parquet spark_catalog.default.catalog_sales (129) + : +- CometBroadcastExchange (133) + : +- CometProject (132) + : +- CometFilter (131) + : +- CometScan parquet spark_catalog.default.date_dim (130) + +- CometProject (139) + +- CometBroadcastHashJoin (138) + :- CometScan parquet spark_catalog.default.web_sales (136) + +- ReusedExchange (137) + + +(125) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_quantity#207, ss_list_price#208, ss_sold_date_sk#209] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ss_sold_date_sk#209), dynamicpruningexpression(ss_sold_date_sk#209 IN dynamicpruning#210)] ReadSchema: struct -(148) ReusedExchange [Reuses operator id: 19] +(126) ReusedExchange [Reuses operator id: 19] Output [1]: [d_date_sk#211] -(149) CometBroadcastHashJoin +(127) CometBroadcastHashJoin Left output [3]: [ss_quantity#207, ss_list_price#208, ss_sold_date_sk#209] Right output [1]: [d_date_sk#211] Arguments: [ss_sold_date_sk#209], [d_date_sk#211], Inner, BuildRight -(150) CometProject +(128) CometProject Input [4]: [ss_quantity#207, ss_list_price#208, ss_sold_date_sk#209, d_date_sk#211] Arguments: [quantity#212, list_price#213], [ss_quantity#207 AS quantity#212, ss_list_price#208 AS list_price#213] -(151) Scan parquet spark_catalog.default.catalog_sales +(129) Scan parquet spark_catalog.default.catalog_sales Output [3]: [cs_quantity#214, cs_list_price#215, cs_sold_date_sk#216] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#216), dynamicpruningexpression(cs_sold_date_sk#216 IN dynamicpruning#217)] ReadSchema: struct -(152) Scan parquet spark_catalog.default.date_dim +(130) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#218, d_year#219] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(153) CometFilter +(131) CometFilter Input [2]: [d_date_sk#218, d_year#219] Condition : (((isnotnull(d_year#219) AND (d_year#219 >= 1998)) AND (d_year#219 <= 2000)) AND isnotnull(d_date_sk#218)) -(154) CometProject +(132) CometProject Input [2]: [d_date_sk#218, d_year#219] Arguments: [d_date_sk#218], [d_date_sk#218] -(155) CometBroadcastExchange +(133) CometBroadcastExchange Input [1]: [d_date_sk#218] Arguments: [d_date_sk#218] -(156) CometBroadcastHashJoin +(134) CometBroadcastHashJoin Left output [3]: [cs_quantity#214, cs_list_price#215, cs_sold_date_sk#216] Right output [1]: [d_date_sk#218] Arguments: [cs_sold_date_sk#216], [d_date_sk#218], Inner, BuildRight -(157) CometProject +(135) CometProject Input [4]: [cs_quantity#214, cs_list_price#215, cs_sold_date_sk#216, d_date_sk#218] Arguments: [quantity#220, list_price#221], [cs_quantity#214 AS quantity#220, cs_list_price#215 AS list_price#221] -(158) Scan parquet spark_catalog.default.web_sales +(136) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_quantity#222, ws_list_price#223, ws_sold_date_sk#224] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(ws_sold_date_sk#224), dynamicpruningexpression(ws_sold_date_sk#224 IN dynamicpruning#225)] ReadSchema: struct -(159) ReusedExchange [Reuses operator id: 155] +(137) ReusedExchange [Reuses operator id: 133] Output [1]: [d_date_sk#226] -(160) CometBroadcastHashJoin +(138) CometBroadcastHashJoin Left output [3]: [ws_quantity#222, ws_list_price#223, ws_sold_date_sk#224] Right output [1]: [d_date_sk#226] Arguments: [ws_sold_date_sk#224], [d_date_sk#226], Inner, BuildRight -(161) CometProject +(139) CometProject Input [4]: [ws_quantity#222, ws_list_price#223, ws_sold_date_sk#224, d_date_sk#226] Arguments: [quantity#227, list_price#228], [ws_quantity#222 AS quantity#227, ws_list_price#223 AS list_price#228] -(162) CometUnion +(140) CometUnion Child 0 Input [2]: [quantity#212, list_price#213] Child 1 Input [2]: [quantity#220, list_price#221] Child 2 Input [2]: [quantity#227, list_price#228] -(163) ColumnarToRow [codegen id : 1] +(141) ColumnarToRow [codegen id : 1] Input [2]: [quantity#212, list_price#213] -(164) HashAggregate [codegen id : 1] +(142) HashAggregate [codegen id : 1] Input [2]: [quantity#212, list_price#213] Keys: [] Functions [1]: [partial_avg((cast(quantity#212 as decimal(10,0)) * list_price#213))] Aggregate Attributes [2]: [sum#229, count#230] Results [2]: [sum#231, count#232] -(165) RowToColumnar -Input [2]: [sum#231, count#232] - -(166) CometColumnarExchange +(143) Exchange Input [2]: [sum#231, count#232] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=11] -(167) ColumnarToRow [codegen id : 2] -Input [2]: [sum#231, count#232] - -(168) HashAggregate [codegen id : 2] +(144) HashAggregate [codegen id : 2] Input [2]: [sum#231, count#232] Keys: [] Functions [1]: [avg((cast(quantity#212 as decimal(10,0)) * list_price#213))] Aggregate Attributes [1]: [avg((cast(quantity#212 as decimal(10,0)) * list_price#213))#233] Results [1]: [avg((cast(quantity#212 as decimal(10,0)) * list_price#213))#233 AS average_sales#234] -Subquery:2 Hosting operator id = 147 Hosting Expression = ss_sold_date_sk#209 IN dynamicpruning#12 +Subquery:2 Hosting operator id = 125 Hosting Expression = ss_sold_date_sk#209 IN dynamicpruning#12 -Subquery:3 Hosting operator id = 151 Hosting Expression = cs_sold_date_sk#216 IN dynamicpruning#217 -BroadcastExchange (173) -+- * ColumnarToRow (172) - +- CometProject (171) - +- CometFilter (170) - +- CometScan parquet spark_catalog.default.date_dim (169) +Subquery:3 Hosting operator id = 129 Hosting Expression = cs_sold_date_sk#216 IN dynamicpruning#217 +BroadcastExchange (149) ++- * ColumnarToRow (148) + +- CometProject (147) + +- CometFilter (146) + +- CometScan parquet spark_catalog.default.date_dim (145) -(169) Scan parquet spark_catalog.default.date_dim +(145) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#218, d_year#219] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1998), LessThanOrEqual(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(170) CometFilter +(146) CometFilter Input [2]: [d_date_sk#218, d_year#219] Condition : (((isnotnull(d_year#219) AND (d_year#219 >= 1998)) AND (d_year#219 <= 2000)) AND isnotnull(d_date_sk#218)) -(171) CometProject +(147) CometProject Input [2]: [d_date_sk#218, d_year#219] Arguments: [d_date_sk#218], [d_date_sk#218] -(172) ColumnarToRow [codegen id : 1] +(148) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#218] -(173) BroadcastExchange +(149) BroadcastExchange Input [1]: [d_date_sk#218] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] -Subquery:4 Hosting operator id = 158 Hosting Expression = ws_sold_date_sk#224 IN dynamicpruning#217 +Subquery:4 Hosting operator id = 136 Hosting Expression = ws_sold_date_sk#224 IN dynamicpruning#217 Subquery:5 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (178) -+- * ColumnarToRow (177) - +- CometProject (176) - +- CometFilter (175) - +- CometScan parquet spark_catalog.default.date_dim (174) +BroadcastExchange (154) ++- * ColumnarToRow (153) + +- CometProject (152) + +- CometFilter (151) + +- CometScan parquet spark_catalog.default.date_dim (150) -(174) Scan parquet spark_catalog.default.date_dim +(150) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#43, d_year#44, d_moy#45] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,11), IsNotNull(d_date_sk)] ReadSchema: struct -(175) CometFilter +(151) CometFilter Input [3]: [d_date_sk#43, d_year#44, d_moy#45] Condition : ((((isnotnull(d_year#44) AND isnotnull(d_moy#45)) AND (d_year#44 = 2000)) AND (d_moy#45 = 11)) AND isnotnull(d_date_sk#43)) -(176) CometProject +(152) CometProject Input [3]: [d_date_sk#43, d_year#44, d_moy#45] Arguments: [d_date_sk#43], [d_date_sk#43] -(177) ColumnarToRow [codegen id : 1] +(153) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#43] -(178) BroadcastExchange +(154) BroadcastExchange Input [1]: [d_date_sk#43] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] Subquery:6 Hosting operator id = 5 Hosting Expression = ss_sold_date_sk#11 IN dynamicpruning#12 -BroadcastExchange (183) -+- * ColumnarToRow (182) - +- CometProject (181) - +- CometFilter (180) - +- CometScan parquet spark_catalog.default.date_dim (179) +BroadcastExchange (159) ++- * ColumnarToRow (158) + +- CometProject (157) + +- CometFilter (156) + +- CometScan parquet spark_catalog.default.date_dim (155) -(179) Scan parquet spark_catalog.default.date_dim +(155) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#26, d_year#235] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), GreaterThanOrEqual(d_year,1999), LessThanOrEqual(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(180) CometFilter +(156) CometFilter Input [2]: [d_date_sk#26, d_year#235] Condition : (((isnotnull(d_year#235) AND (d_year#235 >= 1999)) AND (d_year#235 <= 2001)) AND isnotnull(d_date_sk#26)) -(181) CometProject +(157) CometProject Input [2]: [d_date_sk#26, d_year#235] Arguments: [d_date_sk#26], [d_date_sk#26] -(182) ColumnarToRow [codegen id : 1] +(158) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#26] -(183) BroadcastExchange +(159) BroadcastExchange Input [1]: [d_date_sk#26] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] @@ -1045,12 +951,12 @@ Subquery:7 Hosting operator id = 9 Hosting Expression = cs_sold_date_sk#18 IN dy Subquery:8 Hosting operator id = 33 Hosting Expression = ws_sold_date_sk#31 IN dynamicpruning#12 -Subquery:9 Hosting operator id = 84 Hosting Expression = ReusedSubquery Subquery scalar-subquery#57, [id=#58] +Subquery:9 Hosting operator id = 80 Hosting Expression = ReusedSubquery Subquery scalar-subquery#57, [id=#58] -Subquery:10 Hosting operator id = 68 Hosting Expression = cs_sold_date_sk#62 IN dynamicpruning#5 +Subquery:10 Hosting operator id = 66 Hosting Expression = cs_sold_date_sk#62 IN dynamicpruning#5 -Subquery:11 Hosting operator id = 101 Hosting Expression = ReusedSubquery Subquery scalar-subquery#57, [id=#58] +Subquery:11 Hosting operator id = 95 Hosting Expression = ReusedSubquery Subquery scalar-subquery#57, [id=#58] -Subquery:12 Hosting operator id = 85 Hosting Expression = ws_sold_date_sk#84 IN dynamicpruning#5 +Subquery:12 Hosting operator id = 81 Hosting Expression = ws_sold_date_sk#84 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt index db1e21cb6..1610cd888 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q14a/simplified.txt @@ -1,241 +1,217 @@ -WholeStageCodegen (46) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - CometColumnarExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 - RowToColumnar - WholeStageCodegen (45) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - InputAdapter - Union - WholeStageCodegen (8) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,i_brand_id,i_class_id,i_category_id] #2 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - InputAdapter - Union - WholeStageCodegen (2) - Filter [sales] - Subquery #3 - WholeStageCodegen (2) - HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] +TakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + WholeStageCodegen (46) + HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + InputAdapter + Exchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 + WholeStageCodegen (45) + HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] + InputAdapter + Union + WholeStageCodegen (8) + HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] + InputAdapter + Exchange [channel,i_brand_id,i_class_id,i_category_id] #2 + WholeStageCodegen (7) + HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] + InputAdapter + Union + WholeStageCodegen (2) + Filter [sales] + Subquery #3 + WholeStageCodegen (2) + HashAggregate [sum,count] [avg((cast(quantity as decimal(10,0)) * list_price)),average_sales,sum,count] + InputAdapter + Exchange #16 + WholeStageCodegen (1) + HashAggregate [quantity,list_price] [sum,count,sum,count] ColumnarToRow InputAdapter - CometColumnarExchange #16 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [quantity,list_price] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometUnion [quantity,list_price] - CometProject [ss_quantity,ss_list_price] [quantity,list_price] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk] #12 - CometProject [cs_quantity,cs_list_price] [quantity,list_price] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #4 - BroadcastExchange #17 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #18 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [ws_quantity,ws_list_price] [quantity,list_price] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #4 - ReusedExchange [d_date_sk] #18 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] + CometUnion [quantity,list_price] + CometProject [ss_quantity,ss_list_price] [quantity,list_price] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_quantity,ss_list_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk] #12 + CometProject [cs_quantity,cs_list_price] [quantity,list_price] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_quantity,cs_list_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #4 + BroadcastExchange #17 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk] #18 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [ws_quantity,ws_list_price] [quantity,list_price] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #4 + ReusedExchange [d_date_sk] #18 + HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] + InputAdapter + Exchange [i_brand_id,i_class_id,i_category_id] #3 + WholeStageCodegen (1) + HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] ColumnarToRow InputAdapter - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_brand_id,i_class_id,i_category_id,ss_quantity,ss_list_price] [sum,isEmpty,count,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] - CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [ss_item_sk] #5 - CometProject [i_item_sk] [ss_item_sk] - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [brand_id,class_id,category_id] #6 - CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] - CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #7 - CometHashAggregate [brand_id,class_id,category_id] - CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] - CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ss_item_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk,ss_item_sk] + CometFilter [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_quantity,ss_list_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [ss_item_sk] #5 + CometProject [i_item_sk] [ss_item_sk] + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,brand_id,class_id,category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [brand_id,class_id,category_id] #6 + CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] + CometHashAggregate [brand_id,class_id,category_id] + CometExchange [brand_id,class_id,category_id] #7 + CometHashAggregate [brand_id,class_id,category_id] + CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] + CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ss_item_sk,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ss_item_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #8 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #9 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,i_brand_id,i_class_id,i_category_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [cs_item_sk,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #10 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [cs_item_sk,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastExchange [d_date_sk] #12 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [d_date_sk] #12 - CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #13 - CometProject [i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometFilter [ws_item_sk,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 - ReusedExchange [d_date_sk] #12 - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 - CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] - ReusedExchange [ss_item_sk] #5 - CometBroadcastExchange [d_date_sk] #15 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (4) - Filter [sales] - ReusedSubquery [average_sales] #3 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] + CometBroadcastExchange [d_date_sk] #12 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [d_date_sk] #12 + CometBroadcastExchange [i_brand_id,i_class_id,i_category_id] #13 + CometProject [i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometFilter [ws_item_sk,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #11 + ReusedExchange [d_date_sk] #12 + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 + CometBroadcastHashJoin [i_item_sk,i_brand_id,i_class_id,i_category_id,ss_item_sk] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id] + ReusedExchange [ss_item_sk] #5 + CometBroadcastExchange [d_date_sk] #15 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (4) + Filter [sales] + ReusedSubquery [average_sales] #3 + HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] + InputAdapter + Exchange [i_brand_id,i_class_id,i_category_id] #19 + WholeStageCodegen (3) + HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count] ColumnarToRow InputAdapter - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #19 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [i_brand_id,i_class_id,i_category_id,cs_quantity,cs_list_price] [sum,isEmpty,count,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] - CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #5 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 - ReusedExchange [d_date_sk] #15 - WholeStageCodegen (6) - Filter [sales] - ReusedSubquery [average_sales] #3 - HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] + CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk,ss_item_sk] + CometFilter [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_quantity,cs_list_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [ss_item_sk] #5 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 + ReusedExchange [d_date_sk] #15 + WholeStageCodegen (6) + Filter [sales] + ReusedSubquery [average_sales] #3 + HashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count] [sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1),channel,sales,number_sales,sum,isEmpty,count] + InputAdapter + Exchange [i_brand_id,i_class_id,i_category_id] #20 + WholeStageCodegen (5) + HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count] ColumnarToRow InputAdapter - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #20 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [i_brand_id,i_class_id,i_category_id,ws_quantity,ws_list_price] [sum,isEmpty,count,sum,isEmpty,count] - ColumnarToRow - InputAdapter - CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] - CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] - CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [ss_item_sk] #5 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 - ReusedExchange [d_date_sk] #15 - WholeStageCodegen (17) - HashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,i_brand_id,i_class_id] #21 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - ColumnarToRow - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (26) - HashAggregate [channel,i_brand_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,i_brand_id] #22 - RowToColumnar - WholeStageCodegen (25) - HashAggregate [channel,i_brand_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - ColumnarToRow - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (35) - HashAggregate [channel,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel] #23 - RowToColumnar - WholeStageCodegen (34) - HashAggregate [channel,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - ColumnarToRow - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 - WholeStageCodegen (44) - HashAggregate [sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange #24 - RowToColumnar - WholeStageCodegen (43) - HashAggregate [sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] - HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] - ColumnarToRow - InputAdapter - ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] + CometProject [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk,ss_item_sk] + CometFilter [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_quantity,ws_list_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [ss_item_sk] #5 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 + ReusedExchange [d_date_sk] #15 + WholeStageCodegen (17) + HashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] + InputAdapter + Exchange [channel,i_brand_id,i_class_id] #21 + WholeStageCodegen (16) + HashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] + HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] + InputAdapter + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + WholeStageCodegen (26) + HashAggregate [channel,i_brand_id,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] + InputAdapter + Exchange [channel,i_brand_id] #22 + WholeStageCodegen (25) + HashAggregate [channel,i_brand_id,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] + HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] + InputAdapter + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + WholeStageCodegen (35) + HashAggregate [channel,sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] + InputAdapter + Exchange [channel] #23 + WholeStageCodegen (34) + HashAggregate [channel,sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] + HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] + InputAdapter + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 + WholeStageCodegen (44) + HashAggregate [sum,isEmpty,sum] [sum(sum_sales),sum(number_salesL),channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum] + InputAdapter + Exchange #24 + WholeStageCodegen (43) + HashAggregate [sum_sales,number_sales] [sum,isEmpty,sum,sum,isEmpty,sum] + HashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] [sum(sales),sum(number_salesL),sum_sales,number_sales,sum,isEmpty,sum] + InputAdapter + ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt index 37189e42b..36ccc06de 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/explain.txt @@ -1,161 +1,151 @@ == Physical Plan == -TakeOrderedAndProject (157) -+- Union (156) - :- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometColumnarExchange (39) - : +- RowToColumnar (38) - : +- * HashAggregate (37) - : +- * ColumnarToRow (36) - : +- CometProject (35) - : +- CometBroadcastHashJoin (34) - : :- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometProject (19) - : : : : +- CometBroadcastHashJoin (18) - : : : : :- CometProject (14) - : : : : : +- CometBroadcastHashJoin (13) - : : : : : :- CometProject (8) - : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : :- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : : : +- CometBroadcastExchange (6) - : : : : : : +- CometProject (5) - : : : : : : +- CometFilter (4) - : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) - : : : : : +- CometBroadcastExchange (12) - : : : : : +- CometProject (11) - : : : : : +- CometFilter (10) - : : : : : +- CometScan parquet spark_catalog.default.customer (9) - : : : : +- CometBroadcastExchange (17) - : : : : +- CometFilter (16) - : : : : +- CometScan parquet spark_catalog.default.customer_demographics (15) - : : : +- CometBroadcastExchange (22) - : : : +- CometFilter (21) - : : : +- CometScan parquet spark_catalog.default.customer_address (20) - : : +- CometBroadcastExchange (28) - : : +- CometProject (27) - : : +- CometFilter (26) - : : +- CometScan parquet spark_catalog.default.date_dim (25) - : +- CometBroadcastExchange (33) - : +- CometFilter (32) - : +- CometScan parquet spark_catalog.default.item (31) - :- * HashAggregate (69) - : +- * ColumnarToRow (68) - : +- CometColumnarExchange (67) - : +- RowToColumnar (66) - : +- * HashAggregate (65) - : +- * ColumnarToRow (64) - : +- CometProject (63) - : +- CometBroadcastHashJoin (62) - : :- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (57) - : : : +- CometBroadcastHashJoin (56) - : : : :- CometProject (52) - : : : : +- CometBroadcastHashJoin (51) - : : : : :- CometProject (49) - : : : : : +- CometBroadcastHashJoin (48) - : : : : : :- CometProject (46) - : : : : : : +- CometBroadcastHashJoin (45) - : : : : : : :- CometFilter (43) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (42) - : : : : : : +- ReusedExchange (44) - : : : : : +- ReusedExchange (47) - : : : : +- ReusedExchange (50) - : : : +- CometBroadcastExchange (55) - : : : +- CometFilter (54) - : : : +- CometScan parquet spark_catalog.default.customer_address (53) - : : +- ReusedExchange (58) - : +- ReusedExchange (61) - :- * HashAggregate (98) - : +- * ColumnarToRow (97) - : +- CometColumnarExchange (96) - : +- RowToColumnar (95) - : +- * HashAggregate (94) - : +- * ColumnarToRow (93) - : +- CometProject (92) - : +- CometBroadcastHashJoin (91) - : :- CometProject (89) - : : +- CometBroadcastHashJoin (88) - : : :- CometProject (86) - : : : +- CometBroadcastHashJoin (85) - : : : :- CometProject (80) - : : : : +- CometBroadcastHashJoin (79) - : : : : :- CometProject (77) - : : : : : +- CometBroadcastHashJoin (76) - : : : : : :- CometProject (74) - : : : : : : +- CometBroadcastHashJoin (73) - : : : : : : :- CometFilter (71) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (70) - : : : : : : +- ReusedExchange (72) - : : : : : +- ReusedExchange (75) - : : : : +- ReusedExchange (78) - : : : +- CometBroadcastExchange (84) - : : : +- CometProject (83) - : : : +- CometFilter (82) - : : : +- CometScan parquet spark_catalog.default.customer_address (81) - : : +- ReusedExchange (87) - : +- ReusedExchange (90) - :- * HashAggregate (127) - : +- * ColumnarToRow (126) - : +- CometColumnarExchange (125) - : +- RowToColumnar (124) - : +- * HashAggregate (123) - : +- * ColumnarToRow (122) - : +- CometProject (121) - : +- CometBroadcastHashJoin (120) - : :- CometProject (118) - : : +- CometBroadcastHashJoin (117) - : : :- CometProject (115) - : : : +- CometBroadcastHashJoin (114) - : : : :- CometProject (109) - : : : : +- CometBroadcastHashJoin (108) - : : : : :- CometProject (106) - : : : : : +- CometBroadcastHashJoin (105) - : : : : : :- CometProject (103) - : : : : : : +- CometBroadcastHashJoin (102) - : : : : : : :- CometFilter (100) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (99) - : : : : : : +- ReusedExchange (101) - : : : : : +- ReusedExchange (104) - : : : : +- ReusedExchange (107) - : : : +- CometBroadcastExchange (113) - : : : +- CometProject (112) - : : : +- CometFilter (111) - : : : +- CometScan parquet spark_catalog.default.customer_address (110) - : : +- ReusedExchange (116) - : +- ReusedExchange (119) - +- * HashAggregate (155) - +- * ColumnarToRow (154) - +- CometColumnarExchange (153) - +- RowToColumnar (152) - +- * HashAggregate (151) - +- * ColumnarToRow (150) - +- CometProject (149) - +- CometBroadcastHashJoin (148) - :- CometProject (144) - : +- CometBroadcastHashJoin (143) - : :- CometProject (141) - : : +- CometBroadcastHashJoin (140) - : : :- CometProject (138) - : : : +- CometBroadcastHashJoin (137) - : : : :- CometProject (135) - : : : : +- CometBroadcastHashJoin (134) - : : : : :- CometProject (132) - : : : : : +- CometBroadcastHashJoin (131) - : : : : : :- CometFilter (129) - : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (128) - : : : : : +- ReusedExchange (130) - : : : : +- ReusedExchange (133) - : : : +- ReusedExchange (136) - : : +- ReusedExchange (139) - : +- ReusedExchange (142) - +- CometBroadcastExchange (147) - +- CometFilter (146) - +- CometScan parquet spark_catalog.default.item (145) +TakeOrderedAndProject (147) ++- Union (146) + :- * HashAggregate (39) + : +- Exchange (38) + : +- * HashAggregate (37) + : +- * ColumnarToRow (36) + : +- CometProject (35) + : +- CometBroadcastHashJoin (34) + : :- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometProject (19) + : : : : +- CometBroadcastHashJoin (18) + : : : : :- CometProject (14) + : : : : : +- CometBroadcastHashJoin (13) + : : : : : :- CometProject (8) + : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : :- CometFilter (2) + : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : +- CometBroadcastExchange (6) + : : : : : : +- CometProject (5) + : : : : : : +- CometFilter (4) + : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : : : : +- CometBroadcastExchange (12) + : : : : : +- CometProject (11) + : : : : : +- CometFilter (10) + : : : : : +- CometScan parquet spark_catalog.default.customer (9) + : : : : +- CometBroadcastExchange (17) + : : : : +- CometFilter (16) + : : : : +- CometScan parquet spark_catalog.default.customer_demographics (15) + : : : +- CometBroadcastExchange (22) + : : : +- CometFilter (21) + : : : +- CometScan parquet spark_catalog.default.customer_address (20) + : : +- CometBroadcastExchange (28) + : : +- CometProject (27) + : : +- CometFilter (26) + : : +- CometScan parquet spark_catalog.default.date_dim (25) + : +- CometBroadcastExchange (33) + : +- CometFilter (32) + : +- CometScan parquet spark_catalog.default.item (31) + :- * HashAggregate (65) + : +- Exchange (64) + : +- * HashAggregate (63) + : +- * ColumnarToRow (62) + : +- CometProject (61) + : +- CometBroadcastHashJoin (60) + : :- CometProject (58) + : : +- CometBroadcastHashJoin (57) + : : :- CometProject (55) + : : : +- CometBroadcastHashJoin (54) + : : : :- CometProject (50) + : : : : +- CometBroadcastHashJoin (49) + : : : : :- CometProject (47) + : : : : : +- CometBroadcastHashJoin (46) + : : : : : :- CometProject (44) + : : : : : : +- CometBroadcastHashJoin (43) + : : : : : : :- CometFilter (41) + : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (40) + : : : : : : +- ReusedExchange (42) + : : : : : +- ReusedExchange (45) + : : : : +- ReusedExchange (48) + : : : +- CometBroadcastExchange (53) + : : : +- CometFilter (52) + : : : +- CometScan parquet spark_catalog.default.customer_address (51) + : : +- ReusedExchange (56) + : +- ReusedExchange (59) + :- * HashAggregate (92) + : +- Exchange (91) + : +- * HashAggregate (90) + : +- * ColumnarToRow (89) + : +- CometProject (88) + : +- CometBroadcastHashJoin (87) + : :- CometProject (85) + : : +- CometBroadcastHashJoin (84) + : : :- CometProject (82) + : : : +- CometBroadcastHashJoin (81) + : : : :- CometProject (76) + : : : : +- CometBroadcastHashJoin (75) + : : : : :- CometProject (73) + : : : : : +- CometBroadcastHashJoin (72) + : : : : : :- CometProject (70) + : : : : : : +- CometBroadcastHashJoin (69) + : : : : : : :- CometFilter (67) + : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (66) + : : : : : : +- ReusedExchange (68) + : : : : : +- ReusedExchange (71) + : : : : +- ReusedExchange (74) + : : : +- CometBroadcastExchange (80) + : : : +- CometProject (79) + : : : +- CometFilter (78) + : : : +- CometScan parquet spark_catalog.default.customer_address (77) + : : +- ReusedExchange (83) + : +- ReusedExchange (86) + :- * HashAggregate (119) + : +- Exchange (118) + : +- * HashAggregate (117) + : +- * ColumnarToRow (116) + : +- CometProject (115) + : +- CometBroadcastHashJoin (114) + : :- CometProject (112) + : : +- CometBroadcastHashJoin (111) + : : :- CometProject (109) + : : : +- CometBroadcastHashJoin (108) + : : : :- CometProject (103) + : : : : +- CometBroadcastHashJoin (102) + : : : : :- CometProject (100) + : : : : : +- CometBroadcastHashJoin (99) + : : : : : :- CometProject (97) + : : : : : : +- CometBroadcastHashJoin (96) + : : : : : : :- CometFilter (94) + : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (93) + : : : : : : +- ReusedExchange (95) + : : : : : +- ReusedExchange (98) + : : : : +- ReusedExchange (101) + : : : +- CometBroadcastExchange (107) + : : : +- CometProject (106) + : : : +- CometFilter (105) + : : : +- CometScan parquet spark_catalog.default.customer_address (104) + : : +- ReusedExchange (110) + : +- ReusedExchange (113) + +- * HashAggregate (145) + +- Exchange (144) + +- * HashAggregate (143) + +- * ColumnarToRow (142) + +- CometProject (141) + +- CometBroadcastHashJoin (140) + :- CometProject (136) + : +- CometBroadcastHashJoin (135) + : :- CometProject (133) + : : +- CometBroadcastHashJoin (132) + : : :- CometProject (130) + : : : +- CometBroadcastHashJoin (129) + : : : :- CometProject (127) + : : : : +- CometBroadcastHashJoin (126) + : : : : :- CometProject (124) + : : : : : +- CometBroadcastHashJoin (123) + : : : : : :- CometFilter (121) + : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (120) + : : : : : +- ReusedExchange (122) + : : : : +- ReusedExchange (125) + : : : +- ReusedExchange (128) + : : +- ReusedExchange (131) + : +- ReusedExchange (134) + +- CometBroadcastExchange (139) + +- CometFilter (138) + +- CometScan parquet spark_catalog.default.item (137) (1) Scan parquet spark_catalog.default.catalog_sales @@ -336,24 +326,18 @@ Functions [7]: [partial_avg(agg1#29), partial_avg(agg2#30), partial_avg(agg3#31) Aggregate Attributes [14]: [sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] Results [18]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] -(38) RowToColumnar +(38) Exchange Input [18]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] +Arguments: hashpartitioning(i_item_id#28, ca_country#24, ca_state#23, ca_county#22, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(39) CometColumnarExchange -Input [18]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] -Arguments: hashpartitioning(i_item_id#28, ca_country#24, ca_state#23, ca_county#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(40) ColumnarToRow [codegen id : 2] -Input [18]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] - -(41) HashAggregate [codegen id : 2] +(39) HashAggregate [codegen id : 2] Input [18]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, sum#50, count#51, sum#52, count#53, sum#54, count#55, sum#56, count#57, sum#58, count#59, sum#60, count#61, sum#62, count#63] Keys [4]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22] Functions [7]: [avg(agg1#29), avg(agg2#30), avg(agg3#31), avg(agg4#32), avg(agg5#33), avg(agg6#34), avg(agg7#35)] Aggregate Attributes [7]: [avg(agg1#29)#64, avg(agg2#30)#65, avg(agg3#31)#66, avg(agg4#32)#67, avg(agg5#33)#68, avg(agg6#34)#69, avg(agg7#35)#70] Results [11]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, avg(agg1#29)#64 AS agg1#71, avg(agg2#30)#65 AS agg2#72, avg(agg3#31)#66 AS agg3#73, avg(agg4#32)#67 AS agg4#74, avg(agg5#33)#68 AS agg5#75, avg(agg6#34)#69 AS agg6#76, avg(agg7#35)#70 AS agg7#77] -(42) Scan parquet spark_catalog.default.catalog_sales +(40) Scan parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#78, cs_bill_cdemo_sk#79, cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86] Batched: true Location: InMemoryFileIndex [] @@ -361,122 +345,116 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#86), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(43) CometFilter +(41) CometFilter Input [9]: [cs_bill_customer_sk#78, cs_bill_cdemo_sk#79, cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86] Condition : ((isnotnull(cs_bill_cdemo_sk#79) AND isnotnull(cs_bill_customer_sk#78)) AND isnotnull(cs_item_sk#80)) -(44) ReusedExchange [Reuses operator id: 6] +(42) ReusedExchange [Reuses operator id: 6] Output [2]: [cd_demo_sk#88, cd_dep_count#89] -(45) CometBroadcastHashJoin +(43) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#78, cs_bill_cdemo_sk#79, cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86] Right output [2]: [cd_demo_sk#88, cd_dep_count#89] Arguments: [cs_bill_cdemo_sk#79], [cd_demo_sk#88], Inner, BuildRight -(46) CometProject +(44) CometProject Input [11]: [cs_bill_customer_sk#78, cs_bill_cdemo_sk#79, cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_demo_sk#88, cd_dep_count#89] Arguments: [cs_bill_customer_sk#78, cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89], [cs_bill_customer_sk#78, cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89] -(47) ReusedExchange [Reuses operator id: 12] +(45) ReusedExchange [Reuses operator id: 12] Output [4]: [c_customer_sk#90, c_current_cdemo_sk#91, c_current_addr_sk#92, c_birth_year#93] -(48) CometBroadcastHashJoin +(46) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#78, cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89] Right output [4]: [c_customer_sk#90, c_current_cdemo_sk#91, c_current_addr_sk#92, c_birth_year#93] Arguments: [cs_bill_customer_sk#78], [c_customer_sk#90], Inner, BuildRight -(49) CometProject +(47) CometProject Input [13]: [cs_bill_customer_sk#78, cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_customer_sk#90, c_current_cdemo_sk#91, c_current_addr_sk#92, c_birth_year#93] Arguments: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_current_cdemo_sk#91, c_current_addr_sk#92, c_birth_year#93], [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_current_cdemo_sk#91, c_current_addr_sk#92, c_birth_year#93] -(50) ReusedExchange [Reuses operator id: 17] +(48) ReusedExchange [Reuses operator id: 17] Output [1]: [cd_demo_sk#94] -(51) CometBroadcastHashJoin +(49) CometBroadcastHashJoin Left output [11]: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_current_cdemo_sk#91, c_current_addr_sk#92, c_birth_year#93] Right output [1]: [cd_demo_sk#94] Arguments: [c_current_cdemo_sk#91], [cd_demo_sk#94], Inner, BuildRight -(52) CometProject +(50) CometProject Input [12]: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_current_cdemo_sk#91, c_current_addr_sk#92, c_birth_year#93, cd_demo_sk#94] Arguments: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_current_addr_sk#92, c_birth_year#93], [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_current_addr_sk#92, c_birth_year#93] -(53) Scan parquet spark_catalog.default.customer_address +(51) Scan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#95, ca_state#96, ca_country#97] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] ReadSchema: struct -(54) CometFilter +(52) CometFilter Input [3]: [ca_address_sk#95, ca_state#96, ca_country#97] Condition : (ca_state#96 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#95)) -(55) CometBroadcastExchange +(53) CometBroadcastExchange Input [3]: [ca_address_sk#95, ca_state#96, ca_country#97] Arguments: [ca_address_sk#95, ca_state#96, ca_country#97] -(56) CometBroadcastHashJoin +(54) CometBroadcastHashJoin Left output [10]: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_current_addr_sk#92, c_birth_year#93] Right output [3]: [ca_address_sk#95, ca_state#96, ca_country#97] Arguments: [c_current_addr_sk#92], [ca_address_sk#95], Inner, BuildRight -(57) CometProject +(55) CometProject Input [13]: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_current_addr_sk#92, c_birth_year#93, ca_address_sk#95, ca_state#96, ca_country#97] Arguments: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_birth_year#93, ca_state#96, ca_country#97], [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_birth_year#93, ca_state#96, ca_country#97] -(58) ReusedExchange [Reuses operator id: 28] +(56) ReusedExchange [Reuses operator id: 28] Output [1]: [d_date_sk#98] -(59) CometBroadcastHashJoin +(57) CometBroadcastHashJoin Left output [11]: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_birth_year#93, ca_state#96, ca_country#97] Right output [1]: [d_date_sk#98] Arguments: [cs_sold_date_sk#86], [d_date_sk#98], Inner, BuildRight -(60) CometProject +(58) CometProject Input [12]: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cs_sold_date_sk#86, cd_dep_count#89, c_birth_year#93, ca_state#96, ca_country#97, d_date_sk#98] Arguments: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cd_dep_count#89, c_birth_year#93, ca_state#96, ca_country#97], [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cd_dep_count#89, c_birth_year#93, ca_state#96, ca_country#97] -(61) ReusedExchange [Reuses operator id: 33] +(59) ReusedExchange [Reuses operator id: 33] Output [2]: [i_item_sk#99, i_item_id#100] -(62) CometBroadcastHashJoin +(60) CometBroadcastHashJoin Left output [10]: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cd_dep_count#89, c_birth_year#93, ca_state#96, ca_country#97] Right output [2]: [i_item_sk#99, i_item_id#100] Arguments: [cs_item_sk#80], [i_item_sk#99], Inner, BuildRight -(63) CometProject +(61) CometProject Input [12]: [cs_item_sk#80, cs_quantity#81, cs_list_price#82, cs_sales_price#83, cs_coupon_amt#84, cs_net_profit#85, cd_dep_count#89, c_birth_year#93, ca_state#96, ca_country#97, i_item_sk#99, i_item_id#100] Arguments: [i_item_id#100, ca_country#97, ca_state#96, agg1#101, agg2#102, agg3#103, agg4#104, agg5#105, agg6#106, agg7#107], [i_item_id#100, ca_country#97, ca_state#96, cast(cs_quantity#81 as decimal(12,2)) AS agg1#101, cast(cs_list_price#82 as decimal(12,2)) AS agg2#102, cast(cs_coupon_amt#84 as decimal(12,2)) AS agg3#103, cast(cs_sales_price#83 as decimal(12,2)) AS agg4#104, cast(cs_net_profit#85 as decimal(12,2)) AS agg5#105, cast(c_birth_year#93 as decimal(12,2)) AS agg6#106, cast(cd_dep_count#89 as decimal(12,2)) AS agg7#107] -(64) ColumnarToRow [codegen id : 3] +(62) ColumnarToRow [codegen id : 3] Input [10]: [i_item_id#100, ca_country#97, ca_state#96, agg1#101, agg2#102, agg3#103, agg4#104, agg5#105, agg6#106, agg7#107] -(65) HashAggregate [codegen id : 3] +(63) HashAggregate [codegen id : 3] Input [10]: [i_item_id#100, ca_country#97, ca_state#96, agg1#101, agg2#102, agg3#103, agg4#104, agg5#105, agg6#106, agg7#107] Keys [3]: [i_item_id#100, ca_country#97, ca_state#96] Functions [7]: [partial_avg(agg1#101), partial_avg(agg2#102), partial_avg(agg3#103), partial_avg(agg4#104), partial_avg(agg5#105), partial_avg(agg6#106), partial_avg(agg7#107)] Aggregate Attributes [14]: [sum#108, count#109, sum#110, count#111, sum#112, count#113, sum#114, count#115, sum#116, count#117, sum#118, count#119, sum#120, count#121] Results [17]: [i_item_id#100, ca_country#97, ca_state#96, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135] -(66) RowToColumnar -Input [17]: [i_item_id#100, ca_country#97, ca_state#96, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135] - -(67) CometColumnarExchange -Input [17]: [i_item_id#100, ca_country#97, ca_state#96, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135] -Arguments: hashpartitioning(i_item_id#100, ca_country#97, ca_state#96, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(68) ColumnarToRow [codegen id : 4] +(64) Exchange Input [17]: [i_item_id#100, ca_country#97, ca_state#96, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135] +Arguments: hashpartitioning(i_item_id#100, ca_country#97, ca_state#96, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(69) HashAggregate [codegen id : 4] +(65) HashAggregate [codegen id : 4] Input [17]: [i_item_id#100, ca_country#97, ca_state#96, sum#122, count#123, sum#124, count#125, sum#126, count#127, sum#128, count#129, sum#130, count#131, sum#132, count#133, sum#134, count#135] Keys [3]: [i_item_id#100, ca_country#97, ca_state#96] Functions [7]: [avg(agg1#101), avg(agg2#102), avg(agg3#103), avg(agg4#104), avg(agg5#105), avg(agg6#106), avg(agg7#107)] Aggregate Attributes [7]: [avg(agg1#101)#136, avg(agg2#102)#137, avg(agg3#103)#138, avg(agg4#104)#139, avg(agg5#105)#140, avg(agg6#106)#141, avg(agg7#107)#142] Results [11]: [i_item_id#100, ca_country#97, ca_state#96, null AS county#143, avg(agg1#101)#136 AS agg1#144, avg(agg2#102)#137 AS agg2#145, avg(agg3#103)#138 AS agg3#146, avg(agg4#104)#139 AS agg4#147, avg(agg5#105)#140 AS agg5#148, avg(agg6#106)#141 AS agg6#149, avg(agg7#107)#142 AS agg7#150] -(70) Scan parquet spark_catalog.default.catalog_sales +(66) Scan parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#151, cs_bill_cdemo_sk#152, cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159] Batched: true Location: InMemoryFileIndex [] @@ -484,126 +462,120 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#159), dynamicpruningexpression(cs_s PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(71) CometFilter +(67) CometFilter Input [9]: [cs_bill_customer_sk#151, cs_bill_cdemo_sk#152, cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159] Condition : ((isnotnull(cs_bill_cdemo_sk#152) AND isnotnull(cs_bill_customer_sk#151)) AND isnotnull(cs_item_sk#153)) -(72) ReusedExchange [Reuses operator id: 6] +(68) ReusedExchange [Reuses operator id: 6] Output [2]: [cd_demo_sk#161, cd_dep_count#162] -(73) CometBroadcastHashJoin +(69) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#151, cs_bill_cdemo_sk#152, cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159] Right output [2]: [cd_demo_sk#161, cd_dep_count#162] Arguments: [cs_bill_cdemo_sk#152], [cd_demo_sk#161], Inner, BuildRight -(74) CometProject +(70) CometProject Input [11]: [cs_bill_customer_sk#151, cs_bill_cdemo_sk#152, cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_demo_sk#161, cd_dep_count#162] Arguments: [cs_bill_customer_sk#151, cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162], [cs_bill_customer_sk#151, cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162] -(75) ReusedExchange [Reuses operator id: 12] +(71) ReusedExchange [Reuses operator id: 12] Output [4]: [c_customer_sk#163, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166] -(76) CometBroadcastHashJoin +(72) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#151, cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162] Right output [4]: [c_customer_sk#163, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166] Arguments: [cs_bill_customer_sk#151], [c_customer_sk#163], Inner, BuildRight -(77) CometProject +(73) CometProject Input [13]: [cs_bill_customer_sk#151, cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_customer_sk#163, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166] Arguments: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166], [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166] -(78) ReusedExchange [Reuses operator id: 17] +(74) ReusedExchange [Reuses operator id: 17] Output [1]: [cd_demo_sk#167] -(79) CometBroadcastHashJoin +(75) CometBroadcastHashJoin Left output [11]: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166] Right output [1]: [cd_demo_sk#167] Arguments: [c_current_cdemo_sk#164], [cd_demo_sk#167], Inner, BuildRight -(80) CometProject +(76) CometProject Input [12]: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_current_cdemo_sk#164, c_current_addr_sk#165, c_birth_year#166, cd_demo_sk#167] Arguments: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_current_addr_sk#165, c_birth_year#166], [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_current_addr_sk#165, c_birth_year#166] -(81) Scan parquet spark_catalog.default.customer_address +(77) Scan parquet spark_catalog.default.customer_address Output [3]: [ca_address_sk#168, ca_state#169, ca_country#170] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] ReadSchema: struct -(82) CometFilter +(78) CometFilter Input [3]: [ca_address_sk#168, ca_state#169, ca_country#170] Condition : (ca_state#169 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#168)) -(83) CometProject +(79) CometProject Input [3]: [ca_address_sk#168, ca_state#169, ca_country#170] Arguments: [ca_address_sk#168, ca_country#170], [ca_address_sk#168, ca_country#170] -(84) CometBroadcastExchange +(80) CometBroadcastExchange Input [2]: [ca_address_sk#168, ca_country#170] Arguments: [ca_address_sk#168, ca_country#170] -(85) CometBroadcastHashJoin +(81) CometBroadcastHashJoin Left output [10]: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_current_addr_sk#165, c_birth_year#166] Right output [2]: [ca_address_sk#168, ca_country#170] Arguments: [c_current_addr_sk#165], [ca_address_sk#168], Inner, BuildRight -(86) CometProject +(82) CometProject Input [12]: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_current_addr_sk#165, c_birth_year#166, ca_address_sk#168, ca_country#170] Arguments: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_birth_year#166, ca_country#170], [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_birth_year#166, ca_country#170] -(87) ReusedExchange [Reuses operator id: 28] +(83) ReusedExchange [Reuses operator id: 28] Output [1]: [d_date_sk#171] -(88) CometBroadcastHashJoin +(84) CometBroadcastHashJoin Left output [10]: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_birth_year#166, ca_country#170] Right output [1]: [d_date_sk#171] Arguments: [cs_sold_date_sk#159], [d_date_sk#171], Inner, BuildRight -(89) CometProject +(85) CometProject Input [11]: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cs_sold_date_sk#159, cd_dep_count#162, c_birth_year#166, ca_country#170, d_date_sk#171] Arguments: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cd_dep_count#162, c_birth_year#166, ca_country#170], [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cd_dep_count#162, c_birth_year#166, ca_country#170] -(90) ReusedExchange [Reuses operator id: 33] +(86) ReusedExchange [Reuses operator id: 33] Output [2]: [i_item_sk#172, i_item_id#173] -(91) CometBroadcastHashJoin +(87) CometBroadcastHashJoin Left output [9]: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cd_dep_count#162, c_birth_year#166, ca_country#170] Right output [2]: [i_item_sk#172, i_item_id#173] Arguments: [cs_item_sk#153], [i_item_sk#172], Inner, BuildRight -(92) CometProject +(88) CometProject Input [11]: [cs_item_sk#153, cs_quantity#154, cs_list_price#155, cs_sales_price#156, cs_coupon_amt#157, cs_net_profit#158, cd_dep_count#162, c_birth_year#166, ca_country#170, i_item_sk#172, i_item_id#173] Arguments: [i_item_id#173, ca_country#170, agg1#174, agg2#175, agg3#176, agg4#177, agg5#178, agg6#179, agg7#180], [i_item_id#173, ca_country#170, cast(cs_quantity#154 as decimal(12,2)) AS agg1#174, cast(cs_list_price#155 as decimal(12,2)) AS agg2#175, cast(cs_coupon_amt#157 as decimal(12,2)) AS agg3#176, cast(cs_sales_price#156 as decimal(12,2)) AS agg4#177, cast(cs_net_profit#158 as decimal(12,2)) AS agg5#178, cast(c_birth_year#166 as decimal(12,2)) AS agg6#179, cast(cd_dep_count#162 as decimal(12,2)) AS agg7#180] -(93) ColumnarToRow [codegen id : 5] +(89) ColumnarToRow [codegen id : 5] Input [9]: [i_item_id#173, ca_country#170, agg1#174, agg2#175, agg3#176, agg4#177, agg5#178, agg6#179, agg7#180] -(94) HashAggregate [codegen id : 5] +(90) HashAggregate [codegen id : 5] Input [9]: [i_item_id#173, ca_country#170, agg1#174, agg2#175, agg3#176, agg4#177, agg5#178, agg6#179, agg7#180] Keys [2]: [i_item_id#173, ca_country#170] Functions [7]: [partial_avg(agg1#174), partial_avg(agg2#175), partial_avg(agg3#176), partial_avg(agg4#177), partial_avg(agg5#178), partial_avg(agg6#179), partial_avg(agg7#180)] Aggregate Attributes [14]: [sum#181, count#182, sum#183, count#184, sum#185, count#186, sum#187, count#188, sum#189, count#190, sum#191, count#192, sum#193, count#194] Results [16]: [i_item_id#173, ca_country#170, sum#195, count#196, sum#197, count#198, sum#199, count#200, sum#201, count#202, sum#203, count#204, sum#205, count#206, sum#207, count#208] -(95) RowToColumnar -Input [16]: [i_item_id#173, ca_country#170, sum#195, count#196, sum#197, count#198, sum#199, count#200, sum#201, count#202, sum#203, count#204, sum#205, count#206, sum#207, count#208] - -(96) CometColumnarExchange +(91) Exchange Input [16]: [i_item_id#173, ca_country#170, sum#195, count#196, sum#197, count#198, sum#199, count#200, sum#201, count#202, sum#203, count#204, sum#205, count#206, sum#207, count#208] -Arguments: hashpartitioning(i_item_id#173, ca_country#170, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#173, ca_country#170, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(97) ColumnarToRow [codegen id : 6] -Input [16]: [i_item_id#173, ca_country#170, sum#195, count#196, sum#197, count#198, sum#199, count#200, sum#201, count#202, sum#203, count#204, sum#205, count#206, sum#207, count#208] - -(98) HashAggregate [codegen id : 6] +(92) HashAggregate [codegen id : 6] Input [16]: [i_item_id#173, ca_country#170, sum#195, count#196, sum#197, count#198, sum#199, count#200, sum#201, count#202, sum#203, count#204, sum#205, count#206, sum#207, count#208] Keys [2]: [i_item_id#173, ca_country#170] Functions [7]: [avg(agg1#174), avg(agg2#175), avg(agg3#176), avg(agg4#177), avg(agg5#178), avg(agg6#179), avg(agg7#180)] Aggregate Attributes [7]: [avg(agg1#174)#209, avg(agg2#175)#210, avg(agg3#176)#211, avg(agg4#177)#212, avg(agg5#178)#213, avg(agg6#179)#214, avg(agg7#180)#215] Results [11]: [i_item_id#173, ca_country#170, null AS ca_state#216, null AS county#217, avg(agg1#174)#209 AS agg1#218, avg(agg2#175)#210 AS agg2#219, avg(agg3#176)#211 AS agg3#220, avg(agg4#177)#212 AS agg4#221, avg(agg5#178)#213 AS agg5#222, avg(agg6#179)#214 AS agg6#223, avg(agg7#180)#215 AS agg7#224] -(99) Scan parquet spark_catalog.default.catalog_sales +(93) Scan parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#225, cs_bill_cdemo_sk#226, cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233] Batched: true Location: InMemoryFileIndex [] @@ -611,126 +583,120 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#233), dynamicpruningexpression(cs_s PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(100) CometFilter +(94) CometFilter Input [9]: [cs_bill_customer_sk#225, cs_bill_cdemo_sk#226, cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233] Condition : ((isnotnull(cs_bill_cdemo_sk#226) AND isnotnull(cs_bill_customer_sk#225)) AND isnotnull(cs_item_sk#227)) -(101) ReusedExchange [Reuses operator id: 6] +(95) ReusedExchange [Reuses operator id: 6] Output [2]: [cd_demo_sk#235, cd_dep_count#236] -(102) CometBroadcastHashJoin +(96) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#225, cs_bill_cdemo_sk#226, cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233] Right output [2]: [cd_demo_sk#235, cd_dep_count#236] Arguments: [cs_bill_cdemo_sk#226], [cd_demo_sk#235], Inner, BuildRight -(103) CometProject +(97) CometProject Input [11]: [cs_bill_customer_sk#225, cs_bill_cdemo_sk#226, cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_demo_sk#235, cd_dep_count#236] Arguments: [cs_bill_customer_sk#225, cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236], [cs_bill_customer_sk#225, cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236] -(104) ReusedExchange [Reuses operator id: 12] +(98) ReusedExchange [Reuses operator id: 12] Output [4]: [c_customer_sk#237, c_current_cdemo_sk#238, c_current_addr_sk#239, c_birth_year#240] -(105) CometBroadcastHashJoin +(99) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#225, cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236] Right output [4]: [c_customer_sk#237, c_current_cdemo_sk#238, c_current_addr_sk#239, c_birth_year#240] Arguments: [cs_bill_customer_sk#225], [c_customer_sk#237], Inner, BuildRight -(106) CometProject +(100) CometProject Input [13]: [cs_bill_customer_sk#225, cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_customer_sk#237, c_current_cdemo_sk#238, c_current_addr_sk#239, c_birth_year#240] Arguments: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_current_cdemo_sk#238, c_current_addr_sk#239, c_birth_year#240], [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_current_cdemo_sk#238, c_current_addr_sk#239, c_birth_year#240] -(107) ReusedExchange [Reuses operator id: 17] +(101) ReusedExchange [Reuses operator id: 17] Output [1]: [cd_demo_sk#241] -(108) CometBroadcastHashJoin +(102) CometBroadcastHashJoin Left output [11]: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_current_cdemo_sk#238, c_current_addr_sk#239, c_birth_year#240] Right output [1]: [cd_demo_sk#241] Arguments: [c_current_cdemo_sk#238], [cd_demo_sk#241], Inner, BuildRight -(109) CometProject +(103) CometProject Input [12]: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_current_cdemo_sk#238, c_current_addr_sk#239, c_birth_year#240, cd_demo_sk#241] Arguments: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_current_addr_sk#239, c_birth_year#240], [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_current_addr_sk#239, c_birth_year#240] -(110) Scan parquet spark_catalog.default.customer_address +(104) Scan parquet spark_catalog.default.customer_address Output [2]: [ca_address_sk#242, ca_state#243] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [In(ca_state, [AL,MS,NC,ND,OK,TN,WI]), IsNotNull(ca_address_sk)] ReadSchema: struct -(111) CometFilter +(105) CometFilter Input [2]: [ca_address_sk#242, ca_state#243] Condition : (ca_state#243 IN (ND,WI,AL,NC,OK,MS,TN) AND isnotnull(ca_address_sk#242)) -(112) CometProject +(106) CometProject Input [2]: [ca_address_sk#242, ca_state#243] Arguments: [ca_address_sk#242], [ca_address_sk#242] -(113) CometBroadcastExchange +(107) CometBroadcastExchange Input [1]: [ca_address_sk#242] Arguments: [ca_address_sk#242] -(114) CometBroadcastHashJoin +(108) CometBroadcastHashJoin Left output [10]: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_current_addr_sk#239, c_birth_year#240] Right output [1]: [ca_address_sk#242] Arguments: [c_current_addr_sk#239], [ca_address_sk#242], Inner, BuildRight -(115) CometProject +(109) CometProject Input [11]: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_current_addr_sk#239, c_birth_year#240, ca_address_sk#242] Arguments: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_birth_year#240], [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_birth_year#240] -(116) ReusedExchange [Reuses operator id: 28] +(110) ReusedExchange [Reuses operator id: 28] Output [1]: [d_date_sk#244] -(117) CometBroadcastHashJoin +(111) CometBroadcastHashJoin Left output [9]: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_birth_year#240] Right output [1]: [d_date_sk#244] Arguments: [cs_sold_date_sk#233], [d_date_sk#244], Inner, BuildRight -(118) CometProject +(112) CometProject Input [10]: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cs_sold_date_sk#233, cd_dep_count#236, c_birth_year#240, d_date_sk#244] Arguments: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cd_dep_count#236, c_birth_year#240], [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cd_dep_count#236, c_birth_year#240] -(119) ReusedExchange [Reuses operator id: 33] +(113) ReusedExchange [Reuses operator id: 33] Output [2]: [i_item_sk#245, i_item_id#246] -(120) CometBroadcastHashJoin +(114) CometBroadcastHashJoin Left output [8]: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cd_dep_count#236, c_birth_year#240] Right output [2]: [i_item_sk#245, i_item_id#246] Arguments: [cs_item_sk#227], [i_item_sk#245], Inner, BuildRight -(121) CometProject +(115) CometProject Input [10]: [cs_item_sk#227, cs_quantity#228, cs_list_price#229, cs_sales_price#230, cs_coupon_amt#231, cs_net_profit#232, cd_dep_count#236, c_birth_year#240, i_item_sk#245, i_item_id#246] Arguments: [i_item_id#246, agg1#247, agg2#248, agg3#249, agg4#250, agg5#251, agg6#252, agg7#253], [i_item_id#246, cast(cs_quantity#228 as decimal(12,2)) AS agg1#247, cast(cs_list_price#229 as decimal(12,2)) AS agg2#248, cast(cs_coupon_amt#231 as decimal(12,2)) AS agg3#249, cast(cs_sales_price#230 as decimal(12,2)) AS agg4#250, cast(cs_net_profit#232 as decimal(12,2)) AS agg5#251, cast(c_birth_year#240 as decimal(12,2)) AS agg6#252, cast(cd_dep_count#236 as decimal(12,2)) AS agg7#253] -(122) ColumnarToRow [codegen id : 7] +(116) ColumnarToRow [codegen id : 7] Input [8]: [i_item_id#246, agg1#247, agg2#248, agg3#249, agg4#250, agg5#251, agg6#252, agg7#253] -(123) HashAggregate [codegen id : 7] +(117) HashAggregate [codegen id : 7] Input [8]: [i_item_id#246, agg1#247, agg2#248, agg3#249, agg4#250, agg5#251, agg6#252, agg7#253] Keys [1]: [i_item_id#246] Functions [7]: [partial_avg(agg1#247), partial_avg(agg2#248), partial_avg(agg3#249), partial_avg(agg4#250), partial_avg(agg5#251), partial_avg(agg6#252), partial_avg(agg7#253)] Aggregate Attributes [14]: [sum#254, count#255, sum#256, count#257, sum#258, count#259, sum#260, count#261, sum#262, count#263, sum#264, count#265, sum#266, count#267] Results [15]: [i_item_id#246, sum#268, count#269, sum#270, count#271, sum#272, count#273, sum#274, count#275, sum#276, count#277, sum#278, count#279, sum#280, count#281] -(124) RowToColumnar -Input [15]: [i_item_id#246, sum#268, count#269, sum#270, count#271, sum#272, count#273, sum#274, count#275, sum#276, count#277, sum#278, count#279, sum#280, count#281] - -(125) CometColumnarExchange +(118) Exchange Input [15]: [i_item_id#246, sum#268, count#269, sum#270, count#271, sum#272, count#273, sum#274, count#275, sum#276, count#277, sum#278, count#279, sum#280, count#281] -Arguments: hashpartitioning(i_item_id#246, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_item_id#246, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(126) ColumnarToRow [codegen id : 8] -Input [15]: [i_item_id#246, sum#268, count#269, sum#270, count#271, sum#272, count#273, sum#274, count#275, sum#276, count#277, sum#278, count#279, sum#280, count#281] - -(127) HashAggregate [codegen id : 8] +(119) HashAggregate [codegen id : 8] Input [15]: [i_item_id#246, sum#268, count#269, sum#270, count#271, sum#272, count#273, sum#274, count#275, sum#276, count#277, sum#278, count#279, sum#280, count#281] Keys [1]: [i_item_id#246] Functions [7]: [avg(agg1#247), avg(agg2#248), avg(agg3#249), avg(agg4#250), avg(agg5#251), avg(agg6#252), avg(agg7#253)] Aggregate Attributes [7]: [avg(agg1#247)#282, avg(agg2#248)#283, avg(agg3#249)#284, avg(agg4#250)#285, avg(agg5#251)#286, avg(agg6#252)#287, avg(agg7#253)#288] Results [11]: [i_item_id#246, null AS ca_country#289, null AS ca_state#290, null AS county#291, avg(agg1#247)#282 AS agg1#292, avg(agg2#248)#283 AS agg2#293, avg(agg3#249)#284 AS agg3#294, avg(agg4#250)#285 AS agg4#295, avg(agg5#251)#286 AS agg5#296, avg(agg6#252)#287 AS agg6#297, avg(agg7#253)#288 AS agg7#298] -(128) Scan parquet spark_catalog.default.catalog_sales +(120) Scan parquet spark_catalog.default.catalog_sales Output [9]: [cs_bill_customer_sk#299, cs_bill_cdemo_sk#300, cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307] Batched: true Location: InMemoryFileIndex [] @@ -738,165 +704,159 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#307), dynamicpruningexpression(cs_s PushedFilters: [IsNotNull(cs_bill_cdemo_sk), IsNotNull(cs_bill_customer_sk), IsNotNull(cs_item_sk)] ReadSchema: struct -(129) CometFilter +(121) CometFilter Input [9]: [cs_bill_customer_sk#299, cs_bill_cdemo_sk#300, cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307] Condition : ((isnotnull(cs_bill_cdemo_sk#300) AND isnotnull(cs_bill_customer_sk#299)) AND isnotnull(cs_item_sk#301)) -(130) ReusedExchange [Reuses operator id: 6] +(122) ReusedExchange [Reuses operator id: 6] Output [2]: [cd_demo_sk#309, cd_dep_count#310] -(131) CometBroadcastHashJoin +(123) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#299, cs_bill_cdemo_sk#300, cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307] Right output [2]: [cd_demo_sk#309, cd_dep_count#310] Arguments: [cs_bill_cdemo_sk#300], [cd_demo_sk#309], Inner, BuildRight -(132) CometProject +(124) CometProject Input [11]: [cs_bill_customer_sk#299, cs_bill_cdemo_sk#300, cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_demo_sk#309, cd_dep_count#310] Arguments: [cs_bill_customer_sk#299, cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310], [cs_bill_customer_sk#299, cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310] -(133) ReusedExchange [Reuses operator id: 12] +(125) ReusedExchange [Reuses operator id: 12] Output [4]: [c_customer_sk#311, c_current_cdemo_sk#312, c_current_addr_sk#313, c_birth_year#314] -(134) CometBroadcastHashJoin +(126) CometBroadcastHashJoin Left output [9]: [cs_bill_customer_sk#299, cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310] Right output [4]: [c_customer_sk#311, c_current_cdemo_sk#312, c_current_addr_sk#313, c_birth_year#314] Arguments: [cs_bill_customer_sk#299], [c_customer_sk#311], Inner, BuildRight -(135) CometProject +(127) CometProject Input [13]: [cs_bill_customer_sk#299, cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_customer_sk#311, c_current_cdemo_sk#312, c_current_addr_sk#313, c_birth_year#314] Arguments: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_current_cdemo_sk#312, c_current_addr_sk#313, c_birth_year#314], [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_current_cdemo_sk#312, c_current_addr_sk#313, c_birth_year#314] -(136) ReusedExchange [Reuses operator id: 17] +(128) ReusedExchange [Reuses operator id: 17] Output [1]: [cd_demo_sk#315] -(137) CometBroadcastHashJoin +(129) CometBroadcastHashJoin Left output [11]: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_current_cdemo_sk#312, c_current_addr_sk#313, c_birth_year#314] Right output [1]: [cd_demo_sk#315] Arguments: [c_current_cdemo_sk#312], [cd_demo_sk#315], Inner, BuildRight -(138) CometProject +(130) CometProject Input [12]: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_current_cdemo_sk#312, c_current_addr_sk#313, c_birth_year#314, cd_demo_sk#315] Arguments: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_current_addr_sk#313, c_birth_year#314], [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_current_addr_sk#313, c_birth_year#314] -(139) ReusedExchange [Reuses operator id: 113] +(131) ReusedExchange [Reuses operator id: 107] Output [1]: [ca_address_sk#316] -(140) CometBroadcastHashJoin +(132) CometBroadcastHashJoin Left output [10]: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_current_addr_sk#313, c_birth_year#314] Right output [1]: [ca_address_sk#316] Arguments: [c_current_addr_sk#313], [ca_address_sk#316], Inner, BuildRight -(141) CometProject +(133) CometProject Input [11]: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_current_addr_sk#313, c_birth_year#314, ca_address_sk#316] Arguments: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_birth_year#314], [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_birth_year#314] -(142) ReusedExchange [Reuses operator id: 28] +(134) ReusedExchange [Reuses operator id: 28] Output [1]: [d_date_sk#317] -(143) CometBroadcastHashJoin +(135) CometBroadcastHashJoin Left output [9]: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_birth_year#314] Right output [1]: [d_date_sk#317] Arguments: [cs_sold_date_sk#307], [d_date_sk#317], Inner, BuildRight -(144) CometProject +(136) CometProject Input [10]: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cs_sold_date_sk#307, cd_dep_count#310, c_birth_year#314, d_date_sk#317] Arguments: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cd_dep_count#310, c_birth_year#314], [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cd_dep_count#310, c_birth_year#314] -(145) Scan parquet spark_catalog.default.item +(137) Scan parquet spark_catalog.default.item Output [1]: [i_item_sk#318] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(146) CometFilter +(138) CometFilter Input [1]: [i_item_sk#318] Condition : isnotnull(i_item_sk#318) -(147) CometBroadcastExchange +(139) CometBroadcastExchange Input [1]: [i_item_sk#318] Arguments: [i_item_sk#318] -(148) CometBroadcastHashJoin +(140) CometBroadcastHashJoin Left output [8]: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cd_dep_count#310, c_birth_year#314] Right output [1]: [i_item_sk#318] Arguments: [cs_item_sk#301], [i_item_sk#318], Inner, BuildRight -(149) CometProject +(141) CometProject Input [9]: [cs_item_sk#301, cs_quantity#302, cs_list_price#303, cs_sales_price#304, cs_coupon_amt#305, cs_net_profit#306, cd_dep_count#310, c_birth_year#314, i_item_sk#318] Arguments: [agg1#319, agg2#320, agg3#321, agg4#322, agg5#323, agg6#324, agg7#325], [cast(cs_quantity#302 as decimal(12,2)) AS agg1#319, cast(cs_list_price#303 as decimal(12,2)) AS agg2#320, cast(cs_coupon_amt#305 as decimal(12,2)) AS agg3#321, cast(cs_sales_price#304 as decimal(12,2)) AS agg4#322, cast(cs_net_profit#306 as decimal(12,2)) AS agg5#323, cast(c_birth_year#314 as decimal(12,2)) AS agg6#324, cast(cd_dep_count#310 as decimal(12,2)) AS agg7#325] -(150) ColumnarToRow [codegen id : 9] +(142) ColumnarToRow [codegen id : 9] Input [7]: [agg1#319, agg2#320, agg3#321, agg4#322, agg5#323, agg6#324, agg7#325] -(151) HashAggregate [codegen id : 9] +(143) HashAggregate [codegen id : 9] Input [7]: [agg1#319, agg2#320, agg3#321, agg4#322, agg5#323, agg6#324, agg7#325] Keys: [] Functions [7]: [partial_avg(agg1#319), partial_avg(agg2#320), partial_avg(agg3#321), partial_avg(agg4#322), partial_avg(agg5#323), partial_avg(agg6#324), partial_avg(agg7#325)] Aggregate Attributes [14]: [sum#326, count#327, sum#328, count#329, sum#330, count#331, sum#332, count#333, sum#334, count#335, sum#336, count#337, sum#338, count#339] Results [14]: [sum#340, count#341, sum#342, count#343, sum#344, count#345, sum#346, count#347, sum#348, count#349, sum#350, count#351, sum#352, count#353] -(152) RowToColumnar -Input [14]: [sum#340, count#341, sum#342, count#343, sum#344, count#345, sum#346, count#347, sum#348, count#349, sum#350, count#351, sum#352, count#353] - -(153) CometColumnarExchange -Input [14]: [sum#340, count#341, sum#342, count#343, sum#344, count#345, sum#346, count#347, sum#348, count#349, sum#350, count#351, sum#352, count#353] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(154) ColumnarToRow [codegen id : 10] +(144) Exchange Input [14]: [sum#340, count#341, sum#342, count#343, sum#344, count#345, sum#346, count#347, sum#348, count#349, sum#350, count#351, sum#352, count#353] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] -(155) HashAggregate [codegen id : 10] +(145) HashAggregate [codegen id : 10] Input [14]: [sum#340, count#341, sum#342, count#343, sum#344, count#345, sum#346, count#347, sum#348, count#349, sum#350, count#351, sum#352, count#353] Keys: [] Functions [7]: [avg(agg1#319), avg(agg2#320), avg(agg3#321), avg(agg4#322), avg(agg5#323), avg(agg6#324), avg(agg7#325)] Aggregate Attributes [7]: [avg(agg1#319)#354, avg(agg2#320)#355, avg(agg3#321)#356, avg(agg4#322)#357, avg(agg5#323)#358, avg(agg6#324)#359, avg(agg7#325)#360] Results [11]: [null AS i_item_id#361, null AS ca_country#362, null AS ca_state#363, null AS county#364, avg(agg1#319)#354 AS agg1#365, avg(agg2#320)#355 AS agg2#366, avg(agg3#321)#356 AS agg3#367, avg(agg4#322)#357 AS agg4#368, avg(agg5#323)#358 AS agg5#369, avg(agg6#324)#359 AS agg6#370, avg(agg7#325)#360 AS agg7#371] -(156) Union +(146) Union -(157) TakeOrderedAndProject +(147) TakeOrderedAndProject Input [11]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, agg1#71, agg2#72, agg3#73, agg4#74, agg5#75, agg6#76, agg7#77] Arguments: 100, [ca_country#24 ASC NULLS FIRST, ca_state#23 ASC NULLS FIRST, ca_county#22 ASC NULLS FIRST, i_item_id#28 ASC NULLS FIRST], [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, agg1#71, agg2#72, agg3#73, agg4#74, agg5#75, agg6#76, agg7#77] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#9 IN dynamicpruning#10 -BroadcastExchange (162) -+- * ColumnarToRow (161) - +- CometProject (160) - +- CometFilter (159) - +- CometScan parquet spark_catalog.default.date_dim (158) +BroadcastExchange (152) ++- * ColumnarToRow (151) + +- CometProject (150) + +- CometFilter (149) + +- CometScan parquet spark_catalog.default.date_dim (148) -(158) Scan parquet spark_catalog.default.date_dim +(148) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#25, d_year#26] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(159) CometFilter +(149) CometFilter Input [2]: [d_date_sk#25, d_year#26] Condition : ((isnotnull(d_year#26) AND (d_year#26 = 2001)) AND isnotnull(d_date_sk#25)) -(160) CometProject +(150) CometProject Input [2]: [d_date_sk#25, d_year#26] Arguments: [d_date_sk#25], [d_date_sk#25] -(161) ColumnarToRow [codegen id : 1] +(151) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#25] -(162) BroadcastExchange +(152) BroadcastExchange Input [1]: [d_date_sk#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] -Subquery:2 Hosting operator id = 42 Hosting Expression = cs_sold_date_sk#86 IN dynamicpruning#10 +Subquery:2 Hosting operator id = 40 Hosting Expression = cs_sold_date_sk#86 IN dynamicpruning#10 -Subquery:3 Hosting operator id = 70 Hosting Expression = cs_sold_date_sk#159 IN dynamicpruning#10 +Subquery:3 Hosting operator id = 66 Hosting Expression = cs_sold_date_sk#159 IN dynamicpruning#10 -Subquery:4 Hosting operator id = 99 Hosting Expression = cs_sold_date_sk#233 IN dynamicpruning#10 +Subquery:4 Hosting operator id = 93 Hosting Expression = cs_sold_date_sk#233 IN dynamicpruning#10 -Subquery:5 Hosting operator id = 128 Hosting Expression = cs_sold_date_sk#307 IN dynamicpruning#10 +Subquery:5 Hosting operator id = 120 Hosting Expression = cs_sold_date_sk#307 IN dynamicpruning#10 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt index f904dd231..509d4be06 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q18a/simplified.txt @@ -2,188 +2,178 @@ TakeOrderedAndProject [ca_country,ca_state,ca_county,i_item_id,agg1,agg2,agg3,ag Union WholeStageCodegen (2) HashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,ca_country,ca_state,ca_county] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk,cd_dep_count] #3 - CometProject [cd_demo_sk,cd_dep_count] - CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] - CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] - CometBroadcastExchange [cd_demo_sk] #5 - CometFilter [cd_demo_sk] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] - CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #6 - CometFilter [ca_address_sk,ca_county,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] - CometBroadcastExchange [d_date_sk] #7 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_item_id] #8 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + Exchange [i_item_id,ca_country,ca_state,ca_county] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_county,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk,cd_dep_count] #3 + CometProject [cd_demo_sk,cd_dep_count] + CometFilter [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_education_status,cd_dep_count] + CometBroadcastExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + CometProject [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_month,c_birth_year] + CometBroadcastExchange [cd_demo_sk] #5 + CometFilter [cd_demo_sk] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk] + CometBroadcastExchange [ca_address_sk,ca_county,ca_state,ca_country] #6 + CometFilter [ca_address_sk,ca_county,ca_state,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_county,ca_state,ca_country] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_item_id] #8 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] WholeStageCodegen (4) HashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,ca_country,ca_state] #9 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_state,ca_country] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #10 - CometFilter [ca_address_sk,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + InputAdapter + Exchange [i_item_id,ca_country,ca_state] #9 + WholeStageCodegen (3) + HashAggregate [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_state,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_state,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [cd_demo_sk,cd_dep_count] #3 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + ReusedExchange [cd_demo_sk] #5 + CometBroadcastExchange [ca_address_sk,ca_state,ca_country] #10 + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + ReusedExchange [d_date_sk] #7 + ReusedExchange [i_item_sk,i_item_id] #8 WholeStageCodegen (6) HashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,ca_country] #11 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_country] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk,ca_country] #12 - CometProject [ca_address_sk,ca_country] - CometFilter [ca_address_sk,ca_state,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + InputAdapter + Exchange [i_item_id,ca_country] #11 + WholeStageCodegen (5) + HashAggregate [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,ca_country] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk,ca_country] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [cd_demo_sk,cd_dep_count] #3 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + ReusedExchange [cd_demo_sk] #5 + CometBroadcastExchange [ca_address_sk,ca_country] #12 + CometProject [ca_address_sk,ca_country] + CometFilter [ca_address_sk,ca_state,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_country] + ReusedExchange [d_date_sk] #7 + ReusedExchange [i_item_sk,i_item_id] #8 WholeStageCodegen (8) HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #13 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - CometBroadcastExchange [ca_address_sk] #14 - CometProject [ca_address_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - ReusedExchange [d_date_sk] #7 - ReusedExchange [i_item_sk,i_item_id] #8 + InputAdapter + Exchange [i_item_id] #13 + WholeStageCodegen (7) + HashAggregate [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [cd_demo_sk,cd_dep_count] #3 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + ReusedExchange [cd_demo_sk] #5 + CometBroadcastExchange [ca_address_sk] #14 + CometProject [ca_address_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + ReusedExchange [d_date_sk] #7 + ReusedExchange [i_item_sk,i_item_id] #8 WholeStageCodegen (10) HashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7),i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #15 - RowToColumnar - WholeStageCodegen (9) - HashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] - CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] - CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk,cd_dep_count] #3 - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 - ReusedExchange [cd_demo_sk] #5 - ReusedExchange [ca_address_sk] #14 - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [i_item_sk] #16 - CometFilter [i_item_sk] - CometScan parquet spark_catalog.default.item [i_item_sk] + InputAdapter + Exchange #15 + WholeStageCodegen (9) + HashAggregate [agg1,agg2,agg3,agg4,agg5,agg6,agg7] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year,ca_address_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year,cd_demo_sk] + CometProject [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count,c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_dep_count] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk,cd_demo_sk,cd_dep_count] + CometFilter [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_bill_cdemo_sk,cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [cd_demo_sk,cd_dep_count] #3 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,c_birth_year] #4 + ReusedExchange [cd_demo_sk] #5 + ReusedExchange [ca_address_sk] #14 + ReusedExchange [d_date_sk] #7 + CometBroadcastExchange [i_item_sk] #16 + CometFilter [i_item_sk] + CometScan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt index ebbb95b41..6619cc9d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/explain.txt @@ -1,30 +1,26 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * Project (25) - +- Window (24) - +- * ColumnarToRow (23) - +- CometSort (22) - +- CometColumnarExchange (21) - +- RowToColumnar (20) - +- * HashAggregate (19) - +- * ColumnarToRow (18) - +- CometColumnarExchange (17) - +- RowToColumnar (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) +TakeOrderedAndProject (22) ++- * Project (21) + +- Window (20) + +- * Sort (19) + +- Exchange (18) + +- * HashAggregate (17) + +- Exchange (16) + +- * HashAggregate (15) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometBroadcastHashJoin (12) + :- CometProject (7) + : +- CometBroadcastHashJoin (6) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : +- CometBroadcastExchange (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (11) + +- CometProject (10) + +- CometFilter (9) + +- CometScan parquet spark_catalog.default.date_dim (8) (1) Scan parquet spark_catalog.default.catalog_sales @@ -101,78 +97,66 @@ Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] Aggregate Attributes [1]: [sum#13] Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -(16) RowToColumnar +(16) Exchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 2] -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] - -(19) HashAggregate [codegen id : 2] +(17) HashAggregate [codegen id : 2] Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_ext_sales_price#2))#15] Results [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#15,17,2) AS itemrevenue#16, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#2))#15,17,2) AS _w0#17] -(20) RowToColumnar -Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] - -(21) CometColumnarExchange -Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(22) CometSort +(18) Exchange Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] -Arguments: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17], [i_class#9 ASC NULLS FIRST] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(23) ColumnarToRow [codegen id : 3] +(19) Sort [codegen id : 3] Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] +Arguments: [i_class#9 ASC NULLS FIRST], false, 0 -(24) Window +(20) Window Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] Arguments: [sum(_w0#17) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#9] -(25) Project [codegen id : 4] +(21) Project [codegen id : 4] Output [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, _we0#18] -(26) TakeOrderedAndProject +(22) TakeOrderedAndProject Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] Arguments: 100, [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST], [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * ColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.date_dim (27) +BroadcastExchange (27) ++- * ColumnarToRow (26) + +- CometProject (25) + +- CometFilter (24) + +- CometScan parquet spark_catalog.default.date_dim (23) -(27) Scan parquet spark_catalog.default.date_dim +(23) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(24) CometFilter Input [2]: [d_date_sk#11, d_date#12] Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 1999-02-22)) AND (d_date#12 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(29) CometProject +(25) CometProject Input [2]: [d_date_sk#11, d_date#12] Arguments: [d_date_sk#11], [d_date_sk#11] -(30) ColumnarToRow [codegen id : 1] +(26) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#11] -(31) BroadcastExchange +(27) BroadcastExchange Input [1]: [d_date_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt index 8bb081a5c..cd6bcfd61 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q20/simplified.txt @@ -4,39 +4,35 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c InputAdapter Window [_w0,i_class] WholeStageCodegen (3) - ColumnarToRow + Sort [i_class] InputAdapter - CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometColumnarExchange [i_class] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + Exchange [i_class] #1 + WholeStageCodegen (2) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(cs_ext_sales_price)),itemrevenue,_w0,sum] + InputAdapter + Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,cs_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_ext_sales_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #4 + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt index 1975ac486..2810779ed 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/explain.txt @@ -1,30 +1,28 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometColumnarExchange (23) - +- RowToColumnar (22) - +- * HashAggregate (21) - +- * Expand (20) - +- * Project (19) - +- * BroadcastNestedLoopJoin Inner BuildRight (18) - :- * ColumnarToRow (14) - : +- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.inventory (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.item (9) - +- BroadcastExchange (17) - +- * ColumnarToRow (16) - +- CometScan parquet spark_catalog.default.warehouse (15) +TakeOrderedAndProject (24) ++- * HashAggregate (23) + +- Exchange (22) + +- * HashAggregate (21) + +- * Expand (20) + +- * Project (19) + +- * BroadcastNestedLoopJoin Inner BuildRight (18) + :- * ColumnarToRow (14) + : +- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.item (9) + +- BroadcastExchange (17) + +- * ColumnarToRow (16) + +- CometScan parquet spark_catalog.default.warehouse (15) (1) Scan parquet spark_catalog.default.inventory @@ -126,56 +124,50 @@ Functions [1]: [partial_avg(inv_quantity_on_hand#2)] Aggregate Attributes [2]: [sum#17, count#18] Results [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] -(22) RowToColumnar +(22) Exchange Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] +Arguments: hashpartitioning(i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(23) CometColumnarExchange -Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] -Arguments: hashpartitioning(i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(24) ColumnarToRow [codegen id : 3] -Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] - -(25) HashAggregate [codegen id : 3] +(23) HashAggregate [codegen id : 3] Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] Keys [5]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16] Functions [1]: [avg(inv_quantity_on_hand#2)] Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#21] Results [5]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, avg(inv_quantity_on_hand#2)#21 AS qoh#22] -(26) TakeOrderedAndProject +(24) TakeOrderedAndProject Input [5]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, qoh#22] Arguments: 100, [qoh#22 ASC NULLS FIRST, i_product_name#12 ASC NULLS FIRST, i_brand#13 ASC NULLS FIRST, i_class#14 ASC NULLS FIRST, i_category#15 ASC NULLS FIRST], [i_product_name#12, i_brand#13, i_class#14, i_category#15, qoh#22] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * ColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.date_dim (27) +BroadcastExchange (29) ++- * ColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.date_dim (25) -(27) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) -(29) CometProject +(27) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(30) ColumnarToRow [codegen id : 1] +(28) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(31) BroadcastExchange +(29) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt index f5b281ac6..0e864ab70 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22/simplified.txt @@ -1,41 +1,39 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] WholeStageCodegen (3) HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastNestedLoopJoin - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.warehouse + InputAdapter + Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + WholeStageCodegen (2) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] + Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + BroadcastNestedLoopJoin + ColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometScan parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt index cc3ef34d5..ac15878d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/explain.txt @@ -1,64 +1,50 @@ == Physical Plan == -TakeOrderedAndProject (60) -+- Union (59) - :- * HashAggregate (26) - : +- * HashAggregate (25) - : +- * HashAggregate (24) - : +- * ColumnarToRow (23) - : +- CometColumnarExchange (22) - : +- RowToColumnar (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.inventory (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.item (9) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.warehouse (14) +TakeOrderedAndProject (46) ++- Union (45) + :- * HashAggregate (24) + : +- * HashAggregate (23) + : +- * HashAggregate (22) + : +- Exchange (21) + : +- * HashAggregate (20) + : +- * ColumnarToRow (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan parquet spark_catalog.default.warehouse (14) + :- * HashAggregate (29) + : +- Exchange (28) + : +- * HashAggregate (27) + : +- * HashAggregate (26) + : +- ReusedExchange (25) :- * HashAggregate (34) - : +- * ColumnarToRow (33) - : +- CometColumnarExchange (32) - : +- RowToColumnar (31) - : +- * HashAggregate (30) - : +- * HashAggregate (29) - : +- * ColumnarToRow (28) - : +- ReusedExchange (27) - :- * HashAggregate (42) - : +- * ColumnarToRow (41) - : +- CometColumnarExchange (40) - : +- RowToColumnar (39) - : +- * HashAggregate (38) - : +- * HashAggregate (37) - : +- * ColumnarToRow (36) - : +- ReusedExchange (35) - :- * HashAggregate (50) - : +- * ColumnarToRow (49) - : +- CometColumnarExchange (48) - : +- RowToColumnar (47) - : +- * HashAggregate (46) - : +- * HashAggregate (45) - : +- * ColumnarToRow (44) - : +- ReusedExchange (43) - +- * HashAggregate (58) - +- * ColumnarToRow (57) - +- CometColumnarExchange (56) - +- RowToColumnar (55) - +- * HashAggregate (54) - +- * HashAggregate (53) - +- * ColumnarToRow (52) - +- ReusedExchange (51) + : +- Exchange (33) + : +- * HashAggregate (32) + : +- * HashAggregate (31) + : +- ReusedExchange (30) + :- * HashAggregate (39) + : +- Exchange (38) + : +- * HashAggregate (37) + : +- * HashAggregate (36) + : +- ReusedExchange (35) + +- * HashAggregate (44) + +- Exchange (43) + +- * HashAggregate (42) + +- * HashAggregate (41) + +- ReusedExchange (40) (1) Scan parquet spark_catalog.default.inventory @@ -159,220 +145,178 @@ Functions [1]: [partial_avg(inv_quantity_on_hand#3)] Aggregate Attributes [2]: [sum#14, count#15] Results [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#16, count#17] -(21) RowToColumnar +(21) Exchange Input [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#16, count#17] +Arguments: hashpartitioning(i_product_name#12, i_brand#9, i_class#10, i_category#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(22) CometColumnarExchange -Input [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#16, count#17] -Arguments: hashpartitioning(i_product_name#12, i_brand#9, i_class#10, i_category#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(23) ColumnarToRow [codegen id : 2] -Input [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#16, count#17] - -(24) HashAggregate [codegen id : 2] +(22) HashAggregate [codegen id : 2] Input [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#16, count#17] Keys [4]: [i_product_name#12, i_brand#9, i_class#10, i_category#11] Functions [1]: [avg(inv_quantity_on_hand#3)] Aggregate Attributes [1]: [avg(inv_quantity_on_hand#3)#18] Results [5]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, avg(inv_quantity_on_hand#3)#18 AS qoh#19] -(25) HashAggregate [codegen id : 2] +(23) HashAggregate [codegen id : 2] Input [5]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, qoh#19] Keys [4]: [i_product_name#12, i_brand#9, i_class#10, i_category#11] Functions [1]: [partial_avg(qoh#19)] Aggregate Attributes [2]: [sum#20, count#21] Results [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#22, count#23] -(26) HashAggregate [codegen id : 2] +(24) HashAggregate [codegen id : 2] Input [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#22, count#23] Keys [4]: [i_product_name#12, i_brand#9, i_class#10, i_category#11] Functions [1]: [avg(qoh#19)] Aggregate Attributes [1]: [avg(qoh#19)#24] Results [5]: [i_product_name#12 AS i_product_name#25, i_brand#9 AS i_brand#26, i_class#10 AS i_class#27, i_category#11 AS i_category#28, avg(qoh#19)#24 AS qoh#29] -(27) ReusedExchange [Reuses operator id: 22] +(25) ReusedExchange [Reuses operator id: 21] Output [6]: [i_product_name#30, i_brand#31, i_class#32, i_category#33, sum#34, count#35] -(28) ColumnarToRow [codegen id : 4] -Input [6]: [i_product_name#30, i_brand#31, i_class#32, i_category#33, sum#34, count#35] - -(29) HashAggregate [codegen id : 4] +(26) HashAggregate [codegen id : 4] Input [6]: [i_product_name#30, i_brand#31, i_class#32, i_category#33, sum#34, count#35] Keys [4]: [i_product_name#30, i_brand#31, i_class#32, i_category#33] Functions [1]: [avg(inv_quantity_on_hand#36)] Aggregate Attributes [1]: [avg(inv_quantity_on_hand#36)#18] Results [4]: [i_product_name#30, i_brand#31, i_class#32, avg(inv_quantity_on_hand#36)#18 AS qoh#37] -(30) HashAggregate [codegen id : 4] +(27) HashAggregate [codegen id : 4] Input [4]: [i_product_name#30, i_brand#31, i_class#32, qoh#37] Keys [3]: [i_product_name#30, i_brand#31, i_class#32] Functions [1]: [partial_avg(qoh#37)] Aggregate Attributes [2]: [sum#38, count#39] Results [5]: [i_product_name#30, i_brand#31, i_class#32, sum#40, count#41] -(31) RowToColumnar +(28) Exchange Input [5]: [i_product_name#30, i_brand#31, i_class#32, sum#40, count#41] +Arguments: hashpartitioning(i_product_name#30, i_brand#31, i_class#32, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(32) CometColumnarExchange -Input [5]: [i_product_name#30, i_brand#31, i_class#32, sum#40, count#41] -Arguments: hashpartitioning(i_product_name#30, i_brand#31, i_class#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(33) ColumnarToRow [codegen id : 5] -Input [5]: [i_product_name#30, i_brand#31, i_class#32, sum#40, count#41] - -(34) HashAggregate [codegen id : 5] +(29) HashAggregate [codegen id : 5] Input [5]: [i_product_name#30, i_brand#31, i_class#32, sum#40, count#41] Keys [3]: [i_product_name#30, i_brand#31, i_class#32] Functions [1]: [avg(qoh#37)] Aggregate Attributes [1]: [avg(qoh#37)#42] Results [5]: [i_product_name#30, i_brand#31, i_class#32, null AS i_category#43, avg(qoh#37)#42 AS qoh#44] -(35) ReusedExchange [Reuses operator id: 22] +(30) ReusedExchange [Reuses operator id: 21] Output [6]: [i_product_name#45, i_brand#46, i_class#47, i_category#48, sum#49, count#50] -(36) ColumnarToRow [codegen id : 7] -Input [6]: [i_product_name#45, i_brand#46, i_class#47, i_category#48, sum#49, count#50] - -(37) HashAggregate [codegen id : 7] +(31) HashAggregate [codegen id : 7] Input [6]: [i_product_name#45, i_brand#46, i_class#47, i_category#48, sum#49, count#50] Keys [4]: [i_product_name#45, i_brand#46, i_class#47, i_category#48] Functions [1]: [avg(inv_quantity_on_hand#51)] Aggregate Attributes [1]: [avg(inv_quantity_on_hand#51)#18] Results [3]: [i_product_name#45, i_brand#46, avg(inv_quantity_on_hand#51)#18 AS qoh#52] -(38) HashAggregate [codegen id : 7] +(32) HashAggregate [codegen id : 7] Input [3]: [i_product_name#45, i_brand#46, qoh#52] Keys [2]: [i_product_name#45, i_brand#46] Functions [1]: [partial_avg(qoh#52)] Aggregate Attributes [2]: [sum#53, count#54] Results [4]: [i_product_name#45, i_brand#46, sum#55, count#56] -(39) RowToColumnar -Input [4]: [i_product_name#45, i_brand#46, sum#55, count#56] - -(40) CometColumnarExchange -Input [4]: [i_product_name#45, i_brand#46, sum#55, count#56] -Arguments: hashpartitioning(i_product_name#45, i_brand#46, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(41) ColumnarToRow [codegen id : 8] +(33) Exchange Input [4]: [i_product_name#45, i_brand#46, sum#55, count#56] +Arguments: hashpartitioning(i_product_name#45, i_brand#46, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(42) HashAggregate [codegen id : 8] +(34) HashAggregate [codegen id : 8] Input [4]: [i_product_name#45, i_brand#46, sum#55, count#56] Keys [2]: [i_product_name#45, i_brand#46] Functions [1]: [avg(qoh#52)] Aggregate Attributes [1]: [avg(qoh#52)#57] Results [5]: [i_product_name#45, i_brand#46, null AS i_class#58, null AS i_category#59, avg(qoh#52)#57 AS qoh#60] -(43) ReusedExchange [Reuses operator id: 22] +(35) ReusedExchange [Reuses operator id: 21] Output [6]: [i_product_name#61, i_brand#62, i_class#63, i_category#64, sum#65, count#66] -(44) ColumnarToRow [codegen id : 10] -Input [6]: [i_product_name#61, i_brand#62, i_class#63, i_category#64, sum#65, count#66] - -(45) HashAggregate [codegen id : 10] +(36) HashAggregate [codegen id : 10] Input [6]: [i_product_name#61, i_brand#62, i_class#63, i_category#64, sum#65, count#66] Keys [4]: [i_product_name#61, i_brand#62, i_class#63, i_category#64] Functions [1]: [avg(inv_quantity_on_hand#67)] Aggregate Attributes [1]: [avg(inv_quantity_on_hand#67)#18] Results [2]: [i_product_name#61, avg(inv_quantity_on_hand#67)#18 AS qoh#68] -(46) HashAggregate [codegen id : 10] +(37) HashAggregate [codegen id : 10] Input [2]: [i_product_name#61, qoh#68] Keys [1]: [i_product_name#61] Functions [1]: [partial_avg(qoh#68)] Aggregate Attributes [2]: [sum#69, count#70] Results [3]: [i_product_name#61, sum#71, count#72] -(47) RowToColumnar -Input [3]: [i_product_name#61, sum#71, count#72] - -(48) CometColumnarExchange +(38) Exchange Input [3]: [i_product_name#61, sum#71, count#72] -Arguments: hashpartitioning(i_product_name#61, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_product_name#61, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(49) ColumnarToRow [codegen id : 11] -Input [3]: [i_product_name#61, sum#71, count#72] - -(50) HashAggregate [codegen id : 11] +(39) HashAggregate [codegen id : 11] Input [3]: [i_product_name#61, sum#71, count#72] Keys [1]: [i_product_name#61] Functions [1]: [avg(qoh#68)] Aggregate Attributes [1]: [avg(qoh#68)#73] Results [5]: [i_product_name#61, null AS i_brand#74, null AS i_class#75, null AS i_category#76, avg(qoh#68)#73 AS qoh#77] -(51) ReusedExchange [Reuses operator id: 22] +(40) ReusedExchange [Reuses operator id: 21] Output [6]: [i_product_name#78, i_brand#79, i_class#80, i_category#81, sum#82, count#83] -(52) ColumnarToRow [codegen id : 13] -Input [6]: [i_product_name#78, i_brand#79, i_class#80, i_category#81, sum#82, count#83] - -(53) HashAggregate [codegen id : 13] +(41) HashAggregate [codegen id : 13] Input [6]: [i_product_name#78, i_brand#79, i_class#80, i_category#81, sum#82, count#83] Keys [4]: [i_product_name#78, i_brand#79, i_class#80, i_category#81] Functions [1]: [avg(inv_quantity_on_hand#84)] Aggregate Attributes [1]: [avg(inv_quantity_on_hand#84)#18] Results [1]: [avg(inv_quantity_on_hand#84)#18 AS qoh#85] -(54) HashAggregate [codegen id : 13] +(42) HashAggregate [codegen id : 13] Input [1]: [qoh#85] Keys: [] Functions [1]: [partial_avg(qoh#85)] Aggregate Attributes [2]: [sum#86, count#87] Results [2]: [sum#88, count#89] -(55) RowToColumnar -Input [2]: [sum#88, count#89] - -(56) CometColumnarExchange -Input [2]: [sum#88, count#89] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(57) ColumnarToRow [codegen id : 14] +(43) Exchange Input [2]: [sum#88, count#89] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=5] -(58) HashAggregate [codegen id : 14] +(44) HashAggregate [codegen id : 14] Input [2]: [sum#88, count#89] Keys: [] Functions [1]: [avg(qoh#85)] Aggregate Attributes [1]: [avg(qoh#85)#90] Results [5]: [null AS i_product_name#91, null AS i_brand#92, null AS i_class#93, null AS i_category#94, avg(qoh#85)#90 AS qoh#95] -(59) Union +(45) Union -(60) TakeOrderedAndProject +(46) TakeOrderedAndProject Input [5]: [i_product_name#25, i_brand#26, i_class#27, i_category#28, qoh#29] Arguments: 100, [qoh#29 ASC NULLS FIRST, i_product_name#25 ASC NULLS FIRST, i_brand#26 ASC NULLS FIRST, i_class#27 ASC NULLS FIRST, i_category#28 ASC NULLS FIRST], [i_product_name#25, i_brand#26, i_class#27, i_category#28, qoh#29] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (65) -+- * ColumnarToRow (64) - +- CometProject (63) - +- CometFilter (62) - +- CometScan parquet spark_catalog.default.date_dim (61) +BroadcastExchange (51) ++- * ColumnarToRow (50) + +- CometProject (49) + +- CometFilter (48) + +- CometScan parquet spark_catalog.default.date_dim (47) -(61) Scan parquet spark_catalog.default.date_dim +(47) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(62) CometFilter +(48) CometFilter Input [2]: [d_date_sk#6, d_month_seq#7] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#6)) -(63) CometProject +(49) CometProject Input [2]: [d_date_sk#6, d_month_seq#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(64) ColumnarToRow [codegen id : 1] +(50) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(65) BroadcastExchange +(51) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt index 9f82911a1..fc4778a22 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q22a/simplified.txt @@ -4,85 +4,71 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(qoh),i_product_name,i_brand,i_class,i_category,qoh,sum,count] HashAggregate [i_product_name,i_brand,i_class,i_category,qoh] [sum,count,sum,count] HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_product_name,i_brand,i_class,i_category] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_product_name,i_brand,i_class,i_category,inv_quantity_on_hand] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] - CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometBroadcastExchange [w_warehouse_sk] #5 - CometFilter [w_warehouse_sk] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk] + InputAdapter + Exchange [i_product_name,i_brand,i_class,i_category] #1 + WholeStageCodegen (1) + HashAggregate [i_product_name,i_brand,i_class,i_category,inv_quantity_on_hand] [sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] + CometProject [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometBroadcastExchange [w_warehouse_sk] #5 + CometFilter [w_warehouse_sk] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk] WholeStageCodegen (5) HashAggregate [i_product_name,i_brand,i_class,sum,count] [avg(qoh),i_category,qoh,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_product_name,i_brand,i_class] #6 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [i_product_name,i_brand,i_class,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - ColumnarToRow - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + InputAdapter + Exchange [i_product_name,i_brand,i_class] #6 + WholeStageCodegen (4) + HashAggregate [i_product_name,i_brand,i_class,qoh] [sum,count,sum,count] + HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] + InputAdapter + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 WholeStageCodegen (8) HashAggregate [i_product_name,i_brand,sum,count] [avg(qoh),i_class,i_category,qoh,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_product_name,i_brand] #7 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [i_product_name,i_brand,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - ColumnarToRow - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + InputAdapter + Exchange [i_product_name,i_brand] #7 + WholeStageCodegen (7) + HashAggregate [i_product_name,i_brand,qoh] [sum,count,sum,count] + HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] + InputAdapter + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 WholeStageCodegen (11) HashAggregate [i_product_name,sum,count] [avg(qoh),i_brand,i_class,i_category,qoh,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_product_name] #8 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [i_product_name,qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - ColumnarToRow - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + InputAdapter + Exchange [i_product_name] #8 + WholeStageCodegen (10) + HashAggregate [i_product_name,qoh] [sum,count,sum,count] + HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] + InputAdapter + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 WholeStageCodegen (14) HashAggregate [sum,count] [avg(qoh),i_product_name,i_brand,i_class,i_category,qoh,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #9 - RowToColumnar - WholeStageCodegen (13) - HashAggregate [qoh] [sum,count,sum,count] - HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - ColumnarToRow - InputAdapter - ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 + InputAdapter + Exchange #9 + WholeStageCodegen (13) + HashAggregate [qoh] [sum,count,sum,count] + HashAggregate [i_product_name,i_brand,i_class,i_category,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] + InputAdapter + ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt index 118902e57..9dd9f17b3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/explain.txt @@ -1,54 +1,50 @@ == Physical Plan == -* ColumnarToRow (50) -+- CometSort (49) - +- CometColumnarExchange (48) - +- RowToColumnar (47) - +- * Filter (46) - +- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometColumnarExchange (38) - +- RowToColumnar (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) +* ColumnarToRow (46) ++- CometSort (45) + +- CometColumnarExchange (44) + +- RowToColumnar (43) + +- * Filter (42) + +- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (21) + : : +- CometFilter (20) + : : +- CometScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.customer (24) + +- BroadcastExchange (33) + +- * ColumnarToRow (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.customer_address (30) (1) Scan parquet spark_catalog.default.store_sales @@ -66,9 +62,9 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(4) CometColumnarExchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -89,9 +85,9 @@ Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -220,230 +216,202 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#30] Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -(37) RowToColumnar +(37) Exchange Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(38) CometColumnarExchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(39) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] - -(40) HashAggregate [codegen id : 3] +(38) HashAggregate [codegen id : 3] Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, netpaid#33] Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] Functions [1]: [partial_sum(netpaid#33)] Aggregate Attributes [2]: [sum#34, isEmpty#35] Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -(42) RowToColumnar -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] - -(43) CometColumnarExchange -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] Functions [1]: [sum(netpaid#33)] Aggregate Attributes [1]: [sum(netpaid#33)#38] Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, sum(netpaid#33)#38 AS paid#39] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subquery scalar-subquery#40, [id=#41] as decimal(33,8)))) -(47) RowToColumnar +(43) RowToColumnar Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -(48) CometColumnarExchange +(44) CometColumnarExchange Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(49) CometSort +(45) CometSort Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] Arguments: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] -(50) ColumnarToRow [codegen id : 5] +(46) ColumnarToRow [codegen id : 5] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#40, [id=#41] -* HashAggregate (81) -+- * ColumnarToRow (80) - +- CometColumnarExchange (79) - +- RowToColumnar (78) - +- * HashAggregate (77) - +- * HashAggregate (76) - +- * ColumnarToRow (75) - +- CometColumnarExchange (74) - +- RowToColumnar (73) - +- * HashAggregate (72) - +- * Project (71) - +- * BroadcastHashJoin Inner BuildRight (70) - :- * ColumnarToRow (68) - : +- CometProject (67) - : +- CometBroadcastHashJoin (66) - : :- CometProject (64) - : : +- CometBroadcastHashJoin (63) - : : :- CometProject (59) - : : : +- CometBroadcastHashJoin (58) - : : : :- CometProject (56) - : : : : +- CometSortMergeJoin (55) - : : : : :- CometSort (52) - : : : : : +- ReusedExchange (51) - : : : : +- CometSort (54) - : : : : +- ReusedExchange (53) - : : : +- ReusedExchange (57) - : : +- CometBroadcastExchange (62) - : : +- CometFilter (61) - : : +- CometScan parquet spark_catalog.default.item (60) - : +- ReusedExchange (65) - +- ReusedExchange (69) - - -(51) ReusedExchange [Reuses operator id: 4] +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#40, [id=#41] +* HashAggregate (73) ++- Exchange (72) + +- * HashAggregate (71) + +- * HashAggregate (70) + +- Exchange (69) + +- * HashAggregate (68) + +- * Project (67) + +- * BroadcastHashJoin Inner BuildRight (66) + :- * ColumnarToRow (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (55) + : : : +- CometBroadcastHashJoin (54) + : : : :- CometProject (52) + : : : : +- CometSortMergeJoin (51) + : : : : :- CometSort (48) + : : : : : +- ReusedExchange (47) + : : : : +- CometSort (50) + : : : : +- ReusedExchange (49) + : : : +- ReusedExchange (53) + : : +- CometBroadcastExchange (58) + : : +- CometFilter (57) + : : +- CometScan parquet spark_catalog.default.item (56) + : +- ReusedExchange (61) + +- ReusedExchange (65) + + +(47) ReusedExchange [Reuses operator id: 4] Output [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] -(52) CometSort +(48) CometSort Input [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46], [ss_ticket_number#45 ASC NULLS FIRST, ss_item_sk#42 ASC NULLS FIRST] -(53) ReusedExchange [Reuses operator id: 9] +(49) ReusedExchange [Reuses operator id: 9] Output [2]: [sr_item_sk#47, sr_ticket_number#48] -(54) CometSort +(50) CometSort Input [2]: [sr_item_sk#47, sr_ticket_number#48] Arguments: [sr_item_sk#47, sr_ticket_number#48], [sr_ticket_number#48 ASC NULLS FIRST, sr_item_sk#47 ASC NULLS FIRST] -(55) CometSortMergeJoin +(51) CometSortMergeJoin Left output [5]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46] Right output [2]: [sr_item_sk#47, sr_ticket_number#48] Arguments: [ss_ticket_number#45, ss_item_sk#42], [sr_ticket_number#48, sr_item_sk#47], Inner -(56) CometProject +(52) CometProject Input [7]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_ticket_number#45, ss_net_paid#46, sr_item_sk#47, sr_ticket_number#48] Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46], [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46] -(57) ReusedExchange [Reuses operator id: 16] +(53) ReusedExchange [Reuses operator id: 16] Output [4]: [s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] -(58) CometBroadcastHashJoin +(54) CometBroadcastHashJoin Left output [4]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46] Right output [4]: [s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] Arguments: [ss_store_sk#44], [s_store_sk#49], Inner, BuildRight -(59) CometProject +(55) CometProject Input [8]: [ss_item_sk#42, ss_customer_sk#43, ss_store_sk#44, ss_net_paid#46, s_store_sk#49, s_store_name#50, s_state#51, s_zip#52] Arguments: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52], [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52] -(60) Scan parquet spark_catalog.default.item +(56) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(61) CometFilter +(57) CometFilter Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Condition : isnotnull(i_item_sk#53) -(62) CometBroadcastExchange +(58) CometBroadcastExchange Input [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Arguments: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -(63) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [6]: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52] Right output [6]: [i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Arguments: [ss_item_sk#42], [i_item_sk#53], Inner, BuildRight -(64) CometProject +(60) CometProject Input [12]: [ss_item_sk#42, ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_item_sk#53, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Arguments: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58], [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] -(65) ReusedExchange [Reuses operator id: 26] +(61) ReusedExchange [Reuses operator id: 26] Output [5]: [c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -(66) CometBroadcastHashJoin +(62) CometBroadcastHashJoin Left output [10]: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58] Right output [5]: [c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] Arguments: [ss_customer_sk#43], [c_customer_sk#59], Inner, BuildRight -(67) CometProject +(63) CometProject Input [15]: [ss_customer_sk#43, ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_customer_sk#59, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] Arguments: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63], [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -(68) ColumnarToRow [codegen id : 2] +(64) ColumnarToRow [codegen id : 2] Input [13]: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63] -(69) ReusedExchange [Reuses operator id: 33] +(65) ReusedExchange [Reuses operator id: 33] Output [4]: [ca_address_sk#64, ca_state#65, ca_zip#66, ca_country#67] -(70) BroadcastHashJoin [codegen id : 2] +(66) BroadcastHashJoin [codegen id : 2] Left keys [3]: [c_current_addr_sk#60, c_birth_country#63, s_zip#52] Right keys [3]: [ca_address_sk#64, upper(ca_country#67), ca_zip#66] Join type: Inner Join condition: None -(71) Project [codegen id : 2] +(67) Project [codegen id : 2] Output [11]: [ss_net_paid#46, s_store_name#50, s_state#51, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_first_name#61, c_last_name#62, ca_state#65] Input [17]: [ss_net_paid#46, s_store_name#50, s_state#51, s_zip#52, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_current_addr_sk#60, c_first_name#61, c_last_name#62, c_birth_country#63, ca_address_sk#64, ca_state#65, ca_zip#66, ca_country#67] -(72) HashAggregate [codegen id : 2] +(68) HashAggregate [codegen id : 2] Input [11]: [ss_net_paid#46, s_store_name#50, s_state#51, i_current_price#54, i_size#55, i_color#56, i_units#57, i_manager_id#58, c_first_name#61, c_last_name#62, ca_state#65] Keys [10]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#46))] Aggregate Attributes [1]: [sum#68] Results [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] -(73) RowToColumnar +(69) Exchange Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] +Arguments: hashpartitioning(c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(74) CometColumnarExchange -Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] -Arguments: hashpartitioning(c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(75) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] - -(76) HashAggregate [codegen id : 3] +(70) HashAggregate [codegen id : 3] Input [11]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55, sum#69] Keys [10]: [c_last_name#62, c_first_name#61, s_store_name#50, ca_state#65, s_state#51, i_color#56, i_current_price#54, i_manager_id#58, i_units#57, i_size#55] Functions [1]: [sum(UnscaledValue(ss_net_paid#46))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#46))#32] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#46))#32,17,2) AS netpaid#70] -(77) HashAggregate [codegen id : 3] +(71) HashAggregate [codegen id : 3] Input [1]: [netpaid#70] Keys: [] Functions [1]: [partial_avg(netpaid#70)] Aggregate Attributes [2]: [sum#71, count#72] Results [2]: [sum#73, count#74] -(78) RowToColumnar -Input [2]: [sum#73, count#74] - -(79) CometColumnarExchange -Input [2]: [sum#73, count#74] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(80) ColumnarToRow [codegen id : 4] +(72) Exchange Input [2]: [sum#73, count#74] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] -(81) HashAggregate [codegen id : 4] +(73) HashAggregate [codegen id : 4] Input [2]: [sum#73, count#74] Keys: [] Functions [1]: [avg(netpaid#70)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt index 42c283018..1e6a1a1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q24/simplified.txt @@ -9,92 +9,84 @@ WholeStageCodegen (5) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #10 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #5 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - InputAdapter - ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name] #2 - RowToColumnar + InputAdapter + Exchange #10 WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [netpaid] [sum,count,sum,count] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #4 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #5 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #5 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + InputAdapter + ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name] #2 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #4 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #5 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (1) + ColumnarToRow InputAdapter - BroadcastExchange #9 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] + CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt index c6190b825..6c4d257c2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/explain.txt @@ -1,81 +1,75 @@ == Physical Plan == -TakeOrderedAndProject (77) -+- Union (76) - :- * HashAggregate (30) - : +- * ColumnarToRow (29) - : +- CometColumnarExchange (28) - : +- RowToColumnar (27) - : +- * HashAggregate (26) - : +- * ColumnarToRow (25) - : +- CometProject (24) - : +- CometBroadcastHashJoin (23) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometProject (14) - : : : +- CometBroadcastHashJoin (13) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) - : : : +- CometBroadcastExchange (12) - : : : +- CometProject (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.date_dim (9) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.store (15) - : +- CometBroadcastExchange (22) - : +- CometFilter (21) - : +- CometScan parquet spark_catalog.default.item (20) - :- * HashAggregate (53) - : +- * ColumnarToRow (52) - : +- CometColumnarExchange (51) - : +- RowToColumnar (50) - : +- * HashAggregate (49) - : +- * ColumnarToRow (48) - : +- CometProject (47) - : +- CometBroadcastHashJoin (46) - : :- CometProject (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometProject (38) - : : : +- CometBroadcastHashJoin (37) - : : : :- CometProject (35) - : : : : +- CometBroadcastHashJoin (34) - : : : : :- CometFilter (32) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (31) - : : : : +- ReusedExchange (33) - : : : +- ReusedExchange (36) - : : +- CometBroadcastExchange (42) - : : +- CometProject (41) - : : +- CometFilter (40) - : : +- CometScan parquet spark_catalog.default.store (39) - : +- ReusedExchange (45) - +- * HashAggregate (75) - +- * ColumnarToRow (74) - +- CometColumnarExchange (73) - +- RowToColumnar (72) - +- * HashAggregate (71) - +- * ColumnarToRow (70) - +- CometProject (69) - +- CometBroadcastHashJoin (68) - :- CometProject (64) - : +- CometBroadcastHashJoin (63) - : :- CometProject (61) - : : +- CometBroadcastHashJoin (60) - : : :- CometProject (58) - : : : +- CometBroadcastHashJoin (57) - : : : :- CometFilter (55) - : : : : +- CometScan parquet spark_catalog.default.store_sales (54) - : : : +- ReusedExchange (56) - : : +- ReusedExchange (59) - : +- ReusedExchange (62) - +- CometBroadcastExchange (67) - +- CometFilter (66) - +- CometScan parquet spark_catalog.default.item (65) +TakeOrderedAndProject (71) ++- Union (70) + :- * HashAggregate (28) + : +- Exchange (27) + : +- * HashAggregate (26) + : +- * ColumnarToRow (25) + : +- CometProject (24) + : +- CometBroadcastHashJoin (23) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometProject (14) + : : : +- CometBroadcastHashJoin (13) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.customer_demographics (3) + : : : +- CometBroadcastExchange (12) + : : : +- CometProject (11) + : : : +- CometFilter (10) + : : : +- CometScan parquet spark_catalog.default.date_dim (9) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometScan parquet spark_catalog.default.store (15) + : +- CometBroadcastExchange (22) + : +- CometFilter (21) + : +- CometScan parquet spark_catalog.default.item (20) + :- * HashAggregate (49) + : +- Exchange (48) + : +- * HashAggregate (47) + : +- * ColumnarToRow (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (42) + : : +- CometBroadcastHashJoin (41) + : : :- CometProject (36) + : : : +- CometBroadcastHashJoin (35) + : : : :- CometProject (33) + : : : : +- CometBroadcastHashJoin (32) + : : : : :- CometFilter (30) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (29) + : : : : +- ReusedExchange (31) + : : : +- ReusedExchange (34) + : : +- CometBroadcastExchange (40) + : : +- CometProject (39) + : : +- CometFilter (38) + : : +- CometScan parquet spark_catalog.default.store (37) + : +- ReusedExchange (43) + +- * HashAggregate (69) + +- Exchange (68) + +- * HashAggregate (67) + +- * ColumnarToRow (66) + +- CometProject (65) + +- CometBroadcastHashJoin (64) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (57) + : : +- CometBroadcastHashJoin (56) + : : :- CometProject (54) + : : : +- CometBroadcastHashJoin (53) + : : : :- CometFilter (51) + : : : : +- CometScan parquet spark_catalog.default.store_sales (50) + : : : +- ReusedExchange (52) + : : +- ReusedExchange (55) + : +- ReusedExchange (58) + +- CometBroadcastExchange (63) + +- CometFilter (62) + +- CometScan parquet spark_catalog.default.item (61) (1) Scan parquet spark_catalog.default.store_sales @@ -204,24 +198,18 @@ Functions [4]: [partial_avg(agg1#20), partial_avg(UnscaledValue(agg2#21)), parti Aggregate Attributes [8]: [sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] Results [10]: [i_item_id#19, s_state#17, sum#32, count#33, sum#34, count#35, sum#36, count#37, sum#38, count#39] -(27) RowToColumnar +(27) Exchange Input [10]: [i_item_id#19, s_state#17, sum#32, count#33, sum#34, count#35, sum#36, count#37, sum#38, count#39] +Arguments: hashpartitioning(i_item_id#19, s_state#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(28) CometColumnarExchange -Input [10]: [i_item_id#19, s_state#17, sum#32, count#33, sum#34, count#35, sum#36, count#37, sum#38, count#39] -Arguments: hashpartitioning(i_item_id#19, s_state#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(29) ColumnarToRow [codegen id : 2] -Input [10]: [i_item_id#19, s_state#17, sum#32, count#33, sum#34, count#35, sum#36, count#37, sum#38, count#39] - -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [10]: [i_item_id#19, s_state#17, sum#32, count#33, sum#34, count#35, sum#36, count#37, sum#38, count#39] Keys [2]: [i_item_id#19, s_state#17] Functions [4]: [avg(agg1#20), avg(UnscaledValue(agg2#21)), avg(UnscaledValue(agg3#22)), avg(UnscaledValue(agg4#23))] Aggregate Attributes [4]: [avg(agg1#20)#40, avg(UnscaledValue(agg2#21))#41, avg(UnscaledValue(agg3#22))#42, avg(UnscaledValue(agg4#23))#43] Results [7]: [i_item_id#19, s_state#17, 0 AS g_state#44, avg(agg1#20)#40 AS agg1#45, cast((avg(UnscaledValue(agg2#21))#41 / 100.0) as decimal(11,6)) AS agg2#46, cast((avg(UnscaledValue(agg3#22))#42 / 100.0) as decimal(11,6)) AS agg3#47, cast((avg(UnscaledValue(agg4#23))#43 / 100.0) as decimal(11,6)) AS agg4#48] -(31) Scan parquet spark_catalog.default.store_sales +(29) Scan parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#49, ss_cdemo_sk#50, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55, ss_sold_date_sk#56] Batched: true Location: InMemoryFileIndex [] @@ -229,102 +217,96 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#56), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(32) CometFilter +(30) CometFilter Input [8]: [ss_item_sk#49, ss_cdemo_sk#50, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55, ss_sold_date_sk#56] Condition : ((isnotnull(ss_cdemo_sk#50) AND isnotnull(ss_store_sk#51)) AND isnotnull(ss_item_sk#49)) -(33) ReusedExchange [Reuses operator id: 6] +(31) ReusedExchange [Reuses operator id: 6] Output [1]: [cd_demo_sk#58] -(34) CometBroadcastHashJoin +(32) CometBroadcastHashJoin Left output [8]: [ss_item_sk#49, ss_cdemo_sk#50, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55, ss_sold_date_sk#56] Right output [1]: [cd_demo_sk#58] Arguments: [ss_cdemo_sk#50], [cd_demo_sk#58], Inner, BuildRight -(35) CometProject +(33) CometProject Input [9]: [ss_item_sk#49, ss_cdemo_sk#50, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55, ss_sold_date_sk#56, cd_demo_sk#58] Arguments: [ss_item_sk#49, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55, ss_sold_date_sk#56], [ss_item_sk#49, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55, ss_sold_date_sk#56] -(36) ReusedExchange [Reuses operator id: 12] +(34) ReusedExchange [Reuses operator id: 12] Output [1]: [d_date_sk#59] -(37) CometBroadcastHashJoin +(35) CometBroadcastHashJoin Left output [7]: [ss_item_sk#49, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55, ss_sold_date_sk#56] Right output [1]: [d_date_sk#59] Arguments: [ss_sold_date_sk#56], [d_date_sk#59], Inner, BuildRight -(38) CometProject +(36) CometProject Input [8]: [ss_item_sk#49, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55, ss_sold_date_sk#56, d_date_sk#59] Arguments: [ss_item_sk#49, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55], [ss_item_sk#49, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55] -(39) Scan parquet spark_catalog.default.store +(37) Scan parquet spark_catalog.default.store Output [2]: [s_store_sk#60, s_state#61] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_state), EqualTo(s_state,TN), IsNotNull(s_store_sk)] ReadSchema: struct -(40) CometFilter +(38) CometFilter Input [2]: [s_store_sk#60, s_state#61] Condition : ((isnotnull(s_state#61) AND (s_state#61 = TN)) AND isnotnull(s_store_sk#60)) -(41) CometProject +(39) CometProject Input [2]: [s_store_sk#60, s_state#61] Arguments: [s_store_sk#60], [s_store_sk#60] -(42) CometBroadcastExchange +(40) CometBroadcastExchange Input [1]: [s_store_sk#60] Arguments: [s_store_sk#60] -(43) CometBroadcastHashJoin +(41) CometBroadcastHashJoin Left output [6]: [ss_item_sk#49, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55] Right output [1]: [s_store_sk#60] Arguments: [ss_store_sk#51], [s_store_sk#60], Inner, BuildRight -(44) CometProject +(42) CometProject Input [7]: [ss_item_sk#49, ss_store_sk#51, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55, s_store_sk#60] Arguments: [ss_item_sk#49, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55], [ss_item_sk#49, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55] -(45) ReusedExchange [Reuses operator id: 22] +(43) ReusedExchange [Reuses operator id: 22] Output [2]: [i_item_sk#62, i_item_id#63] -(46) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [5]: [ss_item_sk#49, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55] Right output [2]: [i_item_sk#62, i_item_id#63] Arguments: [ss_item_sk#49], [i_item_sk#62], Inner, BuildRight -(47) CometProject +(45) CometProject Input [7]: [ss_item_sk#49, ss_quantity#52, ss_list_price#53, ss_sales_price#54, ss_coupon_amt#55, i_item_sk#62, i_item_id#63] Arguments: [i_item_id#63, agg1#64, agg2#65, agg3#66, agg4#67], [i_item_id#63, ss_quantity#52 AS agg1#64, ss_list_price#53 AS agg2#65, ss_coupon_amt#55 AS agg3#66, ss_sales_price#54 AS agg4#67] -(48) ColumnarToRow [codegen id : 3] +(46) ColumnarToRow [codegen id : 3] Input [5]: [i_item_id#63, agg1#64, agg2#65, agg3#66, agg4#67] -(49) HashAggregate [codegen id : 3] +(47) HashAggregate [codegen id : 3] Input [5]: [i_item_id#63, agg1#64, agg2#65, agg3#66, agg4#67] Keys [1]: [i_item_id#63] Functions [4]: [partial_avg(agg1#64), partial_avg(UnscaledValue(agg2#65)), partial_avg(UnscaledValue(agg3#66)), partial_avg(UnscaledValue(agg4#67))] Aggregate Attributes [8]: [sum#68, count#69, sum#70, count#71, sum#72, count#73, sum#74, count#75] Results [9]: [i_item_id#63, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] -(50) RowToColumnar -Input [9]: [i_item_id#63, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] - -(51) CometColumnarExchange +(48) Exchange Input [9]: [i_item_id#63, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] -Arguments: hashpartitioning(i_item_id#63, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#63, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(52) ColumnarToRow [codegen id : 4] -Input [9]: [i_item_id#63, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] - -(53) HashAggregate [codegen id : 4] +(49) HashAggregate [codegen id : 4] Input [9]: [i_item_id#63, sum#76, count#77, sum#78, count#79, sum#80, count#81, sum#82, count#83] Keys [1]: [i_item_id#63] Functions [4]: [avg(agg1#64), avg(UnscaledValue(agg2#65)), avg(UnscaledValue(agg3#66)), avg(UnscaledValue(agg4#67))] Aggregate Attributes [4]: [avg(agg1#64)#84, avg(UnscaledValue(agg2#65))#85, avg(UnscaledValue(agg3#66))#86, avg(UnscaledValue(agg4#67))#87] Results [7]: [i_item_id#63, null AS s_state#88, 1 AS g_state#89, avg(agg1#64)#84 AS agg1#90, cast((avg(UnscaledValue(agg2#65))#85 / 100.0) as decimal(11,6)) AS agg2#91, cast((avg(UnscaledValue(agg3#66))#86 / 100.0) as decimal(11,6)) AS agg3#92, cast((avg(UnscaledValue(agg4#67))#87 / 100.0) as decimal(11,6)) AS agg4#93] -(54) Scan parquet spark_catalog.default.store_sales +(50) Scan parquet spark_catalog.default.store_sales Output [8]: [ss_item_sk#94, ss_cdemo_sk#95, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100, ss_sold_date_sk#101] Batched: true Location: InMemoryFileIndex [] @@ -332,137 +314,131 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#101), dynamicpruningexpression(ss_s PushedFilters: [IsNotNull(ss_cdemo_sk), IsNotNull(ss_store_sk), IsNotNull(ss_item_sk)] ReadSchema: struct -(55) CometFilter +(51) CometFilter Input [8]: [ss_item_sk#94, ss_cdemo_sk#95, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100, ss_sold_date_sk#101] Condition : ((isnotnull(ss_cdemo_sk#95) AND isnotnull(ss_store_sk#96)) AND isnotnull(ss_item_sk#94)) -(56) ReusedExchange [Reuses operator id: 6] +(52) ReusedExchange [Reuses operator id: 6] Output [1]: [cd_demo_sk#103] -(57) CometBroadcastHashJoin +(53) CometBroadcastHashJoin Left output [8]: [ss_item_sk#94, ss_cdemo_sk#95, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100, ss_sold_date_sk#101] Right output [1]: [cd_demo_sk#103] Arguments: [ss_cdemo_sk#95], [cd_demo_sk#103], Inner, BuildRight -(58) CometProject +(54) CometProject Input [9]: [ss_item_sk#94, ss_cdemo_sk#95, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100, ss_sold_date_sk#101, cd_demo_sk#103] Arguments: [ss_item_sk#94, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100, ss_sold_date_sk#101], [ss_item_sk#94, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100, ss_sold_date_sk#101] -(59) ReusedExchange [Reuses operator id: 12] +(55) ReusedExchange [Reuses operator id: 12] Output [1]: [d_date_sk#104] -(60) CometBroadcastHashJoin +(56) CometBroadcastHashJoin Left output [7]: [ss_item_sk#94, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100, ss_sold_date_sk#101] Right output [1]: [d_date_sk#104] Arguments: [ss_sold_date_sk#101], [d_date_sk#104], Inner, BuildRight -(61) CometProject +(57) CometProject Input [8]: [ss_item_sk#94, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100, ss_sold_date_sk#101, d_date_sk#104] Arguments: [ss_item_sk#94, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100], [ss_item_sk#94, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100] -(62) ReusedExchange [Reuses operator id: 42] +(58) ReusedExchange [Reuses operator id: 40] Output [1]: [s_store_sk#105] -(63) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [6]: [ss_item_sk#94, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100] Right output [1]: [s_store_sk#105] Arguments: [ss_store_sk#96], [s_store_sk#105], Inner, BuildRight -(64) CometProject +(60) CometProject Input [7]: [ss_item_sk#94, ss_store_sk#96, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100, s_store_sk#105] Arguments: [ss_item_sk#94, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100], [ss_item_sk#94, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100] -(65) Scan parquet spark_catalog.default.item +(61) Scan parquet spark_catalog.default.item Output [1]: [i_item_sk#106] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(66) CometFilter +(62) CometFilter Input [1]: [i_item_sk#106] Condition : isnotnull(i_item_sk#106) -(67) CometBroadcastExchange +(63) CometBroadcastExchange Input [1]: [i_item_sk#106] Arguments: [i_item_sk#106] -(68) CometBroadcastHashJoin +(64) CometBroadcastHashJoin Left output [5]: [ss_item_sk#94, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100] Right output [1]: [i_item_sk#106] Arguments: [ss_item_sk#94], [i_item_sk#106], Inner, BuildRight -(69) CometProject +(65) CometProject Input [6]: [ss_item_sk#94, ss_quantity#97, ss_list_price#98, ss_sales_price#99, ss_coupon_amt#100, i_item_sk#106] Arguments: [agg1#107, agg2#108, agg3#109, agg4#110], [ss_quantity#97 AS agg1#107, ss_list_price#98 AS agg2#108, ss_coupon_amt#100 AS agg3#109, ss_sales_price#99 AS agg4#110] -(70) ColumnarToRow [codegen id : 5] +(66) ColumnarToRow [codegen id : 5] Input [4]: [agg1#107, agg2#108, agg3#109, agg4#110] -(71) HashAggregate [codegen id : 5] +(67) HashAggregate [codegen id : 5] Input [4]: [agg1#107, agg2#108, agg3#109, agg4#110] Keys: [] Functions [4]: [partial_avg(agg1#107), partial_avg(UnscaledValue(agg2#108)), partial_avg(UnscaledValue(agg3#109)), partial_avg(UnscaledValue(agg4#110))] Aggregate Attributes [8]: [sum#111, count#112, sum#113, count#114, sum#115, count#116, sum#117, count#118] Results [8]: [sum#119, count#120, sum#121, count#122, sum#123, count#124, sum#125, count#126] -(72) RowToColumnar -Input [8]: [sum#119, count#120, sum#121, count#122, sum#123, count#124, sum#125, count#126] - -(73) CometColumnarExchange -Input [8]: [sum#119, count#120, sum#121, count#122, sum#123, count#124, sum#125, count#126] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(74) ColumnarToRow [codegen id : 6] +(68) Exchange Input [8]: [sum#119, count#120, sum#121, count#122, sum#123, count#124, sum#125, count#126] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] -(75) HashAggregate [codegen id : 6] +(69) HashAggregate [codegen id : 6] Input [8]: [sum#119, count#120, sum#121, count#122, sum#123, count#124, sum#125, count#126] Keys: [] Functions [4]: [avg(agg1#107), avg(UnscaledValue(agg2#108)), avg(UnscaledValue(agg3#109)), avg(UnscaledValue(agg4#110))] Aggregate Attributes [4]: [avg(agg1#107)#127, avg(UnscaledValue(agg2#108))#128, avg(UnscaledValue(agg3#109))#129, avg(UnscaledValue(agg4#110))#130] Results [7]: [null AS i_item_id#131, null AS s_state#132, 1 AS g_state#133, avg(agg1#107)#127 AS agg1#134, cast((avg(UnscaledValue(agg2#108))#128 / 100.0) as decimal(11,6)) AS agg2#135, cast((avg(UnscaledValue(agg3#109))#129 / 100.0) as decimal(11,6)) AS agg3#136, cast((avg(UnscaledValue(agg4#110))#130 / 100.0) as decimal(11,6)) AS agg4#137] -(76) Union +(70) Union -(77) TakeOrderedAndProject +(71) TakeOrderedAndProject Input [7]: [i_item_id#19, s_state#17, g_state#44, agg1#45, agg2#46, agg3#47, agg4#48] Arguments: 100, [i_item_id#19 ASC NULLS FIRST, s_state#17 ASC NULLS FIRST], [i_item_id#19, s_state#17, g_state#44, agg1#45, agg2#46, agg3#47, agg4#48] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (82) -+- * ColumnarToRow (81) - +- CometProject (80) - +- CometFilter (79) - +- CometScan parquet spark_catalog.default.date_dim (78) +BroadcastExchange (76) ++- * ColumnarToRow (75) + +- CometProject (74) + +- CometFilter (73) + +- CometScan parquet spark_catalog.default.date_dim (72) -(78) Scan parquet spark_catalog.default.date_dim +(72) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_year#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1998), IsNotNull(d_date_sk)] ReadSchema: struct -(79) CometFilter +(73) CometFilter Input [2]: [d_date_sk#14, d_year#15] Condition : ((isnotnull(d_year#15) AND (d_year#15 = 1998)) AND isnotnull(d_date_sk#14)) -(80) CometProject +(74) CometProject Input [2]: [d_date_sk#14, d_year#15] Arguments: [d_date_sk#14], [d_date_sk#14] -(81) ColumnarToRow [codegen id : 1] +(75) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(82) BroadcastExchange +(76) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] -Subquery:2 Hosting operator id = 31 Hosting Expression = ss_sold_date_sk#56 IN dynamicpruning#9 +Subquery:2 Hosting operator id = 29 Hosting Expression = ss_sold_date_sk#56 IN dynamicpruning#9 -Subquery:3 Hosting operator id = 54 Hosting Expression = ss_sold_date_sk#101 IN dynamicpruning#9 +Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#101 IN dynamicpruning#9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt index fd4d4f82e..50e246e15 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q27a/simplified.txt @@ -2,98 +2,92 @@ TakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] Union WholeStageCodegen (2) HashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,s_state] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,s_state,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [cd_demo_sk] #3 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [s_store_sk,s_state] #5 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - CometBroadcastExchange [i_item_sk,i_item_id] #6 - CometFilter [i_item_sk,i_item_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] + InputAdapter + Exchange [i_item_id,s_state] #1 + WholeStageCodegen (1) + HashAggregate [i_item_id,s_state,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk,s_state] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [cd_demo_sk] #3 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_education_status] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [s_store_sk,s_state] #5 + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + CometBroadcastExchange [i_item_sk,i_item_id] #6 + CometFilter [i_item_sk,i_item_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] WholeStageCodegen (4) HashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id] #7 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [i_item_id,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] - CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk] #3 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [s_store_sk] #8 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [i_item_sk,i_item_id] #6 + InputAdapter + Exchange [i_item_id] #7 + WholeStageCodegen (3) + HashAggregate [i_item_id,agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [cd_demo_sk] #3 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [s_store_sk] #8 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [i_item_sk,i_item_id] #6 WholeStageCodegen (6) HashAggregate [sum,count,sum,count,sum,count,sum,count] [avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4)),i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #9 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] - CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] - CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [cd_demo_sk] #3 - ReusedExchange [d_date_sk] #4 - ReusedExchange [s_store_sk] #8 - CometBroadcastExchange [i_item_sk] #10 - CometFilter [i_item_sk] - CometScan parquet spark_catalog.default.item [i_item_sk] + InputAdapter + Exchange #9 + WholeStageCodegen (5) + HashAggregate [agg1,agg2,agg3,agg4] [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] + CometProject [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_store_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk,cd_demo_sk] + CometFilter [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_cdemo_sk,ss_store_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [cd_demo_sk] #3 + ReusedExchange [d_date_sk] #4 + ReusedExchange [s_store_sk] #8 + CometBroadcastExchange [i_item_sk] #10 + CometFilter [i_item_sk] + CometScan parquet spark_catalog.default.item [i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt index d978de885..a8800a70f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/explain.txt @@ -6,7 +6,7 @@ +- CometBroadcastHashJoin (28) :- CometFilter (24) : +- CometHashAggregate (23) - : +- CometColumnarExchange (22) + : +- CometExchange (22) : +- CometHashAggregate (21) : +- CometProject (20) : +- CometBroadcastHashJoin (19) @@ -134,9 +134,9 @@ Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt index fafc612d6..7da6076f5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q34/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometColumnarExchange [ss_ticket_number,ss_customer_sk] #2 + CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt index 721036bfb..790000085 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/explain.txt @@ -1,50 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- Exchange (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * ColumnarToRow (31) + : +- CometFilter (30) + : +- CometScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometFilter (36) + +- CometScan parquet spark_catalog.default.customer_demographics (35) (1) Scan parquet spark_catalog.default.customer @@ -240,56 +238,50 @@ Functions [10]: [partial_count(1), partial_avg(cd_dep_count#25), partial_max(cd_ Aggregate Attributes [13]: [count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] Results [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] -(42) RowToColumnar +(42) Exchange Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) CometColumnarExchange -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] -Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 6] -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] - -(45) HashAggregate [codegen id : 6] +(43) HashAggregate [codegen id : 6] Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] Keys [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [10]: [count(1), avg(cd_dep_count#25), max(cd_dep_count#25), sum(cd_dep_count#25), avg(cd_dep_employed_count#26), max(cd_dep_employed_count#26), sum(cd_dep_employed_count#26), avg(cd_dep_college_count#27), max(cd_dep_college_count#27), sum(cd_dep_college_count#27)] Aggregate Attributes [10]: [count(1)#54, avg(cd_dep_count#25)#55, max(cd_dep_count#25)#56, sum(cd_dep_count#25)#57, avg(cd_dep_employed_count#26)#58, max(cd_dep_employed_count#26)#59, sum(cd_dep_employed_count#26)#60, avg(cd_dep_college_count#27)#61, max(cd_dep_college_count#27)#62, sum(cd_dep_college_count#27)#63] Results [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, count(1)#54 AS cnt1#64, avg(cd_dep_count#25)#55 AS avg(cd_dep_count)#65, max(cd_dep_count#25)#56 AS max(cd_dep_count)#66, sum(cd_dep_count#25)#57 AS sum(cd_dep_count)#67, cd_dep_employed_count#26, count(1)#54 AS cnt2#68, avg(cd_dep_employed_count#26)#58 AS avg(cd_dep_employed_count)#69, max(cd_dep_employed_count#26)#59 AS max(cd_dep_employed_count)#70, sum(cd_dep_employed_count#26)#60 AS sum(cd_dep_employed_count)#71, cd_dep_college_count#27, count(1)#54 AS cnt3#72, avg(cd_dep_college_count#27)#61 AS avg(cd_dep_college_count)#73, max(cd_dep_college_count#27)#62 AS max(cd_dep_college_count)#74, sum(cd_dep_college_count#27)#63 AS sum(cd_dep_college_count)#75] -(46) TakeOrderedAndProject +(44) TakeOrderedAndProject Input [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] Arguments: 100, [ca_state#21 ASC NULLS FIRST, cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (51) -+- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.date_dim (47) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan parquet spark_catalog.default.date_dim (45) -(47) Scan parquet spark_catalog.default.date_dim +(45) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(48) CometFilter +(46) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) -(49) CometProject +(47) CometProject Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Arguments: [d_date_sk#9], [d_date_sk#9] -(50) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(51) BroadcastExchange +(49) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt index 221648657..0b55e23ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35/simplified.txt @@ -1,72 +1,70 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + InputAdapter + Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt index 675f2ac4b..8eafc64ff 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/explain.txt @@ -1,44 +1,42 @@ == Physical Plan == -TakeOrderedAndProject (40) -+- * HashAggregate (39) - +- * ColumnarToRow (38) - +- CometColumnarExchange (37) - +- RowToColumnar (36) - +- * HashAggregate (35) - +- * ColumnarToRow (34) - +- CometProject (33) - +- CometBroadcastHashJoin (32) - :- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometBroadcastHashJoin (11) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : +- CometBroadcastExchange (10) - : : : +- CometProject (9) - : : : +- CometBroadcastHashJoin (8) - : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (7) - : : : +- CometProject (6) - : : : +- CometFilter (5) - : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : +- CometBroadcastExchange (21) - : : +- CometUnion (20) - : : :- CometProject (15) - : : : +- CometBroadcastHashJoin (14) - : : : :- CometScan parquet spark_catalog.default.web_sales (12) - : : : +- ReusedExchange (13) - : : +- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometScan parquet spark_catalog.default.catalog_sales (16) - : : +- ReusedExchange (17) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer_address (24) - +- CometBroadcastExchange (31) - +- CometFilter (30) - +- CometScan parquet spark_catalog.default.customer_demographics (29) +TakeOrderedAndProject (38) ++- * HashAggregate (37) + +- Exchange (36) + +- * HashAggregate (35) + +- * ColumnarToRow (34) + +- CometProject (33) + +- CometBroadcastHashJoin (32) + :- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometBroadcastHashJoin (11) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : +- CometBroadcastExchange (10) + : : : +- CometProject (9) + : : : +- CometBroadcastHashJoin (8) + : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (7) + : : : +- CometProject (6) + : : : +- CometFilter (5) + : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : +- CometBroadcastExchange (21) + : : +- CometUnion (20) + : : :- CometProject (15) + : : : +- CometBroadcastHashJoin (14) + : : : :- CometScan parquet spark_catalog.default.web_sales (12) + : : : +- ReusedExchange (13) + : : +- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometScan parquet spark_catalog.default.catalog_sales (16) + : : +- ReusedExchange (17) + : +- CometBroadcastExchange (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.customer_address (24) + +- CometBroadcastExchange (31) + +- CometFilter (30) + +- CometScan parquet spark_catalog.default.customer_demographics (29) (1) Scan parquet spark_catalog.default.customer @@ -209,56 +207,50 @@ Functions [10]: [partial_count(1), partial_avg(cd_dep_count#25), partial_max(cd_ Aggregate Attributes [13]: [count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] Results [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] -(36) RowToColumnar +(36) Exchange Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(37) CometColumnarExchange -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] -Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(38) ColumnarToRow [codegen id : 2] -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] - -(39) HashAggregate [codegen id : 2] +(37) HashAggregate [codegen id : 2] Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] Keys [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [10]: [count(1), avg(cd_dep_count#25), max(cd_dep_count#25), sum(cd_dep_count#25), avg(cd_dep_employed_count#26), max(cd_dep_employed_count#26), sum(cd_dep_employed_count#26), avg(cd_dep_college_count#27), max(cd_dep_college_count#27), sum(cd_dep_college_count#27)] Aggregate Attributes [10]: [count(1)#54, avg(cd_dep_count#25)#55, max(cd_dep_count#25)#56, sum(cd_dep_count#25)#57, avg(cd_dep_employed_count#26)#58, max(cd_dep_employed_count#26)#59, sum(cd_dep_employed_count#26)#60, avg(cd_dep_college_count#27)#61, max(cd_dep_college_count#27)#62, sum(cd_dep_college_count#27)#63] Results [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, count(1)#54 AS cnt1#64, avg(cd_dep_count#25)#55 AS avg(cd_dep_count)#65, max(cd_dep_count#25)#56 AS max(cd_dep_count)#66, sum(cd_dep_count#25)#57 AS sum(cd_dep_count)#67, cd_dep_employed_count#26, count(1)#54 AS cnt2#68, avg(cd_dep_employed_count#26)#58 AS avg(cd_dep_employed_count)#69, max(cd_dep_employed_count#26)#59 AS max(cd_dep_employed_count)#70, sum(cd_dep_employed_count#26)#60 AS sum(cd_dep_employed_count)#71, cd_dep_college_count#27, count(1)#54 AS cnt3#72, avg(cd_dep_college_count#27)#61 AS avg(cd_dep_college_count)#73, max(cd_dep_college_count#27)#62 AS max(cd_dep_college_count)#74, sum(cd_dep_college_count#27)#63 AS sum(cd_dep_college_count)#75] -(40) TakeOrderedAndProject +(38) TakeOrderedAndProject Input [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] Arguments: 100, [ca_state#21 ASC NULLS FIRST, cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (45) -+- * ColumnarToRow (44) - +- CometProject (43) - +- CometFilter (42) - +- CometScan parquet spark_catalog.default.date_dim (41) +BroadcastExchange (43) ++- * ColumnarToRow (42) + +- CometProject (41) + +- CometFilter (40) + +- CometScan parquet spark_catalog.default.date_dim (39) -(41) Scan parquet spark_catalog.default.date_dim +(39) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#7, d_year#8, d_qoy#9] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,1999), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(42) CometFilter +(40) CometFilter Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] Condition : ((((isnotnull(d_year#8) AND isnotnull(d_qoy#9)) AND (d_year#8 = 1999)) AND (d_qoy#9 < 4)) AND isnotnull(d_date_sk#7)) -(43) CometProject +(41) CometProject Input [3]: [d_date_sk#7, d_year#8, d_qoy#9] Arguments: [d_date_sk#7], [d_date_sk#7] -(44) ColumnarToRow [codegen id : 1] +(42) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#7] -(45) BroadcastExchange +(43) BroadcastExchange Input [1]: [d_date_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=2] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt index c31cfd8da..79ead0464 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q35a/simplified.txt @@ -1,54 +1,52 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] WholeStageCodegen (2) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - ColumnarToRow - InputAdapter - CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometBroadcastHashJoin [c_current_cdemo_sk,ca_state,cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometProject [c_current_cdemo_sk,ca_state] - CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk,ca_state] - CometProject [c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [customsk] #5 - CometUnion [customsk] - CometProject [ws_bill_customer_sk] [customsk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometProject [cs_ship_customer_sk] [customsk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - CometBroadcastExchange [ca_address_sk,ca_state] #6 - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + InputAdapter + Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (1) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + ColumnarToRow + InputAdapter + CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometBroadcastHashJoin [c_current_cdemo_sk,ca_state,cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometProject [c_current_cdemo_sk,ca_state] + CometBroadcastHashJoin [c_current_cdemo_sk,c_current_addr_sk,ca_address_sk,ca_state] + CometProject [c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,customsk] + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [customsk] #5 + CometUnion [customsk] + CometProject [ws_bill_customer_sk] [customsk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + CometProject [cs_ship_customer_sk] [customsk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + CometBroadcastExchange [ca_address_sk,ca_state] #6 + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #7 + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt index df24de6b8..7b3ae5c67 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/explain.txt @@ -1,56 +1,46 @@ == Physical Plan == -TakeOrderedAndProject (52) -+- * Project (51) - +- Window (50) - +- * ColumnarToRow (49) - +- CometSort (48) - +- CometColumnarExchange (47) - +- CometHashAggregate (46) - +- CometColumnarExchange (45) - +- RowToColumnar (44) - +- * HashAggregate (43) - +- Union (42) - :- * HashAggregate (25) - : +- * ColumnarToRow (24) - : +- CometColumnarExchange (23) - : +- RowToColumnar (22) - : +- * HashAggregate (21) - : +- * ColumnarToRow (20) - : +- CometProject (19) - : +- CometBroadcastHashJoin (18) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.item (9) - : +- CometBroadcastExchange (17) - : +- CometProject (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.store (14) - :- * HashAggregate (33) - : +- * ColumnarToRow (32) - : +- CometColumnarExchange (31) - : +- RowToColumnar (30) - : +- * HashAggregate (29) - : +- * HashAggregate (28) - : +- * ColumnarToRow (27) - : +- ReusedExchange (26) - +- * HashAggregate (41) - +- * ColumnarToRow (40) - +- CometColumnarExchange (39) - +- RowToColumnar (38) - +- * HashAggregate (37) - +- * HashAggregate (36) - +- * ColumnarToRow (35) - +- ReusedExchange (34) +TakeOrderedAndProject (42) ++- * Project (41) + +- Window (40) + +- * Sort (39) + +- Exchange (38) + +- * HashAggregate (37) + +- Exchange (36) + +- * HashAggregate (35) + +- Union (34) + :- * HashAggregate (23) + : +- Exchange (22) + : +- * HashAggregate (21) + : +- * ColumnarToRow (20) + : +- CometProject (19) + : +- CometBroadcastHashJoin (18) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.item (9) + : +- CometBroadcastExchange (17) + : +- CometProject (16) + : +- CometFilter (15) + : +- CometScan parquet spark_catalog.default.store (14) + :- * HashAggregate (28) + : +- Exchange (27) + : +- * HashAggregate (26) + : +- * HashAggregate (25) + : +- ReusedExchange (24) + +- * HashAggregate (33) + +- Exchange (32) + +- * HashAggregate (31) + +- * HashAggregate (30) + +- ReusedExchange (29) (1) Scan parquet spark_catalog.default.store_sales @@ -155,170 +145,142 @@ Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(Unscale Aggregate Attributes [2]: [sum#14, sum#15] Results [4]: [i_category#11, i_class#10, sum#16, sum#17] -(22) RowToColumnar +(22) Exchange Input [4]: [i_category#11, i_class#10, sum#16, sum#17] +Arguments: hashpartitioning(i_category#11, i_class#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(23) CometColumnarExchange -Input [4]: [i_category#11, i_class#10, sum#16, sum#17] -Arguments: hashpartitioning(i_category#11, i_class#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(24) ColumnarToRow [codegen id : 2] -Input [4]: [i_category#11, i_class#10, sum#16, sum#17] - -(25) HashAggregate [codegen id : 2] +(23) HashAggregate [codegen id : 2] Input [4]: [i_category#11, i_class#10, sum#16, sum#17] Keys [2]: [i_category#11, i_class#10] Functions [2]: [sum(UnscaledValue(ss_net_profit#4)), sum(UnscaledValue(ss_ext_sales_price#3))] Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#4))#18, sum(UnscaledValue(ss_ext_sales_price#3))#19] Results [6]: [cast((MakeDecimal(sum(UnscaledValue(ss_net_profit#4))#18,17,2) / MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#3))#19,17,2)) as decimal(38,11)) AS gross_margin#20, i_category#11, i_class#10, 0 AS t_category#21, 0 AS t_class#22, 0 AS lochierarchy#23] -(26) ReusedExchange [Reuses operator id: 23] +(24) ReusedExchange [Reuses operator id: 22] Output [4]: [i_category#24, i_class#25, sum#26, sum#27] -(27) ColumnarToRow [codegen id : 4] -Input [4]: [i_category#24, i_class#25, sum#26, sum#27] - -(28) HashAggregate [codegen id : 4] +(25) HashAggregate [codegen id : 4] Input [4]: [i_category#24, i_class#25, sum#26, sum#27] Keys [2]: [i_category#24, i_class#25] Functions [2]: [sum(UnscaledValue(ss_net_profit#28)), sum(UnscaledValue(ss_ext_sales_price#29))] Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#28))#30, sum(UnscaledValue(ss_ext_sales_price#29))#31] Results [3]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#28))#30,17,2) AS ss_net_profit#32, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#29))#31,17,2) AS ss_ext_sales_price#33, i_category#24] -(29) HashAggregate [codegen id : 4] +(26) HashAggregate [codegen id : 4] Input [3]: [ss_net_profit#32, ss_ext_sales_price#33, i_category#24] Keys [1]: [i_category#24] Functions [2]: [partial_sum(ss_net_profit#32), partial_sum(ss_ext_sales_price#33)] Aggregate Attributes [4]: [sum#34, isEmpty#35, sum#36, isEmpty#37] Results [5]: [i_category#24, sum#38, isEmpty#39, sum#40, isEmpty#41] -(30) RowToColumnar +(27) Exchange Input [5]: [i_category#24, sum#38, isEmpty#39, sum#40, isEmpty#41] +Arguments: hashpartitioning(i_category#24, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(31) CometColumnarExchange -Input [5]: [i_category#24, sum#38, isEmpty#39, sum#40, isEmpty#41] -Arguments: hashpartitioning(i_category#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(32) ColumnarToRow [codegen id : 5] -Input [5]: [i_category#24, sum#38, isEmpty#39, sum#40, isEmpty#41] - -(33) HashAggregate [codegen id : 5] +(28) HashAggregate [codegen id : 5] Input [5]: [i_category#24, sum#38, isEmpty#39, sum#40, isEmpty#41] Keys [1]: [i_category#24] Functions [2]: [sum(ss_net_profit#32), sum(ss_ext_sales_price#33)] Aggregate Attributes [2]: [sum(ss_net_profit#32)#42, sum(ss_ext_sales_price#33)#43] Results [6]: [(sum(ss_net_profit#32)#42 / sum(ss_ext_sales_price#33)#43) AS gross_margin#44, i_category#24, null AS i_class#45, 0 AS t_category#46, 1 AS t_class#47, 1 AS lochierarchy#48] -(34) ReusedExchange [Reuses operator id: 23] +(29) ReusedExchange [Reuses operator id: 22] Output [4]: [i_category#49, i_class#50, sum#51, sum#52] -(35) ColumnarToRow [codegen id : 7] -Input [4]: [i_category#49, i_class#50, sum#51, sum#52] - -(36) HashAggregate [codegen id : 7] +(30) HashAggregate [codegen id : 7] Input [4]: [i_category#49, i_class#50, sum#51, sum#52] Keys [2]: [i_category#49, i_class#50] Functions [2]: [sum(UnscaledValue(ss_net_profit#53)), sum(UnscaledValue(ss_ext_sales_price#54))] Aggregate Attributes [2]: [sum(UnscaledValue(ss_net_profit#53))#30, sum(UnscaledValue(ss_ext_sales_price#54))#31] Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#53))#30,17,2) AS ss_net_profit#55, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#54))#31,17,2) AS ss_ext_sales_price#56] -(37) HashAggregate [codegen id : 7] +(31) HashAggregate [codegen id : 7] Input [2]: [ss_net_profit#55, ss_ext_sales_price#56] Keys: [] Functions [2]: [partial_sum(ss_net_profit#55), partial_sum(ss_ext_sales_price#56)] Aggregate Attributes [4]: [sum#57, isEmpty#58, sum#59, isEmpty#60] Results [4]: [sum#61, isEmpty#62, sum#63, isEmpty#64] -(38) RowToColumnar -Input [4]: [sum#61, isEmpty#62, sum#63, isEmpty#64] - -(39) CometColumnarExchange -Input [4]: [sum#61, isEmpty#62, sum#63, isEmpty#64] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(40) ColumnarToRow [codegen id : 8] +(32) Exchange Input [4]: [sum#61, isEmpty#62, sum#63, isEmpty#64] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] -(41) HashAggregate [codegen id : 8] +(33) HashAggregate [codegen id : 8] Input [4]: [sum#61, isEmpty#62, sum#63, isEmpty#64] Keys: [] Functions [2]: [sum(ss_net_profit#55), sum(ss_ext_sales_price#56)] Aggregate Attributes [2]: [sum(ss_net_profit#55)#65, sum(ss_ext_sales_price#56)#66] Results [6]: [(sum(ss_net_profit#55)#65 / sum(ss_ext_sales_price#56)#66) AS gross_margin#67, null AS i_category#68, null AS i_class#69, 1 AS t_category#70, 1 AS t_class#71, 2 AS lochierarchy#72] -(42) Union +(34) Union -(43) HashAggregate [codegen id : 9] +(35) HashAggregate [codegen id : 9] Input [6]: [gross_margin#20, i_category#11, i_class#10, t_category#21, t_class#22, lochierarchy#23] Keys [6]: [gross_margin#20, i_category#11, i_class#10, t_category#21, t_class#22, lochierarchy#23] Functions: [] Aggregate Attributes: [] Results [6]: [gross_margin#20, i_category#11, i_class#10, t_category#21, t_class#22, lochierarchy#23] -(44) RowToColumnar +(36) Exchange Input [6]: [gross_margin#20, i_category#11, i_class#10, t_category#21, t_class#22, lochierarchy#23] +Arguments: hashpartitioning(gross_margin#20, i_category#11, i_class#10, t_category#21, t_class#22, lochierarchy#23, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(45) CometColumnarExchange -Input [6]: [gross_margin#20, i_category#11, i_class#10, t_category#21, t_class#22, lochierarchy#23] -Arguments: hashpartitioning(gross_margin#20, i_category#11, i_class#10, t_category#21, t_class#22, lochierarchy#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(46) CometHashAggregate +(37) HashAggregate [codegen id : 10] Input [6]: [gross_margin#20, i_category#11, i_class#10, t_category#21, t_class#22, lochierarchy#23] Keys [6]: [gross_margin#20, i_category#11, i_class#10, t_category#21, t_class#22, lochierarchy#23] Functions: [] +Aggregate Attributes: [] +Results [5]: [gross_margin#20, i_category#11, i_class#10, lochierarchy#23, CASE WHEN (t_class#22 = 0) THEN i_category#11 END AS _w0#73] -(47) CometColumnarExchange -Input [5]: [gross_margin#20, i_category#11, i_class#10, lochierarchy#23, _w0#73] -Arguments: hashpartitioning(lochierarchy#23, _w0#73, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(48) CometSort +(38) Exchange Input [5]: [gross_margin#20, i_category#11, i_class#10, lochierarchy#23, _w0#73] -Arguments: [gross_margin#20, i_category#11, i_class#10, lochierarchy#23, _w0#73], [lochierarchy#23 ASC NULLS FIRST, _w0#73 ASC NULLS FIRST, gross_margin#20 ASC NULLS FIRST] +Arguments: hashpartitioning(lochierarchy#23, _w0#73, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(49) ColumnarToRow [codegen id : 10] +(39) Sort [codegen id : 11] Input [5]: [gross_margin#20, i_category#11, i_class#10, lochierarchy#23, _w0#73] +Arguments: [lochierarchy#23 ASC NULLS FIRST, _w0#73 ASC NULLS FIRST, gross_margin#20 ASC NULLS FIRST], false, 0 -(50) Window +(40) Window Input [5]: [gross_margin#20, i_category#11, i_class#10, lochierarchy#23, _w0#73] Arguments: [rank(gross_margin#20) windowspecdefinition(lochierarchy#23, _w0#73, gross_margin#20 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#74], [lochierarchy#23, _w0#73], [gross_margin#20 ASC NULLS FIRST] -(51) Project [codegen id : 11] +(41) Project [codegen id : 12] Output [5]: [gross_margin#20, i_category#11, i_class#10, lochierarchy#23, rank_within_parent#74] Input [6]: [gross_margin#20, i_category#11, i_class#10, lochierarchy#23, _w0#73, rank_within_parent#74] -(52) TakeOrderedAndProject +(42) TakeOrderedAndProject Input [5]: [gross_margin#20, i_category#11, i_class#10, lochierarchy#23, rank_within_parent#74] Arguments: 100, [lochierarchy#23 DESC NULLS LAST, CASE WHEN (lochierarchy#23 = 0) THEN i_category#11 END ASC NULLS FIRST, rank_within_parent#74 ASC NULLS FIRST], [gross_margin#20, i_category#11, i_class#10, lochierarchy#23, rank_within_parent#74] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (57) -+- * ColumnarToRow (56) - +- CometProject (55) - +- CometFilter (54) - +- CometScan parquet spark_catalog.default.date_dim (53) +BroadcastExchange (47) ++- * ColumnarToRow (46) + +- CometProject (45) + +- CometFilter (44) + +- CometScan parquet spark_catalog.default.date_dim (43) -(53) Scan parquet spark_catalog.default.date_dim +(43) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#7, d_year#8] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(54) CometFilter +(44) CometFilter Input [2]: [d_date_sk#7, d_year#8] Condition : ((isnotnull(d_year#8) AND (d_year#8 = 2001)) AND isnotnull(d_date_sk#7)) -(55) CometProject +(45) CometProject Input [2]: [d_date_sk#7, d_year#8] Arguments: [d_date_sk#7], [d_date_sk#7] -(56) ColumnarToRow [codegen id : 1] +(46) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#7] -(57) BroadcastExchange +(47) BroadcastExchange Input [1]: [d_date_sk#7] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt index d2c385a14..59c22c335 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q36a/simplified.txt @@ -1,78 +1,70 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i_class] - WholeStageCodegen (11) + WholeStageCodegen (12) Project [gross_margin,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [gross_margin,lochierarchy,_w0] - WholeStageCodegen (10) - ColumnarToRow + WholeStageCodegen (11) + Sort [lochierarchy,_w0,gross_margin] InputAdapter - CometSort [gross_margin,i_category,i_class,lochierarchy,_w0] - CometColumnarExchange [lochierarchy,_w0] #1 - CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,t_category,t_class] - CometColumnarExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 - RowToColumnar + Exchange [lochierarchy,_w0] #1 + WholeStageCodegen (10) + HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] [_w0] + InputAdapter + Exchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 WholeStageCodegen (9) HashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] InputAdapter Union WholeStageCodegen (2) HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),gross_margin,t_category,t_class,lochierarchy,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_class,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] - CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_class,i_category] #6 - CometFilter [i_item_sk,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] - CometBroadcastExchange [s_store_sk] #7 - CometProject [s_store_sk] - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + InputAdapter + Exchange [i_category,i_class] #3 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,ss_net_profit,ss_ext_sales_price] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,i_item_sk,i_class,i_category] + CometProject [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_class,i_category] #6 + CometFilter [i_item_sk,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + CometBroadcastExchange [s_store_sk] #7 + CometProject [s_store_sk] + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] WholeStageCodegen (5) HashAggregate [i_category,sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category] #8 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [i_category,ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,sum,sum] #3 + InputAdapter + Exchange [i_category] #8 + WholeStageCodegen (4) + HashAggregate [i_category,ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] + InputAdapter + ReusedExchange [i_category,i_class,sum,sum] #3 WholeStageCodegen (8) HashAggregate [sum,isEmpty,sum,isEmpty] [sum(ss_net_profit),sum(ss_ext_sales_price),gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #9 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,sum,sum] #3 + InputAdapter + Exchange #9 + WholeStageCodegen (7) + HashAggregate [ss_net_profit,ss_ext_sales_price] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum,sum] [sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price)),ss_net_profit,ss_ext_sales_price,sum,sum] + InputAdapter + ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt index bd9529104..81588ca21 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/explain.txt @@ -1,57 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (53) -+- * Project (52) - +- * BroadcastHashJoin Inner BuildRight (51) - :- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (32) - : : +- * Filter (31) - : : +- Window (30) - : : +- * Filter (29) - : : +- Window (28) - : : +- * ColumnarToRow (27) - : : +- CometSort (26) - : : +- CometColumnarExchange (25) - : : +- RowToColumnar (24) - : : +- * HashAggregate (23) - : : +- * ColumnarToRow (22) - : : +- CometColumnarExchange (21) - : : +- RowToColumnar (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.item (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.store (13) - : +- BroadcastExchange (42) - : +- * Project (41) - : +- Window (40) - : +- * ColumnarToRow (39) - : +- CometSort (38) - : +- CometColumnarExchange (37) - : +- RowToColumnar (36) - : +- * HashAggregate (35) - : +- * ColumnarToRow (34) - : +- ReusedExchange (33) - +- BroadcastExchange (50) - +- * Project (49) - +- Window (48) - +- * ColumnarToRow (47) - +- CometSort (46) - +- ReusedExchange (45) +TakeOrderedAndProject (45) ++- * Project (44) + +- * BroadcastHashJoin Inner BuildRight (43) + :- * Project (37) + : +- * BroadcastHashJoin Inner BuildRight (36) + : :- * Project (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * Filter (25) + : : +- Window (24) + : : +- * Sort (23) + : : +- Exchange (22) + : : +- * HashAggregate (21) + : : +- Exchange (20) + : : +- * HashAggregate (19) + : : +- * ColumnarToRow (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.store (13) + : +- BroadcastExchange (35) + : +- * Project (34) + : +- Window (33) + : +- * Sort (32) + : +- Exchange (31) + : +- * HashAggregate (30) + : +- ReusedExchange (29) + +- BroadcastExchange (42) + +- * Project (41) + +- Window (40) + +- * Sort (39) + +- ReusedExchange (38) (1) Scan parquet spark_catalog.default.item @@ -148,166 +140,142 @@ Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] Aggregate Attributes [1]: [sum#15] Results [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#16] -(20) RowToColumnar +(20) Exchange Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#16] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(21) CometColumnarExchange -Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(22) ColumnarToRow [codegen id : 2] -Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#16] - -(23) HashAggregate [codegen id : 2] +(21) HashAggregate [codegen id : 2] Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#16] Keys [6]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#6))#17] Results [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#17,17,2) AS sum_sales#18, MakeDecimal(sum(UnscaledValue(ss_sales_price#6))#17,17,2) AS _w0#19] -(24) RowToColumnar -Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19] - -(25) CometColumnarExchange +(22) Exchange Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(26) CometSort +(23) Sort [codegen id : 3] Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19] -Arguments: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST] +Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, s_store_name#13 ASC NULLS FIRST, s_company_name#14 ASC NULLS FIRST, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST], false, 0 -(27) ColumnarToRow [codegen id : 3] -Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19] - -(28) Window +(24) Window Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19] Arguments: [rank(d_year#10, d_moy#11) windowspecdefinition(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#20], [i_category#3, i_brand#2, s_store_name#13, s_company_name#14], [d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST] -(29) Filter [codegen id : 4] +(25) Filter [codegen id : 4] Input [9]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19, rn#20] Condition : (isnotnull(d_year#10) AND (d_year#10 = 1999)) -(30) Window +(26) Window Input [9]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19, rn#20] Arguments: [avg(_w0#19) windowspecdefinition(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#21], [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10] -(31) Filter [codegen id : 13] +(27) Filter [codegen id : 13] Input [10]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] Condition : ((isnotnull(avg_monthly_sales#21) AND (avg_monthly_sales#21 > 0.000000)) AND CASE WHEN (avg_monthly_sales#21 > 0.000000) THEN ((abs((sum_sales#18 - avg_monthly_sales#21)) / avg_monthly_sales#21) > 0.1000000000000000) END) -(32) Project [codegen id : 13] +(28) Project [codegen id : 13] Output [9]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, avg_monthly_sales#21, rn#20] Input [10]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, _w0#19, rn#20, avg_monthly_sales#21] -(33) ReusedExchange [Reuses operator id: 21] +(29) ReusedExchange [Reuses operator id: 20] Output [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] -(34) ColumnarToRow [codegen id : 6] -Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] - -(35) HashAggregate [codegen id : 6] +(30) HashAggregate [codegen id : 6] Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum#28] Keys [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27] Functions [1]: [sum(UnscaledValue(ss_sales_price#29))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#29))#17] Results [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, MakeDecimal(sum(UnscaledValue(ss_sales_price#29))#17,17,2) AS sum_sales#30] -(36) RowToColumnar -Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] - -(37) CometColumnarExchange +(31) Exchange Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] -Arguments: hashpartitioning(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(38) CometSort +(32) Sort [codegen id : 7] Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] -Arguments: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30], [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, s_store_name#24 ASC NULLS FIRST, s_company_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] +Arguments: [i_category#22 ASC NULLS FIRST, i_brand#23 ASC NULLS FIRST, s_store_name#24 ASC NULLS FIRST, s_company_name#25 ASC NULLS FIRST, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST], false, 0 -(39) ColumnarToRow [codegen id : 7] -Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] - -(40) Window +(33) Window Input [7]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30] Arguments: [rank(d_year#26, d_moy#27) windowspecdefinition(i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#31], [i_category#22, i_brand#23, s_store_name#24, s_company_name#25], [d_year#26 ASC NULLS FIRST, d_moy#27 ASC NULLS FIRST] -(41) Project [codegen id : 8] +(34) Project [codegen id : 8] Output [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] Input [8]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, d_year#26, d_moy#27, sum_sales#30, rn#31] -(42) BroadcastExchange +(35) BroadcastExchange Input [6]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] + 1)),false), [plan_id=4] -(43) BroadcastHashJoin [codegen id : 13] +(36) BroadcastHashJoin [codegen id : 13] Left keys [5]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, rn#20] Right keys [5]: [i_category#22, i_brand#23, s_store_name#24, s_company_name#25, (rn#31 + 1)] Join type: Inner Join condition: None -(44) Project [codegen id : 13] +(37) Project [codegen id : 13] Output [10]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30] Input [15]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, avg_monthly_sales#21, rn#20, i_category#22, i_brand#23, s_store_name#24, s_company_name#25, sum_sales#30, rn#31] -(45) ReusedExchange [Reuses operator id: 37] +(38) ReusedExchange [Reuses operator id: 31] Output [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] -(46) CometSort +(39) Sort [codegen id : 11] Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] -Arguments: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38], [i_category#32 ASC NULLS FIRST, i_brand#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST, s_company_name#35 ASC NULLS FIRST, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] +Arguments: [i_category#32 ASC NULLS FIRST, i_brand#33 ASC NULLS FIRST, s_store_name#34 ASC NULLS FIRST, s_company_name#35 ASC NULLS FIRST, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST], false, 0 -(47) ColumnarToRow [codegen id : 11] -Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] - -(48) Window +(40) Window Input [7]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38] Arguments: [rank(d_year#36, d_moy#37) windowspecdefinition(i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#39], [i_category#32, i_brand#33, s_store_name#34, s_company_name#35], [d_year#36 ASC NULLS FIRST, d_moy#37 ASC NULLS FIRST] -(49) Project [codegen id : 12] +(41) Project [codegen id : 12] Output [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] Input [8]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, d_year#36, d_moy#37, sum_sales#38, rn#39] -(50) BroadcastExchange +(42) BroadcastExchange Input [6]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], input[3, string, true], (input[5, int, false] - 1)),false), [plan_id=5] -(51) BroadcastHashJoin [codegen id : 13] +(43) BroadcastHashJoin [codegen id : 13] Left keys [5]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, rn#20] Right keys [5]: [i_category#32, i_brand#33, s_store_name#34, s_company_name#35, (rn#39 - 1)] Join type: Inner Join condition: None -(52) Project [codegen id : 13] +(44) Project [codegen id : 13] Output [7]: [i_category#3, d_year#10, d_moy#11, avg_monthly_sales#21, sum_sales#18, sum_sales#30 AS psum#40, sum_sales#38 AS nsum#41] Input [16]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#18, avg_monthly_sales#21, rn#20, sum_sales#30, i_category#32, i_brand#33, s_store_name#34, s_company_name#35, sum_sales#38, rn#39] -(53) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [7]: [i_category#3, d_year#10, d_moy#11, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] Arguments: 100, [(sum_sales#18 - avg_monthly_sales#21) ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST], [i_category#3, d_year#10, d_moy#11, avg_monthly_sales#21, sum_sales#18, psum#40, nsum#41] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (57) -+- * ColumnarToRow (56) - +- CometFilter (55) - +- CometScan parquet spark_catalog.default.date_dim (54) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(54) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(55) CometFilter +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) -(56) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -(57) BroadcastExchange +(49) BroadcastExchange Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt index 163948e42..61f4976b4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q47/simplified.txt @@ -13,45 +13,41 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (3) - ColumnarToRow + Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 - CometFilter [s_store_sk,s_store_name,s_company_name] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] + Exchange [i_category,i_brand,s_store_name,s_company_name] #1 + WholeStageCodegen (2) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,_w0,sum] + InputAdapter + Exchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,ss_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometFilter [i_item_sk,i_brand,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [s_store_sk,s_store_name,s_company_name] #6 + CometFilter [s_store_sk,s_store_name,s_company_name] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_company_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) @@ -59,16 +55,13 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (7) - ColumnarToRow + Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #8 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 + Exchange [i_category,i_brand,s_store_name,s_company_name] #8 + WholeStageCodegen (6) + HashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] [sum(UnscaledValue(ss_sales_price)),sum_sales,sum] + InputAdapter + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #9 WholeStageCodegen (12) @@ -76,7 +69,6 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, InputAdapter Window [d_year,d_moy,i_category,i_brand,s_store_name,s_company_name] WholeStageCodegen (11) - ColumnarToRow + Sort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt index 895ed6510..01c4c5583 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/explain.txt @@ -1,95 +1,81 @@ == Physical Plan == -* ColumnarToRow (91) -+- CometTakeOrderedAndProject (90) - +- CometHashAggregate (89) - +- CometColumnarExchange (88) - +- RowToColumnar (87) - +- * HashAggregate (86) - +- Union (85) - :- * Project (30) - : +- * Filter (29) - : +- Window (28) - : +- * Sort (27) - : +- Window (26) - : +- * ColumnarToRow (25) - : +- CometSort (24) - : +- CometColumnarExchange (23) - : +- RowToColumnar (22) - : +- * HashAggregate (21) - : +- * ColumnarToRow (20) - : +- CometColumnarExchange (19) - : +- RowToColumnar (18) - : +- * HashAggregate (17) - : +- * ColumnarToRow (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometScan parquet spark_catalog.default.web_returns (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.date_dim (10) - :- * Project (57) - : +- * Filter (56) - : +- Window (55) - : +- * Sort (54) - : +- Window (53) - : +- * ColumnarToRow (52) - : +- CometSort (51) - : +- CometColumnarExchange (50) - : +- RowToColumnar (49) - : +- * HashAggregate (48) - : +- * ColumnarToRow (47) - : +- CometColumnarExchange (46) - : +- RowToColumnar (45) - : +- * HashAggregate (44) - : +- * ColumnarToRow (43) - : +- CometProject (42) - : +- CometBroadcastHashJoin (41) - : :- CometProject (39) - : : +- CometBroadcastHashJoin (38) - : : :- CometBroadcastExchange (34) - : : : +- CometProject (33) - : : : +- CometFilter (32) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (31) - : : +- CometProject (37) - : : +- CometFilter (36) - : : +- CometScan parquet spark_catalog.default.catalog_returns (35) - : +- ReusedExchange (40) - +- * Project (84) - +- * Filter (83) - +- Window (82) - +- * Sort (81) - +- Window (80) - +- * ColumnarToRow (79) - +- CometSort (78) - +- CometColumnarExchange (77) - +- RowToColumnar (76) - +- * HashAggregate (75) - +- * ColumnarToRow (74) - +- CometColumnarExchange (73) - +- RowToColumnar (72) - +- * HashAggregate (71) - +- * ColumnarToRow (70) - +- CometProject (69) - +- CometBroadcastHashJoin (68) - :- CometProject (66) - : +- CometBroadcastHashJoin (65) - : :- CometBroadcastExchange (61) - : : +- CometProject (60) - : : +- CometFilter (59) - : : +- CometScan parquet spark_catalog.default.store_sales (58) - : +- CometProject (64) - : +- CometFilter (63) - : +- CometScan parquet spark_catalog.default.store_returns (62) - +- ReusedExchange (67) +TakeOrderedAndProject (77) ++- * HashAggregate (76) + +- Exchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * Sort (21) + : +- Exchange (20) + : +- * HashAggregate (19) + : +- Exchange (18) + : +- * HashAggregate (17) + : +- * ColumnarToRow (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * Sort (44) + : +- Exchange (43) + : +- * HashAggregate (42) + : +- Exchange (41) + : +- * HashAggregate (40) + : +- * ColumnarToRow (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * Sort (67) + +- Exchange (66) + +- * HashAggregate (65) + +- Exchange (64) + +- * HashAggregate (63) + +- * ColumnarToRow (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) (1) Scan parquet spark_catalog.default.web_sales @@ -174,58 +160,46 @@ Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coa Aggregate Attributes [6]: [sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] Results [7]: [ws_item_sk#1, sum#22, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -(18) RowToColumnar +(18) Exchange Input [7]: [ws_item_sk#1, sum#22, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(19) CometColumnarExchange -Input [7]: [ws_item_sk#1, sum#22, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(20) ColumnarToRow [codegen id : 2] -Input [7]: [ws_item_sk#1, sum#22, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] - -(21) HashAggregate [codegen id : 2] +(19) HashAggregate [codegen id : 2] Input [7]: [ws_item_sk#1, sum#22, sum#23, sum#24, isEmpty#25, sum#26, isEmpty#27] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(wr_return_quantity#10, 0))#28, sum(coalesce(ws_quantity#3, 0))#29, sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00))#30, sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#31] Results [3]: [ws_item_sk#1 AS item#32, (cast(sum(coalesce(wr_return_quantity#10, 0))#28 as decimal(15,4)) / cast(sum(coalesce(ws_quantity#3, 0))#29 as decimal(15,4))) AS return_ratio#33, (cast(sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00))#30 as decimal(15,4)) / cast(sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))#31 as decimal(15,4))) AS currency_ratio#34] -(22) RowToColumnar +(20) Exchange Input [3]: [item#32, return_ratio#33, currency_ratio#34] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2] -(23) CometColumnarExchange +(21) Sort [codegen id : 3] Input [3]: [item#32, return_ratio#33, currency_ratio#34] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: [return_ratio#33 ASC NULLS FIRST], false, 0 -(24) CometSort -Input [3]: [item#32, return_ratio#33, currency_ratio#34] -Arguments: [item#32, return_ratio#33, currency_ratio#34], [return_ratio#33 ASC NULLS FIRST] - -(25) ColumnarToRow [codegen id : 3] -Input [3]: [item#32, return_ratio#33, currency_ratio#34] - -(26) Window +(22) Window Input [3]: [item#32, return_ratio#33, currency_ratio#34] Arguments: [rank(return_ratio#33) windowspecdefinition(return_ratio#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#35], [return_ratio#33 ASC NULLS FIRST] -(27) Sort [codegen id : 4] +(23) Sort [codegen id : 4] Input [4]: [item#32, return_ratio#33, currency_ratio#34, return_rank#35] Arguments: [currency_ratio#34 ASC NULLS FIRST], false, 0 -(28) Window +(24) Window Input [4]: [item#32, return_ratio#33, currency_ratio#34, return_rank#35] Arguments: [rank(currency_ratio#34) windowspecdefinition(currency_ratio#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#36], [currency_ratio#34 ASC NULLS FIRST] -(29) Filter [codegen id : 5] +(25) Filter [codegen id : 5] Input [5]: [item#32, return_ratio#33, currency_ratio#34, return_rank#35, currency_rank#36] Condition : ((return_rank#35 <= 10) OR (currency_rank#36 <= 10)) -(30) Project [codegen id : 5] +(26) Project [codegen id : 5] Output [5]: [web AS channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Input [5]: [item#32, return_ratio#33, currency_ratio#34, return_rank#35, currency_rank#36] -(31) Scan parquet spark_catalog.default.catalog_sales +(27) Scan parquet spark_catalog.default.catalog_sales Output [6]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_net_profit#42, cs_sold_date_sk#43] Batched: true Location: InMemoryFileIndex [] @@ -233,116 +207,104 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#43), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_net_profit), IsNotNull(cs_net_paid), IsNotNull(cs_quantity), GreaterThan(cs_net_profit,1.00), GreaterThan(cs_net_paid,0.00), GreaterThan(cs_quantity,0), IsNotNull(cs_order_number), IsNotNull(cs_item_sk)] ReadSchema: struct -(32) CometFilter +(28) CometFilter Input [6]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_net_profit#42, cs_sold_date_sk#43] Condition : (((((((isnotnull(cs_net_profit#42) AND isnotnull(cs_net_paid#41)) AND isnotnull(cs_quantity#40)) AND (cs_net_profit#42 > 1.00)) AND (cs_net_paid#41 > 0.00)) AND (cs_quantity#40 > 0)) AND isnotnull(cs_order_number#39)) AND isnotnull(cs_item_sk#38)) -(33) CometProject +(29) CometProject Input [6]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_net_profit#42, cs_sold_date_sk#43] Arguments: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43], [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43] -(34) CometBroadcastExchange +(30) CometBroadcastExchange Input [5]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43] Arguments: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43] -(35) Scan parquet spark_catalog.default.catalog_returns +(31) Scan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48, cr_returned_date_sk#49] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_return_amount), GreaterThan(cr_return_amount,10000.00), IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(36) CometFilter +(32) CometFilter Input [5]: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48, cr_returned_date_sk#49] Condition : (((isnotnull(cr_return_amount#48) AND (cr_return_amount#48 > 10000.00)) AND isnotnull(cr_order_number#46)) AND isnotnull(cr_item_sk#45)) -(37) CometProject +(33) CometProject Input [5]: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48, cr_returned_date_sk#49] Arguments: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48], [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48] -(38) CometBroadcastHashJoin +(34) CometBroadcastHashJoin Left output [5]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43] Right output [4]: [cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48] Arguments: [cs_order_number#39, cs_item_sk#38], [cr_order_number#46, cr_item_sk#45], Inner, BuildLeft -(39) CometProject +(35) CometProject Input [9]: [cs_item_sk#38, cs_order_number#39, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43, cr_item_sk#45, cr_order_number#46, cr_return_quantity#47, cr_return_amount#48] Arguments: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43, cr_return_quantity#47, cr_return_amount#48], [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43, cr_return_quantity#47, cr_return_amount#48] -(40) ReusedExchange [Reuses operator id: 13] +(36) ReusedExchange [Reuses operator id: 13] Output [1]: [d_date_sk#50] -(41) CometBroadcastHashJoin +(37) CometBroadcastHashJoin Left output [6]: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43, cr_return_quantity#47, cr_return_amount#48] Right output [1]: [d_date_sk#50] Arguments: [cs_sold_date_sk#43], [d_date_sk#50], Inner, BuildRight -(42) CometProject +(38) CometProject Input [7]: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cs_sold_date_sk#43, cr_return_quantity#47, cr_return_amount#48, d_date_sk#50] Arguments: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cr_return_quantity#47, cr_return_amount#48], [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cr_return_quantity#47, cr_return_amount#48] -(43) ColumnarToRow [codegen id : 6] +(39) ColumnarToRow [codegen id : 6] Input [5]: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cr_return_quantity#47, cr_return_amount#48] -(44) HashAggregate [codegen id : 6] +(40) HashAggregate [codegen id : 6] Input [5]: [cs_item_sk#38, cs_quantity#40, cs_net_paid#41, cr_return_quantity#47, cr_return_amount#48] Keys [1]: [cs_item_sk#38] Functions [4]: [partial_sum(coalesce(cr_return_quantity#47, 0)), partial_sum(coalesce(cs_quantity#40, 0)), partial_sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#41 as decimal(12,2)), 0.00))] Aggregate Attributes [6]: [sum#51, sum#52, sum#53, isEmpty#54, sum#55, isEmpty#56] Results [7]: [cs_item_sk#38, sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] -(45) RowToColumnar -Input [7]: [cs_item_sk#38, sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] - -(46) CometColumnarExchange -Input [7]: [cs_item_sk#38, sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] -Arguments: hashpartitioning(cs_item_sk#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(47) ColumnarToRow [codegen id : 7] +(41) Exchange Input [7]: [cs_item_sk#38, sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] +Arguments: hashpartitioning(cs_item_sk#38, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(48) HashAggregate [codegen id : 7] +(42) HashAggregate [codegen id : 7] Input [7]: [cs_item_sk#38, sum#57, sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] Keys [1]: [cs_item_sk#38] Functions [4]: [sum(coalesce(cr_return_quantity#47, 0)), sum(coalesce(cs_quantity#40, 0)), sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#41 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(cr_return_quantity#47, 0))#63, sum(coalesce(cs_quantity#40, 0))#64, sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#65, sum(coalesce(cast(cs_net_paid#41 as decimal(12,2)), 0.00))#66] Results [3]: [cs_item_sk#38 AS item#67, (cast(sum(coalesce(cr_return_quantity#47, 0))#63 as decimal(15,4)) / cast(sum(coalesce(cs_quantity#40, 0))#64 as decimal(15,4))) AS return_ratio#68, (cast(sum(coalesce(cast(cr_return_amount#48 as decimal(12,2)), 0.00))#65 as decimal(15,4)) / cast(sum(coalesce(cast(cs_net_paid#41 as decimal(12,2)), 0.00))#66 as decimal(15,4))) AS currency_ratio#69] -(49) RowToColumnar -Input [3]: [item#67, return_ratio#68, currency_ratio#69] - -(50) CometColumnarExchange -Input [3]: [item#67, return_ratio#68, currency_ratio#69] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(51) CometSort +(43) Exchange Input [3]: [item#67, return_ratio#68, currency_ratio#69] -Arguments: [item#67, return_ratio#68, currency_ratio#69], [return_ratio#68 ASC NULLS FIRST] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4] -(52) ColumnarToRow [codegen id : 8] +(44) Sort [codegen id : 8] Input [3]: [item#67, return_ratio#68, currency_ratio#69] +Arguments: [return_ratio#68 ASC NULLS FIRST], false, 0 -(53) Window +(45) Window Input [3]: [item#67, return_ratio#68, currency_ratio#69] Arguments: [rank(return_ratio#68) windowspecdefinition(return_ratio#68 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#70], [return_ratio#68 ASC NULLS FIRST] -(54) Sort [codegen id : 9] +(46) Sort [codegen id : 9] Input [4]: [item#67, return_ratio#68, currency_ratio#69, return_rank#70] Arguments: [currency_ratio#69 ASC NULLS FIRST], false, 0 -(55) Window +(47) Window Input [4]: [item#67, return_ratio#68, currency_ratio#69, return_rank#70] Arguments: [rank(currency_ratio#69) windowspecdefinition(currency_ratio#69 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#71], [currency_ratio#69 ASC NULLS FIRST] -(56) Filter [codegen id : 10] +(48) Filter [codegen id : 10] Input [5]: [item#67, return_ratio#68, currency_ratio#69, return_rank#70, currency_rank#71] Condition : ((return_rank#70 <= 10) OR (currency_rank#71 <= 10)) -(57) Project [codegen id : 10] +(49) Project [codegen id : 10] Output [5]: [catalog AS channel#72, item#67, return_ratio#68, return_rank#70, currency_rank#71] Input [5]: [item#67, return_ratio#68, currency_ratio#69, return_rank#70, currency_rank#71] -(58) Scan parquet spark_catalog.default.store_sales +(50) Scan parquet spark_catalog.default.store_sales Output [6]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_net_profit#77, ss_sold_date_sk#78] Batched: true Location: InMemoryFileIndex [] @@ -350,177 +312,161 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#78), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_net_profit), IsNotNull(ss_net_paid), IsNotNull(ss_quantity), GreaterThan(ss_net_profit,1.00), GreaterThan(ss_net_paid,0.00), GreaterThan(ss_quantity,0), IsNotNull(ss_ticket_number), IsNotNull(ss_item_sk)] ReadSchema: struct -(59) CometFilter +(51) CometFilter Input [6]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_net_profit#77, ss_sold_date_sk#78] Condition : (((((((isnotnull(ss_net_profit#77) AND isnotnull(ss_net_paid#76)) AND isnotnull(ss_quantity#75)) AND (ss_net_profit#77 > 1.00)) AND (ss_net_paid#76 > 0.00)) AND (ss_quantity#75 > 0)) AND isnotnull(ss_ticket_number#74)) AND isnotnull(ss_item_sk#73)) -(60) CometProject +(52) CometProject Input [6]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_net_profit#77, ss_sold_date_sk#78] Arguments: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78], [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78] -(61) CometBroadcastExchange +(53) CometBroadcastExchange Input [5]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78] Arguments: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78] -(62) Scan parquet spark_catalog.default.store_returns +(54) Scan parquet spark_catalog.default.store_returns Output [5]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83, sr_returned_date_sk#84] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_return_amt), GreaterThan(sr_return_amt,10000.00), IsNotNull(sr_ticket_number), IsNotNull(sr_item_sk)] ReadSchema: struct -(63) CometFilter +(55) CometFilter Input [5]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83, sr_returned_date_sk#84] Condition : (((isnotnull(sr_return_amt#83) AND (sr_return_amt#83 > 10000.00)) AND isnotnull(sr_ticket_number#81)) AND isnotnull(sr_item_sk#80)) -(64) CometProject +(56) CometProject Input [5]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83, sr_returned_date_sk#84] Arguments: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83], [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] -(65) CometBroadcastHashJoin +(57) CometBroadcastHashJoin Left output [5]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78] Right output [4]: [sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] Arguments: [ss_ticket_number#74, ss_item_sk#73], [sr_ticket_number#81, sr_item_sk#80], Inner, BuildLeft -(66) CometProject +(58) CometProject Input [9]: [ss_item_sk#73, ss_ticket_number#74, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78, sr_item_sk#80, sr_ticket_number#81, sr_return_quantity#82, sr_return_amt#83] Arguments: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78, sr_return_quantity#82, sr_return_amt#83], [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78, sr_return_quantity#82, sr_return_amt#83] -(67) ReusedExchange [Reuses operator id: 13] +(59) ReusedExchange [Reuses operator id: 13] Output [1]: [d_date_sk#85] -(68) CometBroadcastHashJoin +(60) CometBroadcastHashJoin Left output [6]: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78, sr_return_quantity#82, sr_return_amt#83] Right output [1]: [d_date_sk#85] Arguments: [ss_sold_date_sk#78], [d_date_sk#85], Inner, BuildRight -(69) CometProject +(61) CometProject Input [7]: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, ss_sold_date_sk#78, sr_return_quantity#82, sr_return_amt#83, d_date_sk#85] Arguments: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, sr_return_quantity#82, sr_return_amt#83], [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, sr_return_quantity#82, sr_return_amt#83] -(70) ColumnarToRow [codegen id : 11] +(62) ColumnarToRow [codegen id : 11] Input [5]: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, sr_return_quantity#82, sr_return_amt#83] -(71) HashAggregate [codegen id : 11] +(63) HashAggregate [codegen id : 11] Input [5]: [ss_item_sk#73, ss_quantity#75, ss_net_paid#76, sr_return_quantity#82, sr_return_amt#83] Keys [1]: [ss_item_sk#73] Functions [4]: [partial_sum(coalesce(sr_return_quantity#82, 0)), partial_sum(coalesce(ss_quantity#75, 0)), partial_sum(coalesce(cast(sr_return_amt#83 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#76 as decimal(12,2)), 0.00))] Aggregate Attributes [6]: [sum#86, sum#87, sum#88, isEmpty#89, sum#90, isEmpty#91] Results [7]: [ss_item_sk#73, sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97] -(72) RowToColumnar -Input [7]: [ss_item_sk#73, sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97] - -(73) CometColumnarExchange +(64) Exchange Input [7]: [ss_item_sk#73, sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97] -Arguments: hashpartitioning(ss_item_sk#73, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ss_item_sk#73, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(74) ColumnarToRow [codegen id : 12] -Input [7]: [ss_item_sk#73, sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97] - -(75) HashAggregate [codegen id : 12] +(65) HashAggregate [codegen id : 12] Input [7]: [ss_item_sk#73, sum#92, sum#93, sum#94, isEmpty#95, sum#96, isEmpty#97] Keys [1]: [ss_item_sk#73] Functions [4]: [sum(coalesce(sr_return_quantity#82, 0)), sum(coalesce(ss_quantity#75, 0)), sum(coalesce(cast(sr_return_amt#83 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#76 as decimal(12,2)), 0.00))] Aggregate Attributes [4]: [sum(coalesce(sr_return_quantity#82, 0))#98, sum(coalesce(ss_quantity#75, 0))#99, sum(coalesce(cast(sr_return_amt#83 as decimal(12,2)), 0.00))#100, sum(coalesce(cast(ss_net_paid#76 as decimal(12,2)), 0.00))#101] Results [3]: [ss_item_sk#73 AS item#102, (cast(sum(coalesce(sr_return_quantity#82, 0))#98 as decimal(15,4)) / cast(sum(coalesce(ss_quantity#75, 0))#99 as decimal(15,4))) AS return_ratio#103, (cast(sum(coalesce(cast(sr_return_amt#83 as decimal(12,2)), 0.00))#100 as decimal(15,4)) / cast(sum(coalesce(cast(ss_net_paid#76 as decimal(12,2)), 0.00))#101 as decimal(15,4))) AS currency_ratio#104] -(76) RowToColumnar -Input [3]: [item#102, return_ratio#103, currency_ratio#104] - -(77) CometColumnarExchange +(66) Exchange Input [3]: [item#102, return_ratio#103, currency_ratio#104] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(78) CometSort +(67) Sort [codegen id : 13] Input [3]: [item#102, return_ratio#103, currency_ratio#104] -Arguments: [item#102, return_ratio#103, currency_ratio#104], [return_ratio#103 ASC NULLS FIRST] +Arguments: [return_ratio#103 ASC NULLS FIRST], false, 0 -(79) ColumnarToRow [codegen id : 13] -Input [3]: [item#102, return_ratio#103, currency_ratio#104] - -(80) Window +(68) Window Input [3]: [item#102, return_ratio#103, currency_ratio#104] Arguments: [rank(return_ratio#103) windowspecdefinition(return_ratio#103 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS return_rank#105], [return_ratio#103 ASC NULLS FIRST] -(81) Sort [codegen id : 14] +(69) Sort [codegen id : 14] Input [4]: [item#102, return_ratio#103, currency_ratio#104, return_rank#105] Arguments: [currency_ratio#104 ASC NULLS FIRST], false, 0 -(82) Window +(70) Window Input [4]: [item#102, return_ratio#103, currency_ratio#104, return_rank#105] Arguments: [rank(currency_ratio#104) windowspecdefinition(currency_ratio#104 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS currency_rank#106], [currency_ratio#104 ASC NULLS FIRST] -(83) Filter [codegen id : 15] +(71) Filter [codegen id : 15] Input [5]: [item#102, return_ratio#103, currency_ratio#104, return_rank#105, currency_rank#106] Condition : ((return_rank#105 <= 10) OR (currency_rank#106 <= 10)) -(84) Project [codegen id : 15] +(72) Project [codegen id : 15] Output [5]: [store AS channel#107, item#102, return_ratio#103, return_rank#105, currency_rank#106] Input [5]: [item#102, return_ratio#103, currency_ratio#104, return_rank#105, currency_rank#106] -(85) Union +(73) Union -(86) HashAggregate [codegen id : 16] +(74) HashAggregate [codegen id : 16] Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Keys [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Functions: [] Aggregate Attributes: [] Results [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] -(87) RowToColumnar -Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] - -(88) CometColumnarExchange +(75) Exchange Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] -Arguments: hashpartitioning(channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(89) CometHashAggregate +(76) HashAggregate [codegen id : 17] Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Keys [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] -(90) CometTakeOrderedAndProject -Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#37 ASC NULLS FIRST,return_rank#35 ASC NULLS FIRST,currency_rank#36 ASC NULLS FIRST,item#32 ASC NULLS FIRST], output=[channel#37,item#32,return_ratio#33,return_rank#35,currency_rank#36]), [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36], 100, [channel#37 ASC NULLS FIRST, return_rank#35 ASC NULLS FIRST, currency_rank#36 ASC NULLS FIRST, item#32 ASC NULLS FIRST], [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] - -(91) ColumnarToRow [codegen id : 17] +(77) TakeOrderedAndProject Input [5]: [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] +Arguments: 100, [channel#37 ASC NULLS FIRST, return_rank#35 ASC NULLS FIRST, currency_rank#36 ASC NULLS FIRST, item#32 ASC NULLS FIRST], [channel#37, item#32, return_ratio#33, return_rank#35, currency_rank#36] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (96) -+- * ColumnarToRow (95) - +- CometProject (94) - +- CometFilter (93) - +- CometScan parquet spark_catalog.default.date_dim (92) +BroadcastExchange (82) ++- * ColumnarToRow (81) + +- CometProject (80) + +- CometFilter (79) + +- CometScan parquet spark_catalog.default.date_dim (78) -(92) Scan parquet spark_catalog.default.date_dim +(78) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(93) CometFilter +(79) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) -(94) CometProject +(80) CometProject Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Arguments: [d_date_sk#13], [d_date_sk#13] -(95) ColumnarToRow [codegen id : 1] +(81) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#13] -(96) BroadcastExchange +(82) BroadcastExchange Input [1]: [d_date_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 31 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#7 +Subquery:2 Hosting operator id = 27 Hosting Expression = cs_sold_date_sk#43 IN dynamicpruning#7 -Subquery:3 Hosting operator id = 58 Hosting Expression = ss_sold_date_sk#78 IN dynamicpruning#7 +Subquery:3 Hosting operator id = 50 Hosting Expression = ss_sold_date_sk#78 IN dynamicpruning#7 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt index 4d4e531a0..bd3b21cdb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q49/simplified.txt @@ -1,135 +1,121 @@ -WholeStageCodegen (17) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] - CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] - CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (5) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (4) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #2 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] +TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] + WholeStageCodegen (17) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (16) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (5) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (4) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (3) + Sort [return_ratio] + InputAdapter + Exchange #2 + WholeStageCodegen (2) + HashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [ws_item_sk] #3 + WholeStageCodegen (1) + HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] ColumnarToRow InputAdapter - CometColumnarExchange [ws_item_sk] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ws_item_sk,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (10) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (9) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (8) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #7 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (10) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (9) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (8) + Sort [return_ratio] + InputAdapter + Exchange #7 + WholeStageCodegen (7) + HashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [cs_item_sk] #8 + WholeStageCodegen (6) + HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] ColumnarToRow InputAdapter - CometColumnarExchange [cs_item_sk] #8 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [cs_item_sk,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - WholeStageCodegen (15) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (14) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (13) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #10 - RowToColumnar - WholeStageCodegen (12) - HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + WholeStageCodegen (15) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (14) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (13) + Sort [return_ratio] + InputAdapter + Exchange #10 + WholeStageCodegen (12) + HashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty] [sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00)),item,return_ratio,currency_ratio,sum,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [ss_item_sk] #11 + WholeStageCodegen (11) + HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] ColumnarToRow InputAdapter - CometColumnarExchange [ss_item_sk] #11 - RowToColumnar - WholeStageCodegen (11) - HashAggregate [ss_item_sk,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] [sum,sum,sum,isEmpty,sum,isEmpty,sum,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt index b92d21780..a5ad8c293 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/explain.txt @@ -1,92 +1,74 @@ == Physical Plan == -TakeOrderedAndProject (88) -+- * Filter (87) - +- * HashAggregate (86) - +- * HashAggregate (85) - +- * Project (84) - +- * BroadcastHashJoin Inner BuildRight (83) - :- Window (76) - : +- * ColumnarToRow (75) - : +- CometSort (74) - : +- CometColumnarExchange (73) - : +- CometProject (72) - : +- CometFilter (71) - : +- CometSortMergeJoin (70) - : :- CometSort (36) - : : +- CometColumnarExchange (35) - : : +- RowToColumnar (34) - : : +- * HashAggregate (33) - : : +- * ColumnarToRow (32) - : : +- CometColumnarExchange (31) - : : +- RowToColumnar (30) - : : +- * HashAggregate (29) - : : +- * Project (28) - : : +- * BroadcastHashJoin Inner BuildRight (27) - : : :- * Project (20) - : : : +- Window (19) - : : : +- * ColumnarToRow (18) - : : : +- CometSort (17) - : : : +- CometColumnarExchange (16) - : : : +- RowToColumnar (15) - : : : +- * HashAggregate (14) - : : : +- * ColumnarToRow (13) - : : : +- CometColumnarExchange (12) - : : : +- RowToColumnar (11) - : : : +- * HashAggregate (10) - : : : +- * ColumnarToRow (9) - : : : +- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- BroadcastExchange (26) - : : +- * Project (25) - : : +- Window (24) - : : +- * ColumnarToRow (23) - : : +- CometSort (22) - : : +- ReusedExchange (21) - : +- CometSort (69) - : +- CometColumnarExchange (68) - : +- RowToColumnar (67) - : +- * HashAggregate (66) - : +- * ColumnarToRow (65) - : +- CometColumnarExchange (64) - : +- RowToColumnar (63) - : +- * HashAggregate (62) - : +- * Project (61) - : +- * BroadcastHashJoin Inner BuildRight (60) - : :- * Project (53) - : : +- Window (52) - : : +- * ColumnarToRow (51) - : : +- CometSort (50) - : : +- CometColumnarExchange (49) - : : +- RowToColumnar (48) - : : +- * HashAggregate (47) - : : +- * ColumnarToRow (46) - : : +- CometColumnarExchange (45) - : : +- RowToColumnar (44) - : : +- * HashAggregate (43) - : : +- * ColumnarToRow (42) - : : +- CometProject (41) - : : +- CometBroadcastHashJoin (40) - : : :- CometFilter (38) - : : : +- CometScan parquet spark_catalog.default.store_sales (37) - : : +- ReusedExchange (39) - : +- BroadcastExchange (59) - : +- * Project (58) - : +- Window (57) - : +- * ColumnarToRow (56) - : +- CometSort (55) - : +- ReusedExchange (54) - +- BroadcastExchange (82) - +- * Project (81) - +- Window (80) - +- * ColumnarToRow (79) - +- CometSort (78) - +- ReusedExchange (77) +TakeOrderedAndProject (70) ++- * Filter (69) + +- * HashAggregate (68) + +- * HashAggregate (67) + +- * Project (66) + +- * BroadcastHashJoin Inner BuildRight (65) + :- Window (59) + : +- * Sort (58) + : +- Exchange (57) + : +- * Project (56) + : +- * Filter (55) + : +- * SortMergeJoin FullOuter (54) + : :- * Sort (28) + : : +- Exchange (27) + : : +- * HashAggregate (26) + : : +- Exchange (25) + : : +- * HashAggregate (24) + : : +- * Project (23) + : : +- * BroadcastHashJoin Inner BuildRight (22) + : : :- * Project (16) + : : : +- Window (15) + : : : +- * Sort (14) + : : : +- Exchange (13) + : : : +- * HashAggregate (12) + : : : +- Exchange (11) + : : : +- * HashAggregate (10) + : : : +- * ColumnarToRow (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (21) + : : +- * Project (20) + : : +- Window (19) + : : +- * Sort (18) + : : +- ReusedExchange (17) + : +- * Sort (53) + : +- Exchange (52) + : +- * HashAggregate (51) + : +- Exchange (50) + : +- * HashAggregate (49) + : +- * Project (48) + : +- * BroadcastHashJoin Inner BuildRight (47) + : :- * Project (41) + : : +- Window (40) + : : +- * Sort (39) + : : +- Exchange (38) + : : +- * HashAggregate (37) + : : +- Exchange (36) + : : +- * HashAggregate (35) + : : +- * ColumnarToRow (34) + : : +- CometProject (33) + : : +- CometBroadcastHashJoin (32) + : : :- CometFilter (30) + : : : +- CometScan parquet spark_catalog.default.store_sales (29) + : : +- ReusedExchange (31) + : +- BroadcastExchange (46) + : +- * Project (45) + : +- Window (44) + : +- * Sort (43) + : +- ReusedExchange (42) + +- BroadcastExchange (64) + +- * Project (63) + +- Window (62) + +- * Sort (61) + +- ReusedExchange (60) (1) Scan parquet spark_catalog.default.web_sales @@ -139,113 +121,89 @@ Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] Aggregate Attributes [1]: [sum#8] Results [3]: [ws_item_sk#1, d_date#6, sum#9] -(11) RowToColumnar +(11) Exchange Input [3]: [ws_item_sk#1, d_date#6, sum#9] +Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(12) CometColumnarExchange -Input [3]: [ws_item_sk#1, d_date#6, sum#9] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(13) ColumnarToRow [codegen id : 2] -Input [3]: [ws_item_sk#1, d_date#6, sum#9] - -(14) HashAggregate [codegen id : 2] +(12) HashAggregate [codegen id : 2] Input [3]: [ws_item_sk#1, d_date#6, sum#9] Keys [2]: [ws_item_sk#1, d_date#6] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_sales_price#2))#10] Results [4]: [ws_item_sk#1 AS item_sk#11, d_date#6, MakeDecimal(sum(UnscaledValue(ws_sales_price#2))#10,17,2) AS sumws#12, ws_item_sk#1] -(15) RowToColumnar -Input [4]: [item_sk#11, d_date#6, sumws#12, ws_item_sk#1] - -(16) CometColumnarExchange +(13) Exchange Input [4]: [item_sk#11, d_date#6, sumws#12, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(17) CometSort +(14) Sort [codegen id : 3] Input [4]: [item_sk#11, d_date#6, sumws#12, ws_item_sk#1] -Arguments: [item_sk#11, d_date#6, sumws#12, ws_item_sk#1], [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Arguments: [ws_item_sk#1 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0 -(18) ColumnarToRow [codegen id : 3] -Input [4]: [item_sk#11, d_date#6, sumws#12, ws_item_sk#1] - -(19) Window +(15) Window Input [4]: [item_sk#11, d_date#6, sumws#12, ws_item_sk#1] Arguments: [row_number() windowspecdefinition(ws_item_sk#1, d_date#6 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#13], [ws_item_sk#1], [d_date#6 ASC NULLS FIRST] -(20) Project [codegen id : 8] +(16) Project [codegen id : 8] Output [4]: [item_sk#11, d_date#6, sumws#12, rk#13] Input [5]: [item_sk#11, d_date#6, sumws#12, ws_item_sk#1, rk#13] -(21) ReusedExchange [Reuses operator id: 16] +(17) ReusedExchange [Reuses operator id: 13] Output [4]: [item_sk#14, d_date#15, sumws#16, ws_item_sk#17] -(22) CometSort +(18) Sort [codegen id : 6] Input [4]: [item_sk#14, d_date#15, sumws#16, ws_item_sk#17] -Arguments: [item_sk#14, d_date#15, sumws#16, ws_item_sk#17], [ws_item_sk#17 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST] +Arguments: [ws_item_sk#17 ASC NULLS FIRST, d_date#15 ASC NULLS FIRST], false, 0 -(23) ColumnarToRow [codegen id : 6] -Input [4]: [item_sk#14, d_date#15, sumws#16, ws_item_sk#17] - -(24) Window +(19) Window Input [4]: [item_sk#14, d_date#15, sumws#16, ws_item_sk#17] Arguments: [row_number() windowspecdefinition(ws_item_sk#17, d_date#15 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#18], [ws_item_sk#17], [d_date#15 ASC NULLS FIRST] -(25) Project [codegen id : 7] +(20) Project [codegen id : 7] Output [3]: [item_sk#14, sumws#16, rk#18] Input [5]: [item_sk#14, d_date#15, sumws#16, ws_item_sk#17, rk#18] -(26) BroadcastExchange +(21) BroadcastExchange Input [3]: [item_sk#14, sumws#16, rk#18] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(27) BroadcastHashJoin [codegen id : 8] +(22) BroadcastHashJoin [codegen id : 8] Left keys [1]: [item_sk#11] Right keys [1]: [item_sk#14] Join type: Inner Join condition: (rk#13 >= rk#18) -(28) Project [codegen id : 8] +(23) Project [codegen id : 8] Output [4]: [item_sk#11, d_date#6, sumws#12, sumws#16] Input [7]: [item_sk#11, d_date#6, sumws#12, rk#13, item_sk#14, sumws#16, rk#18] -(29) HashAggregate [codegen id : 8] +(24) HashAggregate [codegen id : 8] Input [4]: [item_sk#11, d_date#6, sumws#12, sumws#16] Keys [3]: [item_sk#11, d_date#6, sumws#12] Functions [1]: [partial_sum(sumws#16)] Aggregate Attributes [2]: [sum#19, isEmpty#20] Results [5]: [item_sk#11, d_date#6, sumws#12, sum#21, isEmpty#22] -(30) RowToColumnar +(25) Exchange Input [5]: [item_sk#11, d_date#6, sumws#12, sum#21, isEmpty#22] +Arguments: hashpartitioning(item_sk#11, d_date#6, sumws#12, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(31) CometColumnarExchange -Input [5]: [item_sk#11, d_date#6, sumws#12, sum#21, isEmpty#22] -Arguments: hashpartitioning(item_sk#11, d_date#6, sumws#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(32) ColumnarToRow [codegen id : 9] -Input [5]: [item_sk#11, d_date#6, sumws#12, sum#21, isEmpty#22] - -(33) HashAggregate [codegen id : 9] +(26) HashAggregate [codegen id : 9] Input [5]: [item_sk#11, d_date#6, sumws#12, sum#21, isEmpty#22] Keys [3]: [item_sk#11, d_date#6, sumws#12] Functions [1]: [sum(sumws#16)] Aggregate Attributes [1]: [sum(sumws#16)#23] Results [3]: [item_sk#11, d_date#6, sum(sumws#16)#23 AS cume_sales#24] -(34) RowToColumnar -Input [3]: [item_sk#11, d_date#6, cume_sales#24] - -(35) CometColumnarExchange +(27) Exchange Input [3]: [item_sk#11, d_date#6, cume_sales#24] -Arguments: hashpartitioning(item_sk#11, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(item_sk#11, d_date#6, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(36) CometSort +(28) Sort [codegen id : 10] Input [3]: [item_sk#11, d_date#6, cume_sales#24] -Arguments: [item_sk#11, d_date#6, cume_sales#24], [item_sk#11 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] +Arguments: [item_sk#11 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0 -(37) Scan parquet spark_catalog.default.store_sales +(29) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#25, ss_sales_price#26, ss_sold_date_sk#27] Batched: true Location: InMemoryFileIndex [] @@ -253,252 +211,223 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#27), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(38) CometFilter +(30) CometFilter Input [3]: [ss_item_sk#25, ss_sales_price#26, ss_sold_date_sk#27] Condition : isnotnull(ss_item_sk#25) -(39) ReusedExchange [Reuses operator id: 6] +(31) ReusedExchange [Reuses operator id: 6] Output [2]: [d_date_sk#29, d_date#30] -(40) CometBroadcastHashJoin +(32) CometBroadcastHashJoin Left output [3]: [ss_item_sk#25, ss_sales_price#26, ss_sold_date_sk#27] Right output [2]: [d_date_sk#29, d_date#30] Arguments: [ss_sold_date_sk#27], [d_date_sk#29], Inner, BuildRight -(41) CometProject +(33) CometProject Input [5]: [ss_item_sk#25, ss_sales_price#26, ss_sold_date_sk#27, d_date_sk#29, d_date#30] Arguments: [ss_item_sk#25, ss_sales_price#26, d_date#30], [ss_item_sk#25, ss_sales_price#26, d_date#30] -(42) ColumnarToRow [codegen id : 10] +(34) ColumnarToRow [codegen id : 11] Input [3]: [ss_item_sk#25, ss_sales_price#26, d_date#30] -(43) HashAggregate [codegen id : 10] +(35) HashAggregate [codegen id : 11] Input [3]: [ss_item_sk#25, ss_sales_price#26, d_date#30] Keys [2]: [ss_item_sk#25, d_date#30] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#26))] Aggregate Attributes [1]: [sum#31] Results [3]: [ss_item_sk#25, d_date#30, sum#32] -(44) RowToColumnar -Input [3]: [ss_item_sk#25, d_date#30, sum#32] - -(45) CometColumnarExchange -Input [3]: [ss_item_sk#25, d_date#30, sum#32] -Arguments: hashpartitioning(ss_item_sk#25, d_date#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(46) ColumnarToRow [codegen id : 11] +(36) Exchange Input [3]: [ss_item_sk#25, d_date#30, sum#32] +Arguments: hashpartitioning(ss_item_sk#25, d_date#30, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(47) HashAggregate [codegen id : 11] +(37) HashAggregate [codegen id : 12] Input [3]: [ss_item_sk#25, d_date#30, sum#32] Keys [2]: [ss_item_sk#25, d_date#30] Functions [1]: [sum(UnscaledValue(ss_sales_price#26))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#26))#33] Results [4]: [ss_item_sk#25 AS item_sk#34, d_date#30, MakeDecimal(sum(UnscaledValue(ss_sales_price#26))#33,17,2) AS sumss#35, ss_item_sk#25] -(48) RowToColumnar -Input [4]: [item_sk#34, d_date#30, sumss#35, ss_item_sk#25] - -(49) CometColumnarExchange +(38) Exchange Input [4]: [item_sk#34, d_date#30, sumss#35, ss_item_sk#25] -Arguments: hashpartitioning(ss_item_sk#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(ss_item_sk#25, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(50) CometSort +(39) Sort [codegen id : 13] Input [4]: [item_sk#34, d_date#30, sumss#35, ss_item_sk#25] -Arguments: [item_sk#34, d_date#30, sumss#35, ss_item_sk#25], [ss_item_sk#25 ASC NULLS FIRST, d_date#30 ASC NULLS FIRST] +Arguments: [ss_item_sk#25 ASC NULLS FIRST, d_date#30 ASC NULLS FIRST], false, 0 -(51) ColumnarToRow [codegen id : 12] -Input [4]: [item_sk#34, d_date#30, sumss#35, ss_item_sk#25] - -(52) Window +(40) Window Input [4]: [item_sk#34, d_date#30, sumss#35, ss_item_sk#25] Arguments: [row_number() windowspecdefinition(ss_item_sk#25, d_date#30 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#36], [ss_item_sk#25], [d_date#30 ASC NULLS FIRST] -(53) Project [codegen id : 17] +(41) Project [codegen id : 18] Output [4]: [item_sk#34, d_date#30, sumss#35, rk#36] Input [5]: [item_sk#34, d_date#30, sumss#35, ss_item_sk#25, rk#36] -(54) ReusedExchange [Reuses operator id: 49] +(42) ReusedExchange [Reuses operator id: 38] Output [4]: [item_sk#37, d_date#38, sumss#39, ss_item_sk#40] -(55) CometSort +(43) Sort [codegen id : 16] Input [4]: [item_sk#37, d_date#38, sumss#39, ss_item_sk#40] -Arguments: [item_sk#37, d_date#38, sumss#39, ss_item_sk#40], [ss_item_sk#40 ASC NULLS FIRST, d_date#38 ASC NULLS FIRST] +Arguments: [ss_item_sk#40 ASC NULLS FIRST, d_date#38 ASC NULLS FIRST], false, 0 -(56) ColumnarToRow [codegen id : 15] -Input [4]: [item_sk#37, d_date#38, sumss#39, ss_item_sk#40] - -(57) Window +(44) Window Input [4]: [item_sk#37, d_date#38, sumss#39, ss_item_sk#40] Arguments: [row_number() windowspecdefinition(ss_item_sk#40, d_date#38 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#41], [ss_item_sk#40], [d_date#38 ASC NULLS FIRST] -(58) Project [codegen id : 16] +(45) Project [codegen id : 17] Output [3]: [item_sk#37, sumss#39, rk#41] Input [5]: [item_sk#37, d_date#38, sumss#39, ss_item_sk#40, rk#41] -(59) BroadcastExchange +(46) BroadcastExchange Input [3]: [item_sk#37, sumss#39, rk#41] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -(60) BroadcastHashJoin [codegen id : 17] +(47) BroadcastHashJoin [codegen id : 18] Left keys [1]: [item_sk#34] Right keys [1]: [item_sk#37] Join type: Inner Join condition: (rk#36 >= rk#41) -(61) Project [codegen id : 17] +(48) Project [codegen id : 18] Output [4]: [item_sk#34, d_date#30, sumss#35, sumss#39] Input [7]: [item_sk#34, d_date#30, sumss#35, rk#36, item_sk#37, sumss#39, rk#41] -(62) HashAggregate [codegen id : 17] +(49) HashAggregate [codegen id : 18] Input [4]: [item_sk#34, d_date#30, sumss#35, sumss#39] Keys [3]: [item_sk#34, d_date#30, sumss#35] Functions [1]: [partial_sum(sumss#39)] Aggregate Attributes [2]: [sum#42, isEmpty#43] Results [5]: [item_sk#34, d_date#30, sumss#35, sum#44, isEmpty#45] -(63) RowToColumnar +(50) Exchange Input [5]: [item_sk#34, d_date#30, sumss#35, sum#44, isEmpty#45] +Arguments: hashpartitioning(item_sk#34, d_date#30, sumss#35, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(64) CometColumnarExchange -Input [5]: [item_sk#34, d_date#30, sumss#35, sum#44, isEmpty#45] -Arguments: hashpartitioning(item_sk#34, d_date#30, sumss#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(65) ColumnarToRow [codegen id : 18] -Input [5]: [item_sk#34, d_date#30, sumss#35, sum#44, isEmpty#45] - -(66) HashAggregate [codegen id : 18] +(51) HashAggregate [codegen id : 19] Input [5]: [item_sk#34, d_date#30, sumss#35, sum#44, isEmpty#45] Keys [3]: [item_sk#34, d_date#30, sumss#35] Functions [1]: [sum(sumss#39)] Aggregate Attributes [1]: [sum(sumss#39)#46] Results [3]: [item_sk#34, d_date#30, sum(sumss#39)#46 AS cume_sales#47] -(67) RowToColumnar -Input [3]: [item_sk#34, d_date#30, cume_sales#47] - -(68) CometColumnarExchange +(52) Exchange Input [3]: [item_sk#34, d_date#30, cume_sales#47] -Arguments: hashpartitioning(item_sk#34, d_date#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(item_sk#34, d_date#30, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(69) CometSort +(53) Sort [codegen id : 20] Input [3]: [item_sk#34, d_date#30, cume_sales#47] -Arguments: [item_sk#34, d_date#30, cume_sales#47], [item_sk#34 ASC NULLS FIRST, d_date#30 ASC NULLS FIRST] +Arguments: [item_sk#34 ASC NULLS FIRST, d_date#30 ASC NULLS FIRST], false, 0 -(70) CometSortMergeJoin -Left output [3]: [item_sk#11, d_date#6, cume_sales#24] -Right output [3]: [item_sk#34, d_date#30, cume_sales#47] -Arguments: [item_sk#11, d_date#6], [item_sk#34, d_date#30], FullOuter +(54) SortMergeJoin [codegen id : 21] +Left keys [2]: [item_sk#11, d_date#6] +Right keys [2]: [item_sk#34, d_date#30] +Join type: FullOuter +Join condition: None -(71) CometFilter +(55) Filter [codegen id : 21] Input [6]: [item_sk#11, d_date#6, cume_sales#24, item_sk#34, d_date#30, cume_sales#47] Condition : isnotnull(CASE WHEN isnotnull(item_sk#11) THEN item_sk#11 ELSE item_sk#34 END) -(72) CometProject +(56) Project [codegen id : 21] +Output [4]: [CASE WHEN isnotnull(item_sk#11) THEN item_sk#11 ELSE item_sk#34 END AS item_sk#48, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#30 END AS d_date#49, cume_sales#24 AS web_sales#50, cume_sales#47 AS store_sales#51] Input [6]: [item_sk#11, d_date#6, cume_sales#24, item_sk#34, d_date#30, cume_sales#47] -Arguments: [item_sk#48, d_date#49, web_sales#50, store_sales#51], [CASE WHEN isnotnull(item_sk#11) THEN item_sk#11 ELSE item_sk#34 END AS item_sk#48, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#30 END AS d_date#49, cume_sales#24 AS web_sales#50, cume_sales#47 AS store_sales#51] -(73) CometColumnarExchange +(57) Exchange Input [4]: [item_sk#48, d_date#49, web_sales#50, store_sales#51] -Arguments: hashpartitioning(item_sk#48, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(item_sk#48, 5), ENSURE_REQUIREMENTS, [plan_id=11] -(74) CometSort +(58) Sort [codegen id : 22] Input [4]: [item_sk#48, d_date#49, web_sales#50, store_sales#51] -Arguments: [item_sk#48, d_date#49, web_sales#50, store_sales#51], [item_sk#48 ASC NULLS FIRST, d_date#49 ASC NULLS FIRST] +Arguments: [item_sk#48 ASC NULLS FIRST, d_date#49 ASC NULLS FIRST], false, 0 -(75) ColumnarToRow [codegen id : 19] -Input [4]: [item_sk#48, d_date#49, web_sales#50, store_sales#51] - -(76) Window +(59) Window Input [4]: [item_sk#48, d_date#49, web_sales#50, store_sales#51] Arguments: [row_number() windowspecdefinition(item_sk#48, d_date#49 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#52], [item_sk#48], [d_date#49 ASC NULLS FIRST] -(77) ReusedExchange [Reuses operator id: 73] +(60) ReusedExchange [Reuses operator id: 57] Output [4]: [item_sk#53, d_date#54, web_sales#55, store_sales#56] -(78) CometSort -Input [4]: [item_sk#53, d_date#54, web_sales#55, store_sales#56] -Arguments: [item_sk#53, d_date#54, web_sales#55, store_sales#56], [item_sk#53 ASC NULLS FIRST, d_date#54 ASC NULLS FIRST] - -(79) ColumnarToRow [codegen id : 38] +(61) Sort [codegen id : 44] Input [4]: [item_sk#53, d_date#54, web_sales#55, store_sales#56] +Arguments: [item_sk#53 ASC NULLS FIRST, d_date#54 ASC NULLS FIRST], false, 0 -(80) Window +(62) Window Input [4]: [item_sk#53, d_date#54, web_sales#55, store_sales#56] Arguments: [row_number() windowspecdefinition(item_sk#53, d_date#54 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#57], [item_sk#53], [d_date#54 ASC NULLS FIRST] -(81) Project [codegen id : 39] +(63) Project [codegen id : 45] Output [4]: [item_sk#53, web_sales#55, store_sales#56, rk#57] Input [5]: [item_sk#53, d_date#54, web_sales#55, store_sales#56, rk#57] -(82) BroadcastExchange +(64) BroadcastExchange Input [4]: [item_sk#53, web_sales#55, store_sales#56, rk#57] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] -(83) BroadcastHashJoin [codegen id : 40] +(65) BroadcastHashJoin [codegen id : 46] Left keys [1]: [item_sk#48] Right keys [1]: [item_sk#53] Join type: Inner Join condition: (rk#52 >= rk#57) -(84) Project [codegen id : 40] +(66) Project [codegen id : 46] Output [6]: [item_sk#48, d_date#49, web_sales#50, store_sales#51, web_sales#55, store_sales#56] Input [9]: [item_sk#48, d_date#49, web_sales#50, store_sales#51, rk#52, item_sk#53, web_sales#55, store_sales#56, rk#57] -(85) HashAggregate [codegen id : 40] +(67) HashAggregate [codegen id : 46] Input [6]: [item_sk#48, d_date#49, web_sales#50, store_sales#51, web_sales#55, store_sales#56] Keys [4]: [item_sk#48, d_date#49, web_sales#50, store_sales#51] Functions [2]: [partial_max(web_sales#55), partial_max(store_sales#56)] Aggregate Attributes [2]: [max#58, max#59] Results [6]: [item_sk#48, d_date#49, web_sales#50, store_sales#51, max#60, max#61] -(86) HashAggregate [codegen id : 40] +(68) HashAggregate [codegen id : 46] Input [6]: [item_sk#48, d_date#49, web_sales#50, store_sales#51, max#60, max#61] Keys [4]: [item_sk#48, d_date#49, web_sales#50, store_sales#51] Functions [2]: [max(web_sales#55), max(store_sales#56)] Aggregate Attributes [2]: [max(web_sales#55)#62, max(store_sales#56)#63] Results [6]: [item_sk#48, d_date#49, web_sales#50, store_sales#51, max(web_sales#55)#62 AS web_cumulative#64, max(store_sales#56)#63 AS store_cumulative#65] -(87) Filter [codegen id : 40] +(69) Filter [codegen id : 46] Input [6]: [item_sk#48, d_date#49, web_sales#50, store_sales#51, web_cumulative#64, store_cumulative#65] Condition : ((isnotnull(web_cumulative#64) AND isnotnull(store_cumulative#65)) AND (web_cumulative#64 > store_cumulative#65)) -(88) TakeOrderedAndProject +(70) TakeOrderedAndProject Input [6]: [item_sk#48, d_date#49, web_sales#50, store_sales#51, web_cumulative#64, store_cumulative#65] Arguments: 100, [item_sk#48 ASC NULLS FIRST, d_date#49 ASC NULLS FIRST], [item_sk#48, d_date#49, web_sales#50, store_sales#51, web_cumulative#64, store_cumulative#65] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (93) -+- * ColumnarToRow (92) - +- CometProject (91) - +- CometFilter (90) - +- CometScan parquet spark_catalog.default.date_dim (89) +BroadcastExchange (75) ++- * ColumnarToRow (74) + +- CometProject (73) + +- CometFilter (72) + +- CometScan parquet spark_catalog.default.date_dim (71) -(89) Scan parquet spark_catalog.default.date_dim +(71) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(90) CometFilter +(72) CometFilter Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5)) -(91) CometProject +(73) CometProject Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] -(92) ColumnarToRow [codegen id : 1] +(74) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#5, d_date#6] -(93) BroadcastExchange +(75) BroadcastExchange Input [2]: [d_date_sk#5, d_date#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] -Subquery:2 Hosting operator id = 37 Hosting Expression = ss_sold_date_sk#27 IN dynamicpruning#4 +Subquery:2 Hosting operator id = 29 Hosting Expression = ss_sold_date_sk#27 IN dynamicpruning#4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt index f7edf9b01..47ad5e34f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q51a/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (40) + WholeStageCodegen (46) Filter [web_cumulative,store_cumulative] HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] @@ -7,23 +7,23 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store BroadcastHashJoin [item_sk,item_sk,rk,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (19) - ColumnarToRow + WholeStageCodegen (22) + Sort [item_sk,d_date] InputAdapter - CometSort [item_sk,d_date,web_sales,store_sales] - CometColumnarExchange [item_sk] #1 - CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] - CometFilter [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] - CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #2 - RowToColumnar - WholeStageCodegen (9) - HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [item_sk,d_date,sumws] #3 - RowToColumnar + Exchange [item_sk] #1 + WholeStageCodegen (21) + Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] + Filter [item_sk,item_sk] + SortMergeJoin [item_sk,d_date,item_sk,d_date] + InputAdapter + WholeStageCodegen (10) + Sort [item_sk,d_date] + InputAdapter + Exchange [item_sk,d_date] #2 + WholeStageCodegen (9) + HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] + InputAdapter + Exchange [item_sk,d_date,sumws] #3 WholeStageCodegen (8) HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] Project [item_sk,d_date,sumws,sumws] @@ -32,37 +32,33 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter Window [ws_item_sk,d_date] WholeStageCodegen (3) - ColumnarToRow + Sort [ws_item_sk,d_date] InputAdapter - CometSort [item_sk,d_date,sumws,ws_item_sk] - CometColumnarExchange [ws_item_sk] #4 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,sumws,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ws_item_sk,d_date] #5 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_sales_price,d_date] - CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] - CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #6 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] - CometBroadcastExchange [d_date_sk,d_date] #7 - CometProject [d_date_sk,d_date] - CometFilter [d_date_sk,d_date,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + Exchange [ws_item_sk] #4 + WholeStageCodegen (2) + HashAggregate [ws_item_sk,d_date,sum] [sum(UnscaledValue(ws_sales_price)),item_sk,sumws,sum] + InputAdapter + Exchange [ws_item_sk,d_date] #5 + WholeStageCodegen (1) + HashAggregate [ws_item_sk,d_date,ws_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_item_sk,ws_sales_price,d_date] + CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] + CometFilter [ws_item_sk,ws_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_sales_price,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #6 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] + CometBroadcastExchange [d_date_sk,d_date] #7 + CometProject [d_date_sk,d_date] + CometFilter [d_date_sk,d_date,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_month_seq] InputAdapter BroadcastExchange #8 WholeStageCodegen (7) @@ -70,67 +66,60 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter Window [ws_item_sk,d_date] WholeStageCodegen (6) - ColumnarToRow + Sort [ws_item_sk,d_date] InputAdapter - CometSort [item_sk,d_date,sumws,ws_item_sk] - ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #9 - RowToColumnar - WholeStageCodegen (18) - HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [item_sk,d_date,sumss] #10 - RowToColumnar - WholeStageCodegen (17) + ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 + InputAdapter + WholeStageCodegen (20) + Sort [item_sk,d_date] + InputAdapter + Exchange [item_sk,d_date] #9 + WholeStageCodegen (19) + HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] + InputAdapter + Exchange [item_sk,d_date,sumss] #10 + WholeStageCodegen (18) HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] Project [item_sk,d_date,sumss,sumss] BroadcastHashJoin [item_sk,item_sk,rk,rk] Project [item_sk,d_date,sumss,rk] InputAdapter Window [ss_item_sk,d_date] - WholeStageCodegen (12) - ColumnarToRow + WholeStageCodegen (13) + Sort [ss_item_sk,d_date] InputAdapter - CometSort [item_sk,d_date,sumss,ss_item_sk] - CometColumnarExchange [ss_item_sk] #11 - RowToColumnar - WholeStageCodegen (11) - HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,sumss,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ss_item_sk,d_date] #12 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_sales_price,d_date] - CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] - CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_date] #7 + Exchange [ss_item_sk] #11 + WholeStageCodegen (12) + HashAggregate [ss_item_sk,d_date,sum] [sum(UnscaledValue(ss_sales_price)),item_sk,sumss,sum] + InputAdapter + Exchange [ss_item_sk,d_date] #12 + WholeStageCodegen (11) + HashAggregate [ss_item_sk,d_date,ss_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_sales_price,d_date] + CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] + CometFilter [ss_item_sk,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_sales_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_date] #7 InputAdapter BroadcastExchange #13 - WholeStageCodegen (16) + WholeStageCodegen (17) Project [item_sk,sumss,rk] InputAdapter Window [ss_item_sk,d_date] - WholeStageCodegen (15) - ColumnarToRow + WholeStageCodegen (16) + Sort [ss_item_sk,d_date] InputAdapter - CometSort [item_sk,d_date,sumss,ss_item_sk] - ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #11 + ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #11 InputAdapter BroadcastExchange #14 - WholeStageCodegen (39) + WholeStageCodegen (45) Project [item_sk,web_sales,store_sales,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (38) - ColumnarToRow + WholeStageCodegen (44) + Sort [item_sk,d_date] InputAdapter - CometSort [item_sk,d_date,web_sales,store_sales] - ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 + ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt index 56702651b..e361df2d1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/explain.txt @@ -1,57 +1,49 @@ == Physical Plan == -TakeOrderedAndProject (53) -+- * Project (52) - +- * BroadcastHashJoin Inner BuildRight (51) - :- * Project (44) - : +- * BroadcastHashJoin Inner BuildRight (43) - : :- * Project (32) - : : +- * Filter (31) - : : +- Window (30) - : : +- * Filter (29) - : : +- Window (28) - : : +- * ColumnarToRow (27) - : : +- CometSort (26) - : : +- CometColumnarExchange (25) - : : +- RowToColumnar (24) - : : +- * HashAggregate (23) - : : +- * ColumnarToRow (22) - : : +- CometColumnarExchange (21) - : : +- RowToColumnar (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.item (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.call_center (13) - : +- BroadcastExchange (42) - : +- * Project (41) - : +- Window (40) - : +- * ColumnarToRow (39) - : +- CometSort (38) - : +- CometColumnarExchange (37) - : +- RowToColumnar (36) - : +- * HashAggregate (35) - : +- * ColumnarToRow (34) - : +- ReusedExchange (33) - +- BroadcastExchange (50) - +- * Project (49) - +- Window (48) - +- * ColumnarToRow (47) - +- CometSort (46) - +- ReusedExchange (45) +TakeOrderedAndProject (45) ++- * Project (44) + +- * BroadcastHashJoin Inner BuildRight (43) + :- * Project (37) + : +- * BroadcastHashJoin Inner BuildRight (36) + : :- * Project (28) + : : +- * Filter (27) + : : +- Window (26) + : : +- * Filter (25) + : : +- Window (24) + : : +- * Sort (23) + : : +- Exchange (22) + : : +- * HashAggregate (21) + : : +- Exchange (20) + : : +- * HashAggregate (19) + : : +- * ColumnarToRow (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.item (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.call_center (13) + : +- BroadcastExchange (35) + : +- * Project (34) + : +- Window (33) + : +- * Sort (32) + : +- Exchange (31) + : +- * HashAggregate (30) + : +- ReusedExchange (29) + +- BroadcastExchange (42) + +- * Project (41) + +- Window (40) + +- * Sort (39) + +- ReusedExchange (38) (1) Scan parquet spark_catalog.default.item @@ -148,166 +140,142 @@ Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] Aggregate Attributes [1]: [sum#14] Results [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#15] -(20) RowToColumnar +(20) Exchange Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#15] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(21) CometColumnarExchange -Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(22) ColumnarToRow [codegen id : 2] -Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#15] - -(23) HashAggregate [codegen id : 2] +(21) HashAggregate [codegen id : 2] Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#15] Keys [5]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#6))#16] Results [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#16,17,2) AS sum_sales#17, MakeDecimal(sum(UnscaledValue(cs_sales_price#6))#16,17,2) AS _w0#18] -(24) RowToColumnar -Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18] - -(25) CometColumnarExchange +(22) Exchange Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(26) CometSort +(23) Sort [codegen id : 3] Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18] -Arguments: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18], [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#13 ASC NULLS FIRST, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST] +Arguments: [i_category#3 ASC NULLS FIRST, i_brand#2 ASC NULLS FIRST, cc_name#13 ASC NULLS FIRST, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST], false, 0 -(27) ColumnarToRow [codegen id : 3] -Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18] - -(28) Window +(24) Window Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18] Arguments: [rank(d_year#10, d_moy#11) windowspecdefinition(i_category#3, i_brand#2, cc_name#13, d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#19], [i_category#3, i_brand#2, cc_name#13], [d_year#10 ASC NULLS FIRST, d_moy#11 ASC NULLS FIRST] -(29) Filter [codegen id : 4] +(25) Filter [codegen id : 4] Input [8]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18, rn#19] Condition : (isnotnull(d_year#10) AND (d_year#10 = 1999)) -(30) Window +(26) Window Input [8]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18, rn#19] Arguments: [avg(_w0#18) windowspecdefinition(i_category#3, i_brand#2, cc_name#13, d_year#10, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#20], [i_category#3, i_brand#2, cc_name#13, d_year#10] -(31) Filter [codegen id : 13] +(27) Filter [codegen id : 13] Input [9]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] Condition : ((isnotnull(avg_monthly_sales#20) AND (avg_monthly_sales#20 > 0.000000)) AND CASE WHEN (avg_monthly_sales#20 > 0.000000) THEN ((abs((sum_sales#17 - avg_monthly_sales#20)) / avg_monthly_sales#20) > 0.1000000000000000) END) -(32) Project [codegen id : 13] +(28) Project [codegen id : 13] Output [8]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, avg_monthly_sales#20, rn#19] Input [9]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, _w0#18, rn#19, avg_monthly_sales#20] -(33) ReusedExchange [Reuses operator id: 21] +(29) ReusedExchange [Reuses operator id: 20] Output [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] -(34) ColumnarToRow [codegen id : 6] -Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] - -(35) HashAggregate [codegen id : 6] +(30) HashAggregate [codegen id : 6] Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum#26] Keys [5]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25] Functions [1]: [sum(UnscaledValue(cs_sales_price#27))] Aggregate Attributes [1]: [sum(UnscaledValue(cs_sales_price#27))#16] Results [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, MakeDecimal(sum(UnscaledValue(cs_sales_price#27))#16,17,2) AS sum_sales#28] -(36) RowToColumnar -Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] - -(37) CometColumnarExchange +(31) Exchange Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] -Arguments: hashpartitioning(i_category#21, i_brand#22, cc_name#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_category#21, i_brand#22, cc_name#23, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(38) CometSort +(32) Sort [codegen id : 7] Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] -Arguments: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28], [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, cc_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] +Arguments: [i_category#21 ASC NULLS FIRST, i_brand#22 ASC NULLS FIRST, cc_name#23 ASC NULLS FIRST, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST], false, 0 -(39) ColumnarToRow [codegen id : 7] -Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] - -(40) Window +(33) Window Input [6]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28] Arguments: [rank(d_year#24, d_moy#25) windowspecdefinition(i_category#21, i_brand#22, cc_name#23, d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#29], [i_category#21, i_brand#22, cc_name#23], [d_year#24 ASC NULLS FIRST, d_moy#25 ASC NULLS FIRST] -(41) Project [codegen id : 8] +(34) Project [codegen id : 8] Output [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] Input [7]: [i_category#21, i_brand#22, cc_name#23, d_year#24, d_moy#25, sum_sales#28, rn#29] -(42) BroadcastExchange +(35) BroadcastExchange Input [5]: [i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] + 1)),false), [plan_id=4] -(43) BroadcastHashJoin [codegen id : 13] +(36) BroadcastHashJoin [codegen id : 13] Left keys [4]: [i_category#3, i_brand#2, cc_name#13, rn#19] Right keys [4]: [i_category#21, i_brand#22, cc_name#23, (rn#29 + 1)] Join type: Inner Join condition: None -(44) Project [codegen id : 13] +(37) Project [codegen id : 13] Output [9]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28] Input [13]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, avg_monthly_sales#20, rn#19, i_category#21, i_brand#22, cc_name#23, sum_sales#28, rn#29] -(45) ReusedExchange [Reuses operator id: 37] +(38) ReusedExchange [Reuses operator id: 31] Output [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] -(46) CometSort +(39) Sort [codegen id : 11] Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] -Arguments: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35], [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, cc_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] +Arguments: [i_category#30 ASC NULLS FIRST, i_brand#31 ASC NULLS FIRST, cc_name#32 ASC NULLS FIRST, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST], false, 0 -(47) ColumnarToRow [codegen id : 11] -Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] - -(48) Window +(40) Window Input [6]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35] Arguments: [rank(d_year#33, d_moy#34) windowspecdefinition(i_category#30, i_brand#31, cc_name#32, d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rn#36], [i_category#30, i_brand#31, cc_name#32], [d_year#33 ASC NULLS FIRST, d_moy#34 ASC NULLS FIRST] -(49) Project [codegen id : 12] +(41) Project [codegen id : 12] Output [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] Input [7]: [i_category#30, i_brand#31, cc_name#32, d_year#33, d_moy#34, sum_sales#35, rn#36] -(50) BroadcastExchange +(42) BroadcastExchange Input [5]: [i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] Arguments: HashedRelationBroadcastMode(List(input[0, string, true], input[1, string, true], input[2, string, true], (input[4, int, false] - 1)),false), [plan_id=5] -(51) BroadcastHashJoin [codegen id : 13] +(43) BroadcastHashJoin [codegen id : 13] Left keys [4]: [i_category#3, i_brand#2, cc_name#13, rn#19] Right keys [4]: [i_category#30, i_brand#31, cc_name#32, (rn#36 - 1)] Join type: Inner Join condition: None -(52) Project [codegen id : 13] +(44) Project [codegen id : 13] Output [8]: [i_category#3, i_brand#2, d_year#10, d_moy#11, avg_monthly_sales#20, sum_sales#17, sum_sales#28 AS psum#37, sum_sales#35 AS nsum#38] Input [14]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#17, avg_monthly_sales#20, rn#19, sum_sales#28, i_category#30, i_brand#31, cc_name#32, sum_sales#35, rn#36] -(53) TakeOrderedAndProject +(45) TakeOrderedAndProject Input [8]: [i_category#3, i_brand#2, d_year#10, d_moy#11, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] Arguments: 100, [(sum_sales#17 - avg_monthly_sales#20) ASC NULLS FIRST, d_year#10 ASC NULLS FIRST], [i_category#3, i_brand#2, d_year#10, d_moy#11, avg_monthly_sales#20, sum_sales#17, psum#37, nsum#38] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = cs_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (57) -+- * ColumnarToRow (56) - +- CometFilter (55) - +- CometScan parquet spark_catalog.default.date_dim (54) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(54) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_moy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [Or(Or(EqualTo(d_year,1999),And(EqualTo(d_year,1998),EqualTo(d_moy,12))),And(EqualTo(d_year,2000),EqualTo(d_moy,1))), IsNotNull(d_date_sk)] ReadSchema: struct -(55) CometFilter +(47) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Condition : ((((d_year#10 = 1999) OR ((d_year#10 = 1998) AND (d_moy#11 = 12))) OR ((d_year#10 = 2000) AND (d_moy#11 = 1))) AND isnotnull(d_date_sk#9)) -(56) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#9, d_year#10, d_moy#11] -(57) BroadcastExchange +(49) BroadcastExchange Input [3]: [d_date_sk#9, d_year#10, d_moy#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt index c4518b4f1..daeafbf92 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q57/simplified.txt @@ -13,45 +13,41 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (3) - ColumnarToRow + Sort [i_category,i_brand,cc_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,cc_name] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] - CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] - CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] - CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometFilter [i_item_sk,i_brand,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] - CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 - CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometBroadcastExchange [cc_call_center_sk,cc_name] #6 - CometFilter [cc_call_center_sk,cc_name] - CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] + Exchange [i_category,i_brand,cc_name] #1 + WholeStageCodegen (2) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,_w0,sum] + InputAdapter + Exchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + WholeStageCodegen (1) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,cs_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy] + CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year,d_moy] + CometProject [i_brand,i_category,cs_call_center_sk,cs_sales_price,cs_sold_date_sk] + CometBroadcastHashJoin [i_item_sk,i_brand,i_category,cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometFilter [i_item_sk,i_brand,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_category] + CometBroadcastExchange [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] #3 + CometFilter [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_item_sk,cs_sales_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [d_date_sk,d_year,d_moy] #5 + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometBroadcastExchange [cc_call_center_sk,cc_name] #6 + CometFilter [cc_call_center_sk,cc_name] + CometScan parquet spark_catalog.default.call_center [cc_call_center_sk,cc_name] InputAdapter BroadcastExchange #7 WholeStageCodegen (8) @@ -59,16 +55,13 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (7) - ColumnarToRow + Sort [i_category,i_brand,cc_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - CometColumnarExchange [i_category,i_brand,cc_name] #8 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 + Exchange [i_category,i_brand,cc_name] #8 + WholeStageCodegen (6) + HashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum] [sum(UnscaledValue(cs_sales_price)),sum_sales,sum] + InputAdapter + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum] #2 InputAdapter BroadcastExchange #9 WholeStageCodegen (12) @@ -76,7 +69,6 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m InputAdapter Window [d_year,d_moy,i_category,i_brand,cc_name] WholeStageCodegen (11) - ColumnarToRow + Sort [i_category,i_brand,cc_name,d_year,d_moy] InputAdapter - CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] - ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8 + ReusedExchange [i_category,i_brand,cc_name,d_year,d_moy,sum_sales] #8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt index b4abced27..9d660d776 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/explain.txt @@ -1,103 +1,87 @@ == Physical Plan == -* ColumnarToRow (99) -+- CometTakeOrderedAndProject (98) - +- CometHashAggregate (97) - +- CometColumnarExchange (96) - +- RowToColumnar (95) - +- * HashAggregate (94) - +- Union (93) - :- * HashAggregate (76) - : +- * ColumnarToRow (75) - : +- CometColumnarExchange (74) - : +- RowToColumnar (73) - : +- * HashAggregate (72) - : +- Union (71) - : :- * HashAggregate (24) - : : +- * ColumnarToRow (23) - : : +- CometColumnarExchange (22) - : : +- RowToColumnar (21) - : : +- * HashAggregate (20) - : : +- * ColumnarToRow (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometUnion (7) - : : : : :- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.store_returns (4) - : : : +- CometBroadcastExchange (11) - : : : +- CometProject (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometScan parquet spark_catalog.default.store (14) - : :- * HashAggregate (45) - : : +- * ColumnarToRow (44) - : : +- CometColumnarExchange (43) - : : +- RowToColumnar (42) - : : +- * HashAggregate (41) - : : +- * ColumnarToRow (40) - : : +- CometProject (39) - : : +- CometBroadcastHashJoin (38) - : : :- CometProject (34) - : : : +- CometBroadcastHashJoin (33) - : : : :- CometUnion (31) - : : : : :- CometProject (27) - : : : : : +- CometFilter (26) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (25) - : : : : +- CometProject (30) - : : : : +- CometFilter (29) - : : : : +- CometScan parquet spark_catalog.default.catalog_returns (28) - : : : +- ReusedExchange (32) - : : +- CometBroadcastExchange (37) - : : +- CometFilter (36) - : : +- CometScan parquet spark_catalog.default.catalog_page (35) - : +- * HashAggregate (70) - : +- * ColumnarToRow (69) - : +- CometColumnarExchange (68) - : +- RowToColumnar (67) - : +- * HashAggregate (66) - : +- * ColumnarToRow (65) - : +- CometProject (64) - : +- CometBroadcastHashJoin (63) - : :- CometProject (59) - : : +- CometBroadcastHashJoin (58) - : : :- CometUnion (56) - : : : :- CometProject (48) - : : : : +- CometFilter (47) - : : : : +- CometScan parquet spark_catalog.default.web_sales (46) - : : : +- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometBroadcastExchange (50) - : : : : +- CometScan parquet spark_catalog.default.web_returns (49) - : : : +- CometProject (53) - : : : +- CometFilter (52) - : : : +- CometScan parquet spark_catalog.default.web_sales (51) - : : +- ReusedExchange (57) - : +- CometBroadcastExchange (62) - : +- CometFilter (61) - : +- CometScan parquet spark_catalog.default.web_site (60) - :- * HashAggregate (84) - : +- * ColumnarToRow (83) - : +- CometColumnarExchange (82) - : +- RowToColumnar (81) - : +- * HashAggregate (80) - : +- * HashAggregate (79) - : +- * ColumnarToRow (78) - : +- ReusedExchange (77) - +- * HashAggregate (92) - +- * ColumnarToRow (91) - +- CometColumnarExchange (90) - +- RowToColumnar (89) - +- * HashAggregate (88) - +- * HashAggregate (87) - +- * ColumnarToRow (86) - +- ReusedExchange (85) +TakeOrderedAndProject (83) ++- * HashAggregate (82) + +- Exchange (81) + +- * HashAggregate (80) + +- Union (79) + :- * HashAggregate (68) + : +- Exchange (67) + : +- * HashAggregate (66) + : +- Union (65) + : :- * HashAggregate (22) + : : +- Exchange (21) + : : +- * HashAggregate (20) + : : +- * ColumnarToRow (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometUnion (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan parquet spark_catalog.default.store (14) + : :- * HashAggregate (41) + : : +- Exchange (40) + : : +- * HashAggregate (39) + : : +- * ColumnarToRow (38) + : : +- CometProject (37) + : : +- CometBroadcastHashJoin (36) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometUnion (29) + : : : : :- CometProject (25) + : : : : : +- CometFilter (24) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) + : : : : +- CometProject (28) + : : : : +- CometFilter (27) + : : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) + : : : +- ReusedExchange (30) + : : +- CometBroadcastExchange (35) + : : +- CometFilter (34) + : : +- CometScan parquet spark_catalog.default.catalog_page (33) + : +- * HashAggregate (64) + : +- Exchange (63) + : +- * HashAggregate (62) + : +- * ColumnarToRow (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometUnion (52) + : : : :- CometProject (44) + : : : : +- CometFilter (43) + : : : : +- CometScan parquet spark_catalog.default.web_sales (42) + : : : +- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastExchange (46) + : : : : +- CometScan parquet spark_catalog.default.web_returns (45) + : : : +- CometProject (49) + : : : +- CometFilter (48) + : : : +- CometScan parquet spark_catalog.default.web_sales (47) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometScan parquet spark_catalog.default.web_site (56) + :- * HashAggregate (73) + : +- Exchange (72) + : +- * HashAggregate (71) + : +- * HashAggregate (70) + : +- ReusedExchange (69) + +- * HashAggregate (78) + +- Exchange (77) + +- * HashAggregate (76) + +- * HashAggregate (75) + +- ReusedExchange (74) (1) Scan parquet spark_catalog.default.store_sales @@ -198,24 +182,18 @@ Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledV Aggregate Attributes [4]: [sum#26, sum#27, sum#28, sum#29] Results [5]: [s_store_id#25, sum#30, sum#31, sum#32, sum#33] -(21) RowToColumnar +(21) Exchange Input [5]: [s_store_id#25, sum#30, sum#31, sum#32, sum#33] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(22) CometColumnarExchange -Input [5]: [s_store_id#25, sum#30, sum#31, sum#32, sum#33] -Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(23) ColumnarToRow [codegen id : 2] -Input [5]: [s_store_id#25, sum#30, sum#31, sum#32, sum#33] - -(24) HashAggregate [codegen id : 2] +(22) HashAggregate [codegen id : 2] Input [5]: [s_store_id#25, sum#30, sum#31, sum#32, sum#33] Keys [1]: [s_store_id#25] Functions [4]: [sum(UnscaledValue(sales_price#8)), sum(UnscaledValue(return_amt#10)), sum(UnscaledValue(profit#9)), sum(UnscaledValue(net_loss#11))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#8))#34, sum(UnscaledValue(return_amt#10))#35, sum(UnscaledValue(profit#9))#36, sum(UnscaledValue(net_loss#11))#37] Results [5]: [store channel AS channel#38, concat(store, s_store_id#25) AS id#39, MakeDecimal(sum(UnscaledValue(sales_price#8))#34,17,2) AS sales#40, MakeDecimal(sum(UnscaledValue(return_amt#10))#35,17,2) AS returns#41, (MakeDecimal(sum(UnscaledValue(profit#9))#36,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#11))#37,17,2)) AS profit#42] -(25) Scan parquet spark_catalog.default.catalog_sales +(23) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_catalog_page_sk#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Batched: true Location: InMemoryFileIndex [] @@ -223,15 +201,15 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#46), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_catalog_page_sk)] ReadSchema: struct -(26) CometFilter +(24) CometFilter Input [4]: [cs_catalog_page_sk#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Condition : isnotnull(cs_catalog_page_sk#43) -(27) CometProject +(25) CometProject Input [4]: [cs_catalog_page_sk#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Arguments: [page_sk#48, date_sk#49, sales_price#50, profit#51, return_amt#52, net_loss#53], [cs_catalog_page_sk#43 AS page_sk#48, cs_sold_date_sk#46 AS date_sk#49, cs_ext_sales_price#44 AS sales_price#50, cs_net_profit#45 AS profit#51, 0.00 AS return_amt#52, 0.00 AS net_loss#53] -(28) Scan parquet spark_catalog.default.catalog_returns +(26) Scan parquet spark_catalog.default.catalog_returns Output [4]: [cr_catalog_page_sk#54, cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57] Batched: true Location: InMemoryFileIndex [] @@ -239,82 +217,76 @@ PartitionFilters: [isnotnull(cr_returned_date_sk#57), dynamicpruningexpression(c PushedFilters: [IsNotNull(cr_catalog_page_sk)] ReadSchema: struct -(29) CometFilter +(27) CometFilter Input [4]: [cr_catalog_page_sk#54, cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57] Condition : isnotnull(cr_catalog_page_sk#54) -(30) CometProject +(28) CometProject Input [4]: [cr_catalog_page_sk#54, cr_return_amount#55, cr_net_loss#56, cr_returned_date_sk#57] Arguments: [page_sk#58, date_sk#59, sales_price#60, profit#61, return_amt#62, net_loss#63], [cr_catalog_page_sk#54 AS page_sk#58, cr_returned_date_sk#57 AS date_sk#59, 0.00 AS sales_price#60, 0.00 AS profit#61, cr_return_amount#55 AS return_amt#62, cr_net_loss#56 AS net_loss#63] -(31) CometUnion +(29) CometUnion Child 0 Input [6]: [page_sk#48, date_sk#49, sales_price#50, profit#51, return_amt#52, net_loss#53] Child 1 Input [6]: [page_sk#58, date_sk#59, sales_price#60, profit#61, return_amt#62, net_loss#63] -(32) ReusedExchange [Reuses operator id: 11] +(30) ReusedExchange [Reuses operator id: 11] Output [1]: [d_date_sk#64] -(33) CometBroadcastHashJoin +(31) CometBroadcastHashJoin Left output [6]: [page_sk#48, date_sk#49, sales_price#50, profit#51, return_amt#52, net_loss#53] Right output [1]: [d_date_sk#64] Arguments: [date_sk#49], [d_date_sk#64], Inner, BuildRight -(34) CometProject +(32) CometProject Input [7]: [page_sk#48, date_sk#49, sales_price#50, profit#51, return_amt#52, net_loss#53, d_date_sk#64] Arguments: [page_sk#48, sales_price#50, profit#51, return_amt#52, net_loss#53], [page_sk#48, sales_price#50, profit#51, return_amt#52, net_loss#53] -(35) Scan parquet spark_catalog.default.catalog_page +(33) Scan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#65, cp_catalog_page_id#66] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(36) CometFilter +(34) CometFilter Input [2]: [cp_catalog_page_sk#65, cp_catalog_page_id#66] Condition : isnotnull(cp_catalog_page_sk#65) -(37) CometBroadcastExchange +(35) CometBroadcastExchange Input [2]: [cp_catalog_page_sk#65, cp_catalog_page_id#66] Arguments: [cp_catalog_page_sk#65, cp_catalog_page_id#66] -(38) CometBroadcastHashJoin +(36) CometBroadcastHashJoin Left output [5]: [page_sk#48, sales_price#50, profit#51, return_amt#52, net_loss#53] Right output [2]: [cp_catalog_page_sk#65, cp_catalog_page_id#66] Arguments: [page_sk#48], [cp_catalog_page_sk#65], Inner, BuildRight -(39) CometProject +(37) CometProject Input [7]: [page_sk#48, sales_price#50, profit#51, return_amt#52, net_loss#53, cp_catalog_page_sk#65, cp_catalog_page_id#66] Arguments: [sales_price#50, profit#51, return_amt#52, net_loss#53, cp_catalog_page_id#66], [sales_price#50, profit#51, return_amt#52, net_loss#53, cp_catalog_page_id#66] -(40) ColumnarToRow [codegen id : 3] +(38) ColumnarToRow [codegen id : 3] Input [5]: [sales_price#50, profit#51, return_amt#52, net_loss#53, cp_catalog_page_id#66] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [5]: [sales_price#50, profit#51, return_amt#52, net_loss#53, cp_catalog_page_id#66] Keys [1]: [cp_catalog_page_id#66] Functions [4]: [partial_sum(UnscaledValue(sales_price#50)), partial_sum(UnscaledValue(return_amt#52)), partial_sum(UnscaledValue(profit#51)), partial_sum(UnscaledValue(net_loss#53))] Aggregate Attributes [4]: [sum#67, sum#68, sum#69, sum#70] Results [5]: [cp_catalog_page_id#66, sum#71, sum#72, sum#73, sum#74] -(42) RowToColumnar -Input [5]: [cp_catalog_page_id#66, sum#71, sum#72, sum#73, sum#74] - -(43) CometColumnarExchange -Input [5]: [cp_catalog_page_id#66, sum#71, sum#72, sum#73, sum#74] -Arguments: hashpartitioning(cp_catalog_page_id#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [cp_catalog_page_id#66, sum#71, sum#72, sum#73, sum#74] +Arguments: hashpartitioning(cp_catalog_page_id#66, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [cp_catalog_page_id#66, sum#71, sum#72, sum#73, sum#74] Keys [1]: [cp_catalog_page_id#66] Functions [4]: [sum(UnscaledValue(sales_price#50)), sum(UnscaledValue(return_amt#52)), sum(UnscaledValue(profit#51)), sum(UnscaledValue(net_loss#53))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#50))#75, sum(UnscaledValue(return_amt#52))#76, sum(UnscaledValue(profit#51))#77, sum(UnscaledValue(net_loss#53))#78] Results [5]: [catalog channel AS channel#79, concat(catalog_page, cp_catalog_page_id#66) AS id#80, MakeDecimal(sum(UnscaledValue(sales_price#50))#75,17,2) AS sales#81, MakeDecimal(sum(UnscaledValue(return_amt#52))#76,17,2) AS returns#82, (MakeDecimal(sum(UnscaledValue(profit#51))#77,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#53))#78,17,2)) AS profit#83] -(46) Scan parquet spark_catalog.default.web_sales +(42) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_web_site_sk#84, ws_ext_sales_price#85, ws_net_profit#86, ws_sold_date_sk#87] Batched: true Location: InMemoryFileIndex [] @@ -322,284 +294,250 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#87), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_site_sk)] ReadSchema: struct -(47) CometFilter +(43) CometFilter Input [4]: [ws_web_site_sk#84, ws_ext_sales_price#85, ws_net_profit#86, ws_sold_date_sk#87] Condition : isnotnull(ws_web_site_sk#84) -(48) CometProject +(44) CometProject Input [4]: [ws_web_site_sk#84, ws_ext_sales_price#85, ws_net_profit#86, ws_sold_date_sk#87] Arguments: [wsr_web_site_sk#89, date_sk#90, sales_price#91, profit#92, return_amt#93, net_loss#94], [ws_web_site_sk#84 AS wsr_web_site_sk#89, ws_sold_date_sk#87 AS date_sk#90, ws_ext_sales_price#85 AS sales_price#91, ws_net_profit#86 AS profit#92, 0.00 AS return_amt#93, 0.00 AS net_loss#94] -(49) Scan parquet spark_catalog.default.web_returns +(45) Scan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#95, wr_order_number#96, wr_return_amt#97, wr_net_loss#98, wr_returned_date_sk#99] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(wr_returned_date_sk#99), dynamicpruningexpression(wr_returned_date_sk#99 IN dynamicpruning#88)] ReadSchema: struct -(50) CometBroadcastExchange +(46) CometBroadcastExchange Input [5]: [wr_item_sk#95, wr_order_number#96, wr_return_amt#97, wr_net_loss#98, wr_returned_date_sk#99] Arguments: [wr_item_sk#95, wr_order_number#96, wr_return_amt#97, wr_net_loss#98, wr_returned_date_sk#99] -(51) Scan parquet spark_catalog.default.web_sales +(47) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102, ws_sold_date_sk#103] Batched: true Location [not included in comparison]/{warehouse_dir}/web_sales] PushedFilters: [IsNotNull(ws_item_sk), IsNotNull(ws_order_number), IsNotNull(ws_web_site_sk)] ReadSchema: struct -(52) CometFilter +(48) CometFilter Input [4]: [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102, ws_sold_date_sk#103] Condition : ((isnotnull(ws_item_sk#100) AND isnotnull(ws_order_number#102)) AND isnotnull(ws_web_site_sk#101)) -(53) CometProject +(49) CometProject Input [4]: [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102, ws_sold_date_sk#103] Arguments: [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102], [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102] -(54) CometBroadcastHashJoin +(50) CometBroadcastHashJoin Left output [5]: [wr_item_sk#95, wr_order_number#96, wr_return_amt#97, wr_net_loss#98, wr_returned_date_sk#99] Right output [3]: [ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102] Arguments: [wr_item_sk#95, wr_order_number#96], [ws_item_sk#100, ws_order_number#102], Inner, BuildLeft -(55) CometProject +(51) CometProject Input [8]: [wr_item_sk#95, wr_order_number#96, wr_return_amt#97, wr_net_loss#98, wr_returned_date_sk#99, ws_item_sk#100, ws_web_site_sk#101, ws_order_number#102] Arguments: [wsr_web_site_sk#104, date_sk#105, sales_price#106, profit#107, return_amt#108, net_loss#109], [ws_web_site_sk#101 AS wsr_web_site_sk#104, wr_returned_date_sk#99 AS date_sk#105, 0.00 AS sales_price#106, 0.00 AS profit#107, wr_return_amt#97 AS return_amt#108, wr_net_loss#98 AS net_loss#109] -(56) CometUnion +(52) CometUnion Child 0 Input [6]: [wsr_web_site_sk#89, date_sk#90, sales_price#91, profit#92, return_amt#93, net_loss#94] Child 1 Input [6]: [wsr_web_site_sk#104, date_sk#105, sales_price#106, profit#107, return_amt#108, net_loss#109] -(57) ReusedExchange [Reuses operator id: 11] +(53) ReusedExchange [Reuses operator id: 11] Output [1]: [d_date_sk#110] -(58) CometBroadcastHashJoin +(54) CometBroadcastHashJoin Left output [6]: [wsr_web_site_sk#89, date_sk#90, sales_price#91, profit#92, return_amt#93, net_loss#94] Right output [1]: [d_date_sk#110] Arguments: [date_sk#90], [d_date_sk#110], Inner, BuildRight -(59) CometProject +(55) CometProject Input [7]: [wsr_web_site_sk#89, date_sk#90, sales_price#91, profit#92, return_amt#93, net_loss#94, d_date_sk#110] Arguments: [wsr_web_site_sk#89, sales_price#91, profit#92, return_amt#93, net_loss#94], [wsr_web_site_sk#89, sales_price#91, profit#92, return_amt#93, net_loss#94] -(60) Scan parquet spark_catalog.default.web_site +(56) Scan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#111, web_site_id#112] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(61) CometFilter +(57) CometFilter Input [2]: [web_site_sk#111, web_site_id#112] Condition : isnotnull(web_site_sk#111) -(62) CometBroadcastExchange +(58) CometBroadcastExchange Input [2]: [web_site_sk#111, web_site_id#112] Arguments: [web_site_sk#111, web_site_id#112] -(63) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [5]: [wsr_web_site_sk#89, sales_price#91, profit#92, return_amt#93, net_loss#94] Right output [2]: [web_site_sk#111, web_site_id#112] Arguments: [wsr_web_site_sk#89], [web_site_sk#111], Inner, BuildRight -(64) CometProject +(60) CometProject Input [7]: [wsr_web_site_sk#89, sales_price#91, profit#92, return_amt#93, net_loss#94, web_site_sk#111, web_site_id#112] Arguments: [sales_price#91, profit#92, return_amt#93, net_loss#94, web_site_id#112], [sales_price#91, profit#92, return_amt#93, net_loss#94, web_site_id#112] -(65) ColumnarToRow [codegen id : 5] +(61) ColumnarToRow [codegen id : 5] Input [5]: [sales_price#91, profit#92, return_amt#93, net_loss#94, web_site_id#112] -(66) HashAggregate [codegen id : 5] +(62) HashAggregate [codegen id : 5] Input [5]: [sales_price#91, profit#92, return_amt#93, net_loss#94, web_site_id#112] Keys [1]: [web_site_id#112] Functions [4]: [partial_sum(UnscaledValue(sales_price#91)), partial_sum(UnscaledValue(return_amt#93)), partial_sum(UnscaledValue(profit#92)), partial_sum(UnscaledValue(net_loss#94))] Aggregate Attributes [4]: [sum#113, sum#114, sum#115, sum#116] Results [5]: [web_site_id#112, sum#117, sum#118, sum#119, sum#120] -(67) RowToColumnar +(63) Exchange Input [5]: [web_site_id#112, sum#117, sum#118, sum#119, sum#120] +Arguments: hashpartitioning(web_site_id#112, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(68) CometColumnarExchange -Input [5]: [web_site_id#112, sum#117, sum#118, sum#119, sum#120] -Arguments: hashpartitioning(web_site_id#112, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(69) ColumnarToRow [codegen id : 6] -Input [5]: [web_site_id#112, sum#117, sum#118, sum#119, sum#120] - -(70) HashAggregate [codegen id : 6] +(64) HashAggregate [codegen id : 6] Input [5]: [web_site_id#112, sum#117, sum#118, sum#119, sum#120] Keys [1]: [web_site_id#112] Functions [4]: [sum(UnscaledValue(sales_price#91)), sum(UnscaledValue(return_amt#93)), sum(UnscaledValue(profit#92)), sum(UnscaledValue(net_loss#94))] Aggregate Attributes [4]: [sum(UnscaledValue(sales_price#91))#121, sum(UnscaledValue(return_amt#93))#122, sum(UnscaledValue(profit#92))#123, sum(UnscaledValue(net_loss#94))#124] Results [5]: [web channel AS channel#125, concat(web_site, web_site_id#112) AS id#126, MakeDecimal(sum(UnscaledValue(sales_price#91))#121,17,2) AS sales#127, MakeDecimal(sum(UnscaledValue(return_amt#93))#122,17,2) AS returns#128, (MakeDecimal(sum(UnscaledValue(profit#92))#123,17,2) - MakeDecimal(sum(UnscaledValue(net_loss#94))#124,17,2)) AS profit#129] -(71) Union +(65) Union -(72) HashAggregate [codegen id : 7] +(66) HashAggregate [codegen id : 7] Input [5]: [channel#38, id#39, sales#40, returns#41, profit#42] Keys [2]: [channel#38, id#39] Functions [3]: [partial_sum(sales#40), partial_sum(returns#41), partial_sum(profit#42)] Aggregate Attributes [6]: [sum#130, isEmpty#131, sum#132, isEmpty#133, sum#134, isEmpty#135] Results [8]: [channel#38, id#39, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -(73) RowToColumnar -Input [8]: [channel#38, id#39, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] - -(74) CometColumnarExchange -Input [8]: [channel#38, id#39, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] -Arguments: hashpartitioning(channel#38, id#39, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(75) ColumnarToRow [codegen id : 8] +(67) Exchange Input [8]: [channel#38, id#39, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] +Arguments: hashpartitioning(channel#38, id#39, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(76) HashAggregate [codegen id : 8] +(68) HashAggregate [codegen id : 8] Input [8]: [channel#38, id#39, sum#136, isEmpty#137, sum#138, isEmpty#139, sum#140, isEmpty#141] Keys [2]: [channel#38, id#39] Functions [3]: [sum(sales#40), sum(returns#41), sum(profit#42)] Aggregate Attributes [3]: [sum(sales#40)#142, sum(returns#41)#143, sum(profit#42)#144] Results [5]: [channel#38, id#39, cast(sum(sales#40)#142 as decimal(37,2)) AS sales#145, cast(sum(returns#41)#143 as decimal(37,2)) AS returns#146, cast(sum(profit#42)#144 as decimal(38,2)) AS profit#147] -(77) ReusedExchange [Reuses operator id: 74] +(69) ReusedExchange [Reuses operator id: 67] Output [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] -(78) ColumnarToRow [codegen id : 16] -Input [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] - -(79) HashAggregate [codegen id : 16] +(70) HashAggregate [codegen id : 16] Input [8]: [channel#148, id#149, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] Keys [2]: [channel#148, id#149] Functions [3]: [sum(sales#156), sum(returns#157), sum(profit#158)] Aggregate Attributes [3]: [sum(sales#156)#142, sum(returns#157)#143, sum(profit#158)#144] Results [4]: [channel#148, sum(sales#156)#142 AS sales#159, sum(returns#157)#143 AS returns#160, sum(profit#158)#144 AS profit#161] -(80) HashAggregate [codegen id : 16] +(71) HashAggregate [codegen id : 16] Input [4]: [channel#148, sales#159, returns#160, profit#161] Keys [1]: [channel#148] Functions [3]: [partial_sum(sales#159), partial_sum(returns#160), partial_sum(profit#161)] Aggregate Attributes [6]: [sum#162, isEmpty#163, sum#164, isEmpty#165, sum#166, isEmpty#167] Results [7]: [channel#148, sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] -(81) RowToColumnar -Input [7]: [channel#148, sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] - -(82) CometColumnarExchange +(72) Exchange Input [7]: [channel#148, sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] -Arguments: hashpartitioning(channel#148, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(channel#148, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(83) ColumnarToRow [codegen id : 17] -Input [7]: [channel#148, sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] - -(84) HashAggregate [codegen id : 17] +(73) HashAggregate [codegen id : 17] Input [7]: [channel#148, sum#168, isEmpty#169, sum#170, isEmpty#171, sum#172, isEmpty#173] Keys [1]: [channel#148] Functions [3]: [sum(sales#159), sum(returns#160), sum(profit#161)] Aggregate Attributes [3]: [sum(sales#159)#174, sum(returns#160)#175, sum(profit#161)#176] Results [5]: [channel#148, null AS id#177, sum(sales#159)#174 AS sum(sales)#178, sum(returns#160)#175 AS sum(returns)#179, sum(profit#161)#176 AS sum(profit)#180] -(85) ReusedExchange [Reuses operator id: 74] +(74) ReusedExchange [Reuses operator id: 67] Output [8]: [channel#181, id#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] -(86) ColumnarToRow [codegen id : 25] -Input [8]: [channel#181, id#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] - -(87) HashAggregate [codegen id : 25] +(75) HashAggregate [codegen id : 25] Input [8]: [channel#181, id#182, sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] Keys [2]: [channel#181, id#182] Functions [3]: [sum(sales#189), sum(returns#190), sum(profit#191)] Aggregate Attributes [3]: [sum(sales#189)#142, sum(returns#190)#143, sum(profit#191)#144] Results [3]: [sum(sales#189)#142 AS sales#192, sum(returns#190)#143 AS returns#193, sum(profit#191)#144 AS profit#194] -(88) HashAggregate [codegen id : 25] +(76) HashAggregate [codegen id : 25] Input [3]: [sales#192, returns#193, profit#194] Keys: [] Functions [3]: [partial_sum(sales#192), partial_sum(returns#193), partial_sum(profit#194)] Aggregate Attributes [6]: [sum#195, isEmpty#196, sum#197, isEmpty#198, sum#199, isEmpty#200] Results [6]: [sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206] -(89) RowToColumnar +(77) Exchange Input [6]: [sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(90) CometColumnarExchange -Input [6]: [sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(91) ColumnarToRow [codegen id : 26] -Input [6]: [sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206] - -(92) HashAggregate [codegen id : 26] +(78) HashAggregate [codegen id : 26] Input [6]: [sum#201, isEmpty#202, sum#203, isEmpty#204, sum#205, isEmpty#206] Keys: [] Functions [3]: [sum(sales#192), sum(returns#193), sum(profit#194)] Aggregate Attributes [3]: [sum(sales#192)#207, sum(returns#193)#208, sum(profit#194)#209] Results [5]: [null AS channel#210, null AS id#211, sum(sales#192)#207 AS sum(sales)#212, sum(returns#193)#208 AS sum(returns)#213, sum(profit#194)#209 AS sum(profit)#214] -(93) Union +(79) Union -(94) HashAggregate [codegen id : 27] +(80) HashAggregate [codegen id : 27] Input [5]: [channel#38, id#39, sales#145, returns#146, profit#147] Keys [5]: [channel#38, id#39, sales#145, returns#146, profit#147] Functions: [] Aggregate Attributes: [] Results [5]: [channel#38, id#39, sales#145, returns#146, profit#147] -(95) RowToColumnar -Input [5]: [channel#38, id#39, sales#145, returns#146, profit#147] - -(96) CometColumnarExchange +(81) Exchange Input [5]: [channel#38, id#39, sales#145, returns#146, profit#147] -Arguments: hashpartitioning(channel#38, id#39, sales#145, returns#146, profit#147, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(channel#38, id#39, sales#145, returns#146, profit#147, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(97) CometHashAggregate +(82) HashAggregate [codegen id : 28] Input [5]: [channel#38, id#39, sales#145, returns#146, profit#147] Keys [5]: [channel#38, id#39, sales#145, returns#146, profit#147] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#38, id#39, sales#145, returns#146, profit#147] -(98) CometTakeOrderedAndProject -Input [5]: [channel#38, id#39, sales#145, returns#146, profit#147] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#38 ASC NULLS FIRST,id#39 ASC NULLS FIRST], output=[channel#38,id#39,sales#145,returns#146,profit#147]), [channel#38, id#39, sales#145, returns#146, profit#147], 100, [channel#38 ASC NULLS FIRST, id#39 ASC NULLS FIRST], [channel#38, id#39, sales#145, returns#146, profit#147] - -(99) ColumnarToRow [codegen id : 28] +(83) TakeOrderedAndProject Input [5]: [channel#38, id#39, sales#145, returns#146, profit#147] +Arguments: 100, [channel#38 ASC NULLS FIRST, id#39 ASC NULLS FIRST], [channel#38, id#39, sales#145, returns#146, profit#147] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (104) -+- * ColumnarToRow (103) - +- CometProject (102) - +- CometFilter (101) - +- CometScan parquet spark_catalog.default.date_dim (100) +BroadcastExchange (88) ++- * ColumnarToRow (87) + +- CometProject (86) + +- CometFilter (85) + +- CometScan parquet spark_catalog.default.date_dim (84) -(100) Scan parquet spark_catalog.default.date_dim +(84) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] ReadSchema: struct -(101) CometFilter +(85) CometFilter Input [2]: [d_date_sk#22, d_date#23] Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-08-18)) AND isnotnull(d_date_sk#22)) -(102) CometProject +(86) CometProject Input [2]: [d_date_sk#22, d_date#23] Arguments: [d_date_sk#22], [d_date_sk#22] -(103) ColumnarToRow [codegen id : 1] +(87) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#22] -(104) BroadcastExchange +(88) BroadcastExchange Input [1]: [d_date_sk#22] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] Subquery:2 Hosting operator id = 4 Hosting Expression = sr_returned_date_sk#15 IN dynamicpruning#5 -Subquery:3 Hosting operator id = 25 Hosting Expression = cs_sold_date_sk#46 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 23 Hosting Expression = cs_sold_date_sk#46 IN dynamicpruning#5 -Subquery:4 Hosting operator id = 28 Hosting Expression = cr_returned_date_sk#57 IN dynamicpruning#5 +Subquery:4 Hosting operator id = 26 Hosting Expression = cr_returned_date_sk#57 IN dynamicpruning#5 -Subquery:5 Hosting operator id = 46 Hosting Expression = ws_sold_date_sk#87 IN dynamicpruning#5 +Subquery:5 Hosting operator id = 42 Hosting Expression = ws_sold_date_sk#87 IN dynamicpruning#5 -Subquery:6 Hosting operator id = 49 Hosting Expression = wr_returned_date_sk#99 IN dynamicpruning#5 +Subquery:6 Hosting operator id = 45 Hosting Expression = wr_returned_date_sk#99 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt index 733975649..c3b00538b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q5a/simplified.txt @@ -1,140 +1,124 @@ -WholeStageCodegen (28) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,id,sales,returns,profit] - CometHashAggregate [channel,id,sales,returns,profit] - CometColumnarExchange [channel,id,sales,returns,profit] #1 - RowToColumnar - WholeStageCodegen (27) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (8) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id] #2 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (28) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Exchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (27) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (8) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel,id] #2 + WholeStageCodegen (7) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (2) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + Exchange [s_store_id] #3 + WholeStageCodegen (1) + HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] ColumnarToRow InputAdapter - CometColumnarExchange [s_store_id] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_store_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (4) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (4) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + InputAdapter + Exchange [cp_catalog_page_id] #7 + WholeStageCodegen (3) + HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] ColumnarToRow InputAdapter - CometColumnarExchange [cp_catalog_page_id] #7 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [cp_catalog_page_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [page_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #8 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (6) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #8 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (6) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + InputAdapter + Exchange [web_site_id] #9 + WholeStageCodegen (5) + HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] ColumnarToRow InputAdapter - CometColumnarExchange [web_site_id] #9 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [web_site_id,sales_price,return_amt,profit,net_loss] [sum,sum,sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #10 - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [web_site_sk,web_site_id] #11 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - WholeStageCodegen (17) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel] #12 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (26) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #13 - RowToColumnar - WholeStageCodegen (25) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #10 + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #11 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + WholeStageCodegen (17) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel] #12 + WholeStageCodegen (16) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (26) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange #13 + WholeStageCodegen (25) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt index 4e037389f..4a043872d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/explain.txt @@ -1,48 +1,44 @@ == Physical Plan == -TakeOrderedAndProject (44) -+- * Filter (43) - +- * HashAggregate (42) - +- * ColumnarToRow (41) - +- CometColumnarExchange (40) - +- RowToColumnar (39) - +- * HashAggregate (38) - +- * Project (37) - +- * BroadcastHashJoin Inner BuildRight (36) - :- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (12) - : : +- CometBroadcastHashJoin (11) - : : :- CometProject (7) - : : : +- CometBroadcastHashJoin (6) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.customer_address (1) - : : : +- CometBroadcastExchange (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.customer (3) - : : +- CometBroadcastExchange (10) - : : +- CometFilter (9) - : : +- CometScan parquet spark_catalog.default.store_sales (8) - : +- CometBroadcastExchange (16) - : +- CometProject (15) - : +- CometFilter (14) - : +- CometScan parquet spark_catalog.default.date_dim (13) - +- BroadcastExchange (35) - +- * Project (34) - +- * BroadcastHashJoin Inner BuildRight (33) - :- * ColumnarToRow (22) - : +- CometFilter (21) - : +- CometScan parquet spark_catalog.default.item (20) - +- BroadcastExchange (32) - +- * Filter (31) - +- * HashAggregate (30) - +- * ColumnarToRow (29) - +- CometColumnarExchange (28) - +- RowToColumnar (27) - +- * HashAggregate (26) - +- * ColumnarToRow (25) - +- CometFilter (24) - +- CometScan parquet spark_catalog.default.item (23) +TakeOrderedAndProject (40) ++- * Filter (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (12) + : : +- CometBroadcastHashJoin (11) + : : :- CometProject (7) + : : : +- CometBroadcastHashJoin (6) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.customer_address (1) + : : : +- CometBroadcastExchange (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.customer (3) + : : +- CometBroadcastExchange (10) + : : +- CometFilter (9) + : : +- CometScan parquet spark_catalog.default.store_sales (8) + : +- CometBroadcastExchange (16) + : +- CometProject (15) + : +- CometFilter (14) + : +- CometScan parquet spark_catalog.default.date_dim (13) + +- BroadcastExchange (33) + +- * Project (32) + +- * BroadcastHashJoin Inner BuildRight (31) + :- * ColumnarToRow (22) + : +- CometFilter (21) + : +- CometScan parquet spark_catalog.default.item (20) + +- BroadcastExchange (30) + +- * Filter (29) + +- * HashAggregate (28) + +- Exchange (27) + +- * HashAggregate (26) + +- * ColumnarToRow (25) + +- CometFilter (24) + +- CometScan parquet spark_catalog.default.item (23) (1) Scan parquet spark_catalog.default.customer_address @@ -171,161 +167,149 @@ Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] Aggregate Attributes [2]: [sum#18, count#19] Results [3]: [i_category#17, sum#20, count#21] -(27) RowToColumnar +(27) Exchange Input [3]: [i_category#17, sum#20, count#21] +Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(28) CometColumnarExchange -Input [3]: [i_category#17, sum#20, count#21] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(29) ColumnarToRow [codegen id : 2] -Input [3]: [i_category#17, sum#20, count#21] - -(30) HashAggregate [codegen id : 2] +(28) HashAggregate [codegen id : 2] Input [3]: [i_category#17, sum#20, count#21] Keys [1]: [i_category#17] Functions [1]: [avg(UnscaledValue(i_current_price#16))] Aggregate Attributes [1]: [avg(UnscaledValue(i_current_price#16))#22] Results [2]: [cast((avg(UnscaledValue(i_current_price#16))#22 / 100.0) as decimal(11,6)) AS avg(i_current_price)#23, i_category#17] -(31) Filter [codegen id : 2] +(29) Filter [codegen id : 2] Input [2]: [avg(i_current_price)#23, i_category#17] Condition : isnotnull(avg(i_current_price)#23) -(32) BroadcastExchange +(30) BroadcastExchange Input [2]: [avg(i_current_price)#23, i_category#17] Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=2] -(33) BroadcastHashJoin [codegen id : 3] +(31) BroadcastHashJoin [codegen id : 3] Left keys [1]: [i_category#15] Right keys [1]: [i_category#17] Join type: Inner Join condition: (cast(i_current_price#14 as decimal(14,7)) > (1.2 * avg(i_current_price)#23)) -(34) Project [codegen id : 3] +(32) Project [codegen id : 3] Output [1]: [i_item_sk#13] Input [5]: [i_item_sk#13, i_current_price#14, i_category#15, avg(i_current_price)#23, i_category#17] -(35) BroadcastExchange +(33) BroadcastExchange Input [1]: [i_item_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(36) BroadcastHashJoin [codegen id : 4] +(34) BroadcastHashJoin [codegen id : 4] Left keys [1]: [ss_item_sk#5] Right keys [1]: [i_item_sk#13] Join type: Inner Join condition: None -(37) Project [codegen id : 4] +(35) Project [codegen id : 4] Output [1]: [ca_state#2] Input [3]: [ca_state#2, ss_item_sk#5, i_item_sk#13] -(38) HashAggregate [codegen id : 4] +(36) HashAggregate [codegen id : 4] Input [1]: [ca_state#2] Keys [1]: [ca_state#2] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#24] Results [2]: [ca_state#2, count#25] -(39) RowToColumnar -Input [2]: [ca_state#2, count#25] - -(40) CometColumnarExchange -Input [2]: [ca_state#2, count#25] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(41) ColumnarToRow [codegen id : 5] +(37) Exchange Input [2]: [ca_state#2, count#25] +Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(42) HashAggregate [codegen id : 5] +(38) HashAggregate [codegen id : 5] Input [2]: [ca_state#2, count#25] Keys [1]: [ca_state#2] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#26] Results [2]: [ca_state#2 AS state#27, count(1)#26 AS cnt#28] -(43) Filter [codegen id : 5] +(39) Filter [codegen id : 5] Input [2]: [state#27, cnt#28] Condition : (cnt#28 >= 10) -(44) TakeOrderedAndProject +(40) TakeOrderedAndProject Input [2]: [state#27, cnt#28] Arguments: 100, [cnt#28 ASC NULLS FIRST, state#27 ASC NULLS FIRST], [state#27, cnt#28] ===== Subqueries ===== Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (49) -+- * ColumnarToRow (48) - +- CometProject (47) - +- CometFilter (46) - +- CometScan parquet spark_catalog.default.date_dim (45) +BroadcastExchange (45) ++- * ColumnarToRow (44) + +- CometProject (43) + +- CometFilter (42) + +- CometScan parquet spark_catalog.default.date_dim (41) -(45) Scan parquet spark_catalog.default.date_dim +(41) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_month_seq#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), EqualTo(d_month_seq,ScalarSubquery#11), IsNotNull(d_date_sk)] ReadSchema: struct -(46) CometFilter +(42) CometFilter Input [2]: [d_date_sk#9, d_month_seq#10] Condition : ((isnotnull(d_month_seq#10) AND (d_month_seq#10 = ReusedSubquery Subquery scalar-subquery#11, [id=#12])) AND isnotnull(d_date_sk#9)) -(47) CometProject +(43) CometProject Input [2]: [d_date_sk#9, d_month_seq#10] Arguments: [d_date_sk#9], [d_date_sk#9] -(48) ColumnarToRow [codegen id : 1] +(44) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(49) BroadcastExchange +(45) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] -Subquery:2 Hosting operator id = 46 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] +Subquery:2 Hosting operator id = 42 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] -Subquery:3 Hosting operator id = 45 Hosting Expression = Subquery scalar-subquery#11, [id=#12] -* ColumnarToRow (56) -+- CometHashAggregate (55) - +- CometColumnarExchange (54) - +- CometHashAggregate (53) - +- CometProject (52) - +- CometFilter (51) - +- CometScan parquet spark_catalog.default.date_dim (50) +Subquery:3 Hosting operator id = 41 Hosting Expression = Subquery scalar-subquery#11, [id=#12] +* ColumnarToRow (52) ++- CometHashAggregate (51) + +- CometExchange (50) + +- CometHashAggregate (49) + +- CometProject (48) + +- CometFilter (47) + +- CometScan parquet spark_catalog.default.date_dim (46) -(50) Scan parquet spark_catalog.default.date_dim +(46) Scan parquet spark_catalog.default.date_dim Output [3]: [d_month_seq#29, d_year#30, d_moy#31] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2000), EqualTo(d_moy,1)] ReadSchema: struct -(51) CometFilter +(47) CometFilter Input [3]: [d_month_seq#29, d_year#30, d_moy#31] Condition : (((isnotnull(d_year#30) AND isnotnull(d_moy#31)) AND (d_year#30 = 2000)) AND (d_moy#31 = 1)) -(52) CometProject +(48) CometProject Input [3]: [d_month_seq#29, d_year#30, d_moy#31] Arguments: [d_month_seq#29], [d_month_seq#29] -(53) CometHashAggregate +(49) CometHashAggregate Input [1]: [d_month_seq#29] Keys [1]: [d_month_seq#29] Functions: [] -(54) CometColumnarExchange +(50) CometExchange Input [1]: [d_month_seq#29] -Arguments: hashpartitioning(d_month_seq#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(d_month_seq#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(55) CometHashAggregate +(51) CometHashAggregate Input [1]: [d_month_seq#29] Keys [1]: [d_month_seq#29] Functions: [] -(56) ColumnarToRow [codegen id : 1] +(52) ColumnarToRow [codegen id : 1] Input [1]: [d_month_seq#29] Subquery:4 Hosting operator id = 14 Hosting Expression = ReusedSubquery Subquery scalar-subquery#11, [id=#12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt index e28908700..c5bbfc7d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q6/simplified.txt @@ -2,76 +2,72 @@ TakeOrderedAndProject [cnt,state] WholeStageCodegen (5) Filter [cnt] HashAggregate [ca_state,count] [count(1),state,cnt,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_state] #1 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [ca_state] [count,count] - Project [ca_state] - BroadcastHashJoin [ss_item_sk,i_item_sk] - ColumnarToRow - InputAdapter - CometProject [ca_state,ss_item_sk] - CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] - CometProject [ca_state,ss_item_sk,ss_sold_date_sk] - CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometProject [ca_state,c_customer_sk] - CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 - CometFilter [c_customer_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 - CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - Subquery #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometHashAggregate [d_month_seq] - CometColumnarExchange [d_month_seq] #5 - CometHashAggregate [d_month_seq] - CometProject [d_month_seq] - CometFilter [d_month_seq,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - ReusedSubquery [d_month_seq] #2 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - Project [i_item_sk] - BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] - ColumnarToRow - InputAdapter - CometFilter [i_item_sk,i_current_price,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] - InputAdapter - BroadcastExchange #8 - WholeStageCodegen (2) - Filter [avg(i_current_price)] - HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count] + InputAdapter + Exchange [ca_state] #1 + WholeStageCodegen (4) + HashAggregate [ca_state] [count,count] + Project [ca_state] + BroadcastHashJoin [ss_item_sk,i_item_sk] + ColumnarToRow + InputAdapter + CometProject [ca_state,ss_item_sk] + CometBroadcastHashJoin [ca_state,ss_item_sk,ss_sold_date_sk,d_date_sk] + CometProject [ca_state,ss_item_sk,ss_sold_date_sk] + CometBroadcastHashJoin [ca_state,c_customer_sk,ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometProject [ca_state,c_customer_sk] + CometBroadcastHashJoin [ca_address_sk,ca_state,c_customer_sk,c_current_addr_sk] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk] #2 + CometFilter [c_customer_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_sold_date_sk] #3 + CometFilter [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) ColumnarToRow InputAdapter - CometColumnarExchange [i_category] #9 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_current_price] [sum,count,sum,count] - ColumnarToRow - InputAdapter - CometFilter [i_current_price,i_category] - CometScan parquet spark_catalog.default.item [i_current_price,i_category] + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + ReusedSubquery [d_month_seq] #2 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + Subquery #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometHashAggregate [d_month_seq] + CometExchange [d_month_seq] #5 + CometHashAggregate [d_month_seq] + CometProject [d_month_seq] + CometFilter [d_month_seq,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_month_seq,d_year,d_moy] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + ReusedSubquery [d_month_seq] #2 + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + ReusedSubquery [d_month_seq] #2 + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + Project [i_item_sk] + BroadcastHashJoin [i_category,i_category,i_current_price,avg(i_current_price)] + ColumnarToRow + InputAdapter + CometFilter [i_item_sk,i_current_price,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_category] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (2) + Filter [avg(i_current_price)] + HashAggregate [i_category,sum,count] [avg(UnscaledValue(i_current_price)),avg(i_current_price),sum,count] + InputAdapter + Exchange [i_category] #9 + WholeStageCodegen (1) + HashAggregate [i_category,i_current_price] [sum,count,sum,count] + ColumnarToRow + InputAdapter + CometFilter [i_current_price,i_category] + CometScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt index 30c42999c..419e3670a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/explain.txt @@ -1,193 +1,186 @@ == Physical Plan == -* ColumnarToRow (189) -+- CometSort (188) - +- CometColumnarExchange (187) - +- RowToColumnar (186) - +- * Project (185) - +- * SortMergeJoin Inner (184) - :- * ColumnarToRow (113) - : +- CometSort (112) - : +- CometColumnarExchange (111) - : +- RowToColumnar (110) - : +- * HashAggregate (109) - : +- * HashAggregate (108) - : +- * Project (107) - : +- * BroadcastHashJoin Inner BuildRight (106) - : :- * Project (100) - : : +- * BroadcastHashJoin Inner BuildRight (99) - : : :- * Project (97) - : : : +- * BroadcastHashJoin Inner BuildRight (96) - : : : :- * Project (91) - : : : : +- * BroadcastHashJoin Inner BuildRight (90) - : : : : :- * Project (88) - : : : : : +- * BroadcastHashJoin Inner BuildRight (87) - : : : : : :- * Project (82) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (81) - : : : : : : :- * Project (79) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (78) - : : : : : : : :- * Project (73) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (72) - : : : : : : : : :- * Project (67) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (66) - : : : : : : : : : :- * Project (64) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (63) - : : : : : : : : : : :- * Project (58) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (57) - : : : : : : : : : : : :- * Project (55) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (54) - : : : : : : : : : : : : :- * Project (49) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (48) - : : : : : : : : : : : : : :- * Project (43) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (42) - : : : : : : : : : : : : : : :- * Project (37) - : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (36) - : : : : : : : : : : : : : : : :- * Project (34) - : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (33) - : : : : : : : : : : : : : : : : :- * ColumnarToRow (11) - : : : : : : : : : : : : : : : : : +- CometSort (10) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (9) - : : : : : : : : : : : : : : : : : +- CometProject (8) - : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) - : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) - : : : : : : : : : : : : : : : : : : +- CometFilter (2) - : : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : : : : : : : : : : : : +- CometProject (6) - : : : : : : : : : : : : : : : : : +- CometFilter (5) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) - : : : : : : : : : : : : : : : : +- * Sort (32) - : : : : : : : : : : : : : : : : +- * Project (31) - : : : : : : : : : : : : : : : : +- * Filter (30) - : : : : : : : : : : : : : : : : +- * HashAggregate (29) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (28) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (27) - : : : : : : : : : : : : : : : : +- RowToColumnar (26) - : : : : : : : : : : : : : : : : +- * HashAggregate (25) - : : : : : : : : : : : : : : : : +- * ColumnarToRow (24) - : : : : : : : : : : : : : : : : +- CometProject (23) - : : : : : : : : : : : : : : : : +- CometSortMergeJoin (22) - : : : : : : : : : : : : : : : : :- CometSort (16) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (15) - : : : : : : : : : : : : : : : : : +- CometProject (14) - : : : : : : : : : : : : : : : : : +- CometFilter (13) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (12) - : : : : : : : : : : : : : : : : +- CometSort (21) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (20) - : : : : : : : : : : : : : : : : +- CometProject (19) - : : : : : : : : : : : : : : : : +- CometFilter (18) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (17) - : : : : : : : : : : : : : : : +- ReusedExchange (35) - : : : : : : : : : : : : : : +- BroadcastExchange (41) - : : : : : : : : : : : : : : +- * ColumnarToRow (40) - : : : : : : : : : : : : : : +- CometFilter (39) - : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store (38) - : : : : : : : : : : : : : +- BroadcastExchange (47) - : : : : : : : : : : : : : +- * ColumnarToRow (46) - : : : : : : : : : : : : : +- CometFilter (45) - : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer (44) - : : : : : : : : : : : : +- BroadcastExchange (53) - : : : : : : : : : : : : +- * ColumnarToRow (52) - : : : : : : : : : : : : +- CometFilter (51) - : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (50) - : : : : : : : : : : : +- ReusedExchange (56) - : : : : : : : : : : +- BroadcastExchange (62) - : : : : : : : : : : +- * ColumnarToRow (61) - : : : : : : : : : : +- CometFilter (60) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (59) - : : : : : : : : : +- ReusedExchange (65) - : : : : : : : : +- BroadcastExchange (71) - : : : : : : : : +- * ColumnarToRow (70) - : : : : : : : : +- CometFilter (69) - : : : : : : : : +- CometScan parquet spark_catalog.default.promotion (68) - : : : : : : : +- BroadcastExchange (77) - : : : : : : : +- * ColumnarToRow (76) - : : : : : : : +- CometFilter (75) - : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (74) - : : : : : : +- ReusedExchange (80) - : : : : : +- BroadcastExchange (86) - : : : : : +- * ColumnarToRow (85) - : : : : : +- CometFilter (84) - : : : : : +- CometScan parquet spark_catalog.default.customer_address (83) - : : : : +- ReusedExchange (89) - : : : +- BroadcastExchange (95) - : : : +- * ColumnarToRow (94) - : : : +- CometFilter (93) - : : : +- CometScan parquet spark_catalog.default.income_band (92) - : : +- ReusedExchange (98) - : +- BroadcastExchange (105) - : +- * ColumnarToRow (104) - : +- CometProject (103) - : +- CometFilter (102) - : +- CometScan parquet spark_catalog.default.item (101) - +- * ColumnarToRow (183) - +- CometSort (182) - +- CometColumnarExchange (181) - +- RowToColumnar (180) - +- * HashAggregate (179) - +- * HashAggregate (178) - +- * Project (177) - +- * BroadcastHashJoin Inner BuildRight (176) - :- * Project (174) - : +- * BroadcastHashJoin Inner BuildRight (173) - : :- * Project (171) - : : +- * BroadcastHashJoin Inner BuildRight (170) - : : :- * Project (168) - : : : +- * BroadcastHashJoin Inner BuildRight (167) - : : : :- * Project (165) - : : : : +- * BroadcastHashJoin Inner BuildRight (164) - : : : : :- * Project (162) - : : : : : +- * BroadcastHashJoin Inner BuildRight (161) - : : : : : :- * Project (159) - : : : : : : +- * BroadcastHashJoin Inner BuildRight (158) - : : : : : : :- * Project (156) - : : : : : : : +- * BroadcastHashJoin Inner BuildRight (155) - : : : : : : : :- * Project (153) - : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (152) - : : : : : : : : :- * Project (150) - : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (149) - : : : : : : : : : :- * Project (147) - : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (146) - : : : : : : : : : : :- * Project (144) - : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (143) - : : : : : : : : : : : :- * Project (141) - : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (140) - : : : : : : : : : : : : :- * Project (138) - : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (137) - : : : : : : : : : : : : : :- * Project (135) - : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (134) - : : : : : : : : : : : : : : :- * Project (132) - : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (131) - : : : : : : : : : : : : : : : :- * ColumnarToRow (124) - : : : : : : : : : : : : : : : : +- CometSort (123) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (122) - : : : : : : : : : : : : : : : : +- CometProject (121) - : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (120) - : : : : : : : : : : : : : : : : :- CometBroadcastExchange (116) - : : : : : : : : : : : : : : : : : +- CometFilter (115) - : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (114) - : : : : : : : : : : : : : : : : +- CometProject (119) - : : : : : : : : : : : : : : : : +- CometFilter (118) - : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (117) - : : : : : : : : : : : : : : : +- * Sort (130) - : : : : : : : : : : : : : : : +- * Project (129) - : : : : : : : : : : : : : : : +- * Filter (128) - : : : : : : : : : : : : : : : +- * HashAggregate (127) - : : : : : : : : : : : : : : : +- * ColumnarToRow (126) - : : : : : : : : : : : : : : : +- ReusedExchange (125) - : : : : : : : : : : : : : : +- ReusedExchange (133) - : : : : : : : : : : : : : +- ReusedExchange (136) - : : : : : : : : : : : : +- ReusedExchange (139) - : : : : : : : : : : : +- ReusedExchange (142) - : : : : : : : : : : +- ReusedExchange (145) - : : : : : : : : : +- ReusedExchange (148) - : : : : : : : : +- ReusedExchange (151) - : : : : : : : +- ReusedExchange (154) - : : : : : : +- ReusedExchange (157) - : : : : : +- ReusedExchange (160) - : : : : +- ReusedExchange (163) - : : : +- ReusedExchange (166) - : : +- ReusedExchange (169) - : +- ReusedExchange (172) - +- ReusedExchange (175) +* ColumnarToRow (182) ++- CometSort (181) + +- CometColumnarExchange (180) + +- RowToColumnar (179) + +- * Project (178) + +- * SortMergeJoin Inner (177) + :- * Sort (109) + : +- Exchange (108) + : +- * HashAggregate (107) + : +- * HashAggregate (106) + : +- * Project (105) + : +- * BroadcastHashJoin Inner BuildRight (104) + : :- * Project (98) + : : +- * BroadcastHashJoin Inner BuildRight (97) + : : :- * Project (95) + : : : +- * BroadcastHashJoin Inner BuildRight (94) + : : : :- * Project (89) + : : : : +- * BroadcastHashJoin Inner BuildRight (88) + : : : : :- * Project (86) + : : : : : +- * BroadcastHashJoin Inner BuildRight (85) + : : : : : :- * Project (80) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (79) + : : : : : : :- * Project (77) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (76) + : : : : : : : :- * Project (71) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (70) + : : : : : : : : :- * Project (65) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (64) + : : : : : : : : : :- * Project (62) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (61) + : : : : : : : : : : :- * Project (56) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (55) + : : : : : : : : : : : :- * Project (53) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (52) + : : : : : : : : : : : : :- * Project (47) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (46) + : : : : : : : : : : : : : :- * Project (41) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (40) + : : : : : : : : : : : : : : :- * Project (35) + : : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (34) + : : : : : : : : : : : : : : : :- * Project (32) + : : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (31) + : : : : : : : : : : : : : : : : :- * ColumnarToRow (11) + : : : : : : : : : : : : : : : : : +- CometSort (10) + : : : : : : : : : : : : : : : : : +- CometExchange (9) + : : : : : : : : : : : : : : : : : +- CometProject (8) + : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) + : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) + : : : : : : : : : : : : : : : : : : +- CometFilter (2) + : : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : : : : : : : : : : : : +- CometProject (6) + : : : : : : : : : : : : : : : : : +- CometFilter (5) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : : : : : : : : : : : : : : : +- * Sort (30) + : : : : : : : : : : : : : : : : +- * Project (29) + : : : : : : : : : : : : : : : : +- * Filter (28) + : : : : : : : : : : : : : : : : +- * HashAggregate (27) + : : : : : : : : : : : : : : : : +- Exchange (26) + : : : : : : : : : : : : : : : : +- * HashAggregate (25) + : : : : : : : : : : : : : : : : +- * ColumnarToRow (24) + : : : : : : : : : : : : : : : : +- CometProject (23) + : : : : : : : : : : : : : : : : +- CometSortMergeJoin (22) + : : : : : : : : : : : : : : : : :- CometSort (16) + : : : : : : : : : : : : : : : : : +- CometExchange (15) + : : : : : : : : : : : : : : : : : +- CometProject (14) + : : : : : : : : : : : : : : : : : +- CometFilter (13) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (12) + : : : : : : : : : : : : : : : : +- CometSort (21) + : : : : : : : : : : : : : : : : +- CometExchange (20) + : : : : : : : : : : : : : : : : +- CometProject (19) + : : : : : : : : : : : : : : : : +- CometFilter (18) + : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (17) + : : : : : : : : : : : : : : : +- ReusedExchange (33) + : : : : : : : : : : : : : : +- BroadcastExchange (39) + : : : : : : : : : : : : : : +- * ColumnarToRow (38) + : : : : : : : : : : : : : : +- CometFilter (37) + : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store (36) + : : : : : : : : : : : : : +- BroadcastExchange (45) + : : : : : : : : : : : : : +- * ColumnarToRow (44) + : : : : : : : : : : : : : +- CometFilter (43) + : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer (42) + : : : : : : : : : : : : +- BroadcastExchange (51) + : : : : : : : : : : : : +- * ColumnarToRow (50) + : : : : : : : : : : : : +- CometFilter (49) + : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.date_dim (48) + : : : : : : : : : : : +- ReusedExchange (54) + : : : : : : : : : : +- BroadcastExchange (60) + : : : : : : : : : : +- * ColumnarToRow (59) + : : : : : : : : : : +- CometFilter (58) + : : : : : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (57) + : : : : : : : : : +- ReusedExchange (63) + : : : : : : : : +- BroadcastExchange (69) + : : : : : : : : +- * ColumnarToRow (68) + : : : : : : : : +- CometFilter (67) + : : : : : : : : +- CometScan parquet spark_catalog.default.promotion (66) + : : : : : : : +- BroadcastExchange (75) + : : : : : : : +- * ColumnarToRow (74) + : : : : : : : +- CometFilter (73) + : : : : : : : +- CometScan parquet spark_catalog.default.household_demographics (72) + : : : : : : +- ReusedExchange (78) + : : : : : +- BroadcastExchange (84) + : : : : : +- * ColumnarToRow (83) + : : : : : +- CometFilter (82) + : : : : : +- CometScan parquet spark_catalog.default.customer_address (81) + : : : : +- ReusedExchange (87) + : : : +- BroadcastExchange (93) + : : : +- * ColumnarToRow (92) + : : : +- CometFilter (91) + : : : +- CometScan parquet spark_catalog.default.income_band (90) + : : +- ReusedExchange (96) + : +- BroadcastExchange (103) + : +- * ColumnarToRow (102) + : +- CometProject (101) + : +- CometFilter (100) + : +- CometScan parquet spark_catalog.default.item (99) + +- * Sort (176) + +- Exchange (175) + +- * HashAggregate (174) + +- * HashAggregate (173) + +- * Project (172) + +- * BroadcastHashJoin Inner BuildRight (171) + :- * Project (169) + : +- * BroadcastHashJoin Inner BuildRight (168) + : :- * Project (166) + : : +- * BroadcastHashJoin Inner BuildRight (165) + : : :- * Project (163) + : : : +- * BroadcastHashJoin Inner BuildRight (162) + : : : :- * Project (160) + : : : : +- * BroadcastHashJoin Inner BuildRight (159) + : : : : :- * Project (157) + : : : : : +- * BroadcastHashJoin Inner BuildRight (156) + : : : : : :- * Project (154) + : : : : : : +- * BroadcastHashJoin Inner BuildRight (153) + : : : : : : :- * Project (151) + : : : : : : : +- * BroadcastHashJoin Inner BuildRight (150) + : : : : : : : :- * Project (148) + : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (147) + : : : : : : : : :- * Project (145) + : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (144) + : : : : : : : : : :- * Project (142) + : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (141) + : : : : : : : : : : :- * Project (139) + : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (138) + : : : : : : : : : : : :- * Project (136) + : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (135) + : : : : : : : : : : : : :- * Project (133) + : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (132) + : : : : : : : : : : : : : :- * Project (130) + : : : : : : : : : : : : : : +- * BroadcastHashJoin Inner BuildRight (129) + : : : : : : : : : : : : : : :- * Project (127) + : : : : : : : : : : : : : : : +- * SortMergeJoin Inner (126) + : : : : : : : : : : : : : : : :- * ColumnarToRow (120) + : : : : : : : : : : : : : : : : +- CometSort (119) + : : : : : : : : : : : : : : : : +- CometExchange (118) + : : : : : : : : : : : : : : : : +- CometProject (117) + : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (116) + : : : : : : : : : : : : : : : : :- CometBroadcastExchange (112) + : : : : : : : : : : : : : : : : : +- CometFilter (111) + : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (110) + : : : : : : : : : : : : : : : : +- CometProject (115) + : : : : : : : : : : : : : : : : +- CometFilter (114) + : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.store_returns (113) + : : : : : : : : : : : : : : : +- * Sort (125) + : : : : : : : : : : : : : : : +- * Project (124) + : : : : : : : : : : : : : : : +- * Filter (123) + : : : : : : : : : : : : : : : +- * HashAggregate (122) + : : : : : : : : : : : : : : : +- ReusedExchange (121) + : : : : : : : : : : : : : : +- ReusedExchange (128) + : : : : : : : : : : : : : +- ReusedExchange (131) + : : : : : : : : : : : : +- ReusedExchange (134) + : : : : : : : : : : : +- ReusedExchange (137) + : : : : : : : : : : +- ReusedExchange (140) + : : : : : : : : : +- ReusedExchange (143) + : : : : : : : : +- ReusedExchange (146) + : : : : : : : +- ReusedExchange (149) + : : : : : : +- ReusedExchange (152) + : : : : : +- ReusedExchange (155) + : : : : +- ReusedExchange (158) + : : : +- ReusedExchange (161) + : : +- ReusedExchange (164) + : +- ReusedExchange (167) + +- ReusedExchange (170) (1) Scan parquet spark_catalog.default.store_sales @@ -230,9 +223,9 @@ Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number# Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(9) CometColumnarExchange +(9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -256,9 +249,9 @@ Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (16) CometSort Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] @@ -279,9 +272,9 @@ Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -(20) CometColumnarExchange +(20) CometExchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (21) CometSort Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] @@ -306,408 +299,396 @@ Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum((( Aggregate Attributes [3]: [sum#27, sum#28, isEmpty#29] Results [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] -(26) RowToColumnar +(26) Exchange Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] +Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(27) CometColumnarExchange -Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(28) ColumnarToRow [codegen id : 3] -Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] - -(29) HashAggregate [codegen id : 3] +(27) HashAggregate [codegen id : 3] Input [4]: [cs_item_sk#17, sum#30, sum#31, isEmpty#32] Keys [1]: [cs_item_sk#17] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#19)), sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#19))#33, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#34] Results [3]: [cs_item_sk#17, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#19))#33,17,2) AS sale#35, sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))#34 AS refund#36] -(30) Filter [codegen id : 3] +(28) Filter [codegen id : 3] Input [3]: [cs_item_sk#17, sale#35, refund#36] Condition : ((isnotnull(sale#35) AND isnotnull(refund#36)) AND (cast(sale#35 as decimal(21,2)) > (2 * refund#36))) -(31) Project [codegen id : 3] +(29) Project [codegen id : 3] Output [1]: [cs_item_sk#17] Input [3]: [cs_item_sk#17, sale#35, refund#36] -(32) Sort [codegen id : 3] +(30) Sort [codegen id : 3] Input [1]: [cs_item_sk#17] Arguments: [cs_item_sk#17 ASC NULLS FIRST], false, 0 -(33) SortMergeJoin [codegen id : 19] +(31) SortMergeJoin [codegen id : 19] Left keys [1]: [ss_item_sk#1] Right keys [1]: [cs_item_sk#17] Join type: Inner Join condition: None -(34) Project [codegen id : 19] +(32) Project [codegen id : 19] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, cs_item_sk#17] -(35) ReusedExchange [Reuses operator id: 193] +(33) ReusedExchange [Reuses operator id: 186] Output [2]: [d_date_sk#37, d_year#38] -(36) BroadcastHashJoin [codegen id : 19] +(34) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_sold_date_sk#12] Right keys [1]: [d_date_sk#37] Join type: Inner Join condition: None -(37) Project [codegen id : 19] +(35) Project [codegen id : 19] Output [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38] Input [13]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, d_date_sk#37, d_year#38] -(38) Scan parquet spark_catalog.default.store +(36) Scan parquet spark_catalog.default.store Output [3]: [s_store_sk#39, s_store_name#40, s_zip#41] Batched: true Location [not included in comparison]/{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk), IsNotNull(s_store_name), IsNotNull(s_zip)] ReadSchema: struct -(39) CometFilter +(37) CometFilter Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] Condition : ((isnotnull(s_store_sk#39) AND isnotnull(s_store_name#40)) AND isnotnull(s_zip#41)) -(40) ColumnarToRow [codegen id : 5] +(38) ColumnarToRow [codegen id : 5] Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] -(41) BroadcastExchange +(39) BroadcastExchange Input [3]: [s_store_sk#39, s_store_name#40, s_zip#41] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=5] -(42) BroadcastHashJoin [codegen id : 19] +(40) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_store_sk#6] Right keys [1]: [s_store_sk#39] Join type: Inner Join condition: None -(43) Project [codegen id : 19] +(41) Project [codegen id : 19] Output [12]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41] Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_sk#39, s_store_name#40, s_zip#41] -(44) Scan parquet spark_catalog.default.customer +(42) Scan parquet spark_catalog.default.customer Output [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_first_sales_date_sk), IsNotNull(c_first_shipto_date_sk), IsNotNull(c_current_cdemo_sk), IsNotNull(c_current_hdemo_sk), IsNotNull(c_current_addr_sk)] ReadSchema: struct -(45) CometFilter +(43) CometFilter Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Condition : (((((isnotnull(c_customer_sk#42) AND isnotnull(c_first_sales_date_sk#47)) AND isnotnull(c_first_shipto_date_sk#46)) AND isnotnull(c_current_cdemo_sk#43)) AND isnotnull(c_current_hdemo_sk#44)) AND isnotnull(c_current_addr_sk#45)) -(46) ColumnarToRow [codegen id : 6] +(44) ColumnarToRow [codegen id : 6] Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] -(47) BroadcastExchange +(45) BroadcastExchange Input [6]: [c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=6] -(48) BroadcastHashJoin [codegen id : 19] +(46) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_customer_sk#2] Right keys [1]: [c_customer_sk#42] Join type: Inner Join condition: None -(49) Project [codegen id : 19] +(47) Project [codegen id : 19] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] Input [18]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_customer_sk#42, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47] -(50) Scan parquet spark_catalog.default.date_dim +(48) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#48, d_year#49] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date_sk)] ReadSchema: struct -(51) CometFilter +(49) CometFilter Input [2]: [d_date_sk#48, d_year#49] Condition : isnotnull(d_date_sk#48) -(52) ColumnarToRow [codegen id : 7] +(50) ColumnarToRow [codegen id : 7] Input [2]: [d_date_sk#48, d_year#49] -(53) BroadcastExchange +(51) BroadcastExchange Input [2]: [d_date_sk#48, d_year#49] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=7] -(54) BroadcastHashJoin [codegen id : 19] +(52) BroadcastHashJoin [codegen id : 19] Left keys [1]: [c_first_sales_date_sk#47] Right keys [1]: [d_date_sk#48] Join type: Inner Join condition: None -(55) Project [codegen id : 19] +(53) Project [codegen id : 19] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, d_year#49] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, c_first_sales_date_sk#47, d_date_sk#48, d_year#49] -(56) ReusedExchange [Reuses operator id: 53] +(54) ReusedExchange [Reuses operator id: 51] Output [2]: [d_date_sk#50, d_year#51] -(57) BroadcastHashJoin [codegen id : 19] +(55) BroadcastHashJoin [codegen id : 19] Left keys [1]: [c_first_shipto_date_sk#46] Right keys [1]: [d_date_sk#50] Join type: Inner Join condition: None -(58) Project [codegen id : 19] +(56) Project [codegen id : 19] Output [16]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, c_first_shipto_date_sk#46, d_year#49, d_date_sk#50, d_year#51] -(59) Scan parquet spark_catalog.default.customer_demographics +(57) Scan parquet spark_catalog.default.customer_demographics Output [2]: [cd_demo_sk#52, cd_marital_status#53] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_demographics] PushedFilters: [IsNotNull(cd_demo_sk), IsNotNull(cd_marital_status)] ReadSchema: struct -(60) CometFilter +(58) CometFilter Input [2]: [cd_demo_sk#52, cd_marital_status#53] Condition : (isnotnull(cd_demo_sk#52) AND isnotnull(cd_marital_status#53)) -(61) ColumnarToRow [codegen id : 9] +(59) ColumnarToRow [codegen id : 9] Input [2]: [cd_demo_sk#52, cd_marital_status#53] -(62) BroadcastExchange +(60) BroadcastExchange Input [2]: [cd_demo_sk#52, cd_marital_status#53] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -(63) BroadcastHashJoin [codegen id : 19] +(61) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_cdemo_sk#3] Right keys [1]: [cd_demo_sk#52] Join type: Inner Join condition: None -(64) Project [codegen id : 19] +(62) Project [codegen id : 19] Output [16]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_marital_status#53] Input [18]: [ss_item_sk#1, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_demo_sk#52, cd_marital_status#53] -(65) ReusedExchange [Reuses operator id: 62] +(63) ReusedExchange [Reuses operator id: 60] Output [2]: [cd_demo_sk#54, cd_marital_status#55] -(66) BroadcastHashJoin [codegen id : 19] +(64) BroadcastHashJoin [codegen id : 19] Left keys [1]: [c_current_cdemo_sk#43] Right keys [1]: [cd_demo_sk#54] Join type: Inner Join condition: NOT (cd_marital_status#53 = cd_marital_status#55) -(67) Project [codegen id : 19] +(65) Project [codegen id : 19] Output [14]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [18]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_cdemo_sk#43, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, cd_marital_status#53, cd_demo_sk#54, cd_marital_status#55] -(68) Scan parquet spark_catalog.default.promotion +(66) Scan parquet spark_catalog.default.promotion Output [1]: [p_promo_sk#56] Batched: true Location [not included in comparison]/{warehouse_dir}/promotion] PushedFilters: [IsNotNull(p_promo_sk)] ReadSchema: struct -(69) CometFilter +(67) CometFilter Input [1]: [p_promo_sk#56] Condition : isnotnull(p_promo_sk#56) -(70) ColumnarToRow [codegen id : 11] +(68) ColumnarToRow [codegen id : 11] Input [1]: [p_promo_sk#56] -(71) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_promo_sk#56] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] -(72) BroadcastHashJoin [codegen id : 19] +(70) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_promo_sk#7] Right keys [1]: [p_promo_sk#56] Join type: Inner Join condition: None -(73) Project [codegen id : 19] +(71) Project [codegen id : 19] Output [13]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51] Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, p_promo_sk#56] -(74) Scan parquet spark_catalog.default.household_demographics +(72) Scan parquet spark_catalog.default.household_demographics Output [2]: [hd_demo_sk#57, hd_income_band_sk#58] Batched: true Location [not included in comparison]/{warehouse_dir}/household_demographics] PushedFilters: [IsNotNull(hd_demo_sk), IsNotNull(hd_income_band_sk)] ReadSchema: struct -(75) CometFilter +(73) CometFilter Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] Condition : (isnotnull(hd_demo_sk#57) AND isnotnull(hd_income_band_sk#58)) -(76) ColumnarToRow [codegen id : 12] +(74) ColumnarToRow [codegen id : 12] Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] -(77) BroadcastExchange +(75) BroadcastExchange Input [2]: [hd_demo_sk#57, hd_income_band_sk#58] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] -(78) BroadcastHashJoin [codegen id : 19] +(76) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_hdemo_sk#4] Right keys [1]: [hd_demo_sk#57] Join type: Inner Join condition: None -(79) Project [codegen id : 19] +(77) Project [codegen id : 19] Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58] Input [15]: [ss_item_sk#1, ss_hdemo_sk#4, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_demo_sk#57, hd_income_band_sk#58] -(80) ReusedExchange [Reuses operator id: 77] +(78) ReusedExchange [Reuses operator id: 75] Output [2]: [hd_demo_sk#59, hd_income_band_sk#60] -(81) BroadcastHashJoin [codegen id : 19] +(79) BroadcastHashJoin [codegen id : 19] Left keys [1]: [c_current_hdemo_sk#44] Right keys [1]: [hd_demo_sk#59] Join type: Inner Join condition: None -(82) Project [codegen id : 19] +(80) Project [codegen id : 19] Output [13]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60] Input [15]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_hdemo_sk#44, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_demo_sk#59, hd_income_band_sk#60] -(83) Scan parquet spark_catalog.default.customer_address +(81) Scan parquet spark_catalog.default.customer_address Output [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Batched: true Location [not included in comparison]/{warehouse_dir}/customer_address] PushedFilters: [IsNotNull(ca_address_sk)] ReadSchema: struct -(84) CometFilter +(82) CometFilter Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Condition : isnotnull(ca_address_sk#61) -(85) ColumnarToRow [codegen id : 14] +(83) ColumnarToRow [codegen id : 14] Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] -(86) BroadcastExchange +(84) BroadcastExchange Input [5]: [ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=11] -(87) BroadcastHashJoin [codegen id : 19] +(85) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_addr_sk#5] Right keys [1]: [ca_address_sk#61] Join type: Inner Join condition: None -(88) Project [codegen id : 19] +(86) Project [codegen id : 19] Output [16]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] Input [18]: [ss_item_sk#1, ss_addr_sk#5, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_address_sk#61, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65] -(89) ReusedExchange [Reuses operator id: 86] +(87) ReusedExchange [Reuses operator id: 84] Output [5]: [ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(90) BroadcastHashJoin [codegen id : 19] +(88) BroadcastHashJoin [codegen id : 19] Left keys [1]: [c_current_addr_sk#45] Right keys [1]: [ca_address_sk#66] Join type: Inner Join condition: None -(91) Project [codegen id : 19] +(89) Project [codegen id : 19] Output [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [21]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, c_current_addr_sk#45, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_address_sk#66, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] -(92) Scan parquet spark_catalog.default.income_band +(90) Scan parquet spark_catalog.default.income_band Output [1]: [ib_income_band_sk#71] Batched: true Location [not included in comparison]/{warehouse_dir}/income_band] PushedFilters: [IsNotNull(ib_income_band_sk)] ReadSchema: struct -(93) CometFilter +(91) CometFilter Input [1]: [ib_income_band_sk#71] Condition : isnotnull(ib_income_band_sk#71) -(94) ColumnarToRow [codegen id : 16] +(92) ColumnarToRow [codegen id : 16] Input [1]: [ib_income_band_sk#71] -(95) BroadcastExchange +(93) BroadcastExchange Input [1]: [ib_income_band_sk#71] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=12] -(96) BroadcastHashJoin [codegen id : 19] +(94) BroadcastHashJoin [codegen id : 19] Left keys [1]: [hd_income_band_sk#58] Right keys [1]: [ib_income_band_sk#71] Join type: Inner Join condition: None -(97) Project [codegen id : 19] +(95) Project [codegen id : 19] Output [18]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [20]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#58, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#71] -(98) ReusedExchange [Reuses operator id: 95] +(96) ReusedExchange [Reuses operator id: 93] Output [1]: [ib_income_band_sk#72] -(99) BroadcastHashJoin [codegen id : 19] +(97) BroadcastHashJoin [codegen id : 19] Left keys [1]: [hd_income_band_sk#60] Right keys [1]: [ib_income_band_sk#72] Join type: Inner Join condition: None -(100) Project [codegen id : 19] +(98) Project [codegen id : 19] Output [17]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70] Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, hd_income_band_sk#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, ib_income_band_sk#72] -(101) Scan parquet spark_catalog.default.item +(99) Scan parquet spark_catalog.default.item Output [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_current_price), In(i_color, [burlywood ,floral ,indian ,medium ,purple ,spring ]), GreaterThanOrEqual(i_current_price,64.00), LessThanOrEqual(i_current_price,74.00), GreaterThanOrEqual(i_current_price,65.00), LessThanOrEqual(i_current_price,79.00), IsNotNull(i_item_sk)] ReadSchema: struct -(102) CometFilter +(100) CometFilter Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Condition : ((((((isnotnull(i_current_price#74) AND i_color#75 IN (purple ,burlywood ,indian ,spring ,floral ,medium )) AND (i_current_price#74 >= 64.00)) AND (i_current_price#74 <= 74.00)) AND (i_current_price#74 >= 65.00)) AND (i_current_price#74 <= 79.00)) AND isnotnull(i_item_sk#73)) -(103) CometProject +(101) CometProject Input [4]: [i_item_sk#73, i_current_price#74, i_color#75, i_product_name#76] Arguments: [i_item_sk#73, i_product_name#76], [i_item_sk#73, i_product_name#76] -(104) ColumnarToRow [codegen id : 18] +(102) ColumnarToRow [codegen id : 18] Input [2]: [i_item_sk#73, i_product_name#76] -(105) BroadcastExchange +(103) BroadcastExchange Input [2]: [i_item_sk#73, i_product_name#76] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] -(106) BroadcastHashJoin [codegen id : 19] +(104) BroadcastHashJoin [codegen id : 19] Left keys [1]: [ss_item_sk#1] Right keys [1]: [i_item_sk#73] Join type: Inner Join condition: None -(107) Project [codegen id : 19] +(105) Project [codegen id : 19] Output [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, d_year#49, d_year#51, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] Input [19]: [ss_item_sk#1, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, s_store_name#40, s_zip#41, d_year#49, d_year#51, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] -(108) HashAggregate [codegen id : 19] +(106) HashAggregate [codegen id : 19] Input [18]: [ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, d_year#38, d_year#49, d_year#51, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, i_item_sk#73, i_product_name#76] Keys [15]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#9)), partial_sum(UnscaledValue(ss_list_price#10)), partial_sum(UnscaledValue(ss_coupon_amt#11))] Aggregate Attributes [4]: [count#77, sum#78, sum#79, sum#80] Results [19]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51, count#81, sum#82, sum#83, sum#84] -(109) HashAggregate [codegen id : 19] +(107) HashAggregate [codegen id : 19] Input [19]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51, count#81, sum#82, sum#83, sum#84] Keys [15]: [i_product_name#76, i_item_sk#73, s_store_name#40, s_zip#41, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, ca_street_number#67, ca_street_name#68, ca_city#69, ca_zip#70, d_year#38, d_year#49, d_year#51] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] Aggregate Attributes [4]: [count(1)#85, sum(UnscaledValue(ss_wholesale_cost#9))#86, sum(UnscaledValue(ss_list_price#10))#87, sum(UnscaledValue(ss_coupon_amt#11))#88] Results [17]: [i_product_name#76 AS product_name#89, i_item_sk#73 AS item_sk#90, s_store_name#40 AS store_name#91, s_zip#41 AS store_zip#92, ca_street_number#62 AS b_street_number#93, ca_street_name#63 AS b_streen_name#94, ca_city#64 AS b_city#95, ca_zip#65 AS b_zip#96, ca_street_number#67 AS c_street_number#97, ca_street_name#68 AS c_street_name#98, ca_city#69 AS c_city#99, ca_zip#70 AS c_zip#100, d_year#38 AS syear#101, count(1)#85 AS cnt#102, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#9))#86,17,2) AS s1#103, MakeDecimal(sum(UnscaledValue(ss_list_price#10))#87,17,2) AS s2#104, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#11))#88,17,2) AS s3#105] -(110) RowToColumnar -Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] - -(111) CometColumnarExchange -Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] -Arguments: hashpartitioning(item_sk#90, store_name#91, store_zip#92, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] - -(112) CometSort +(108) Exchange Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] -Arguments: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105], [item_sk#90 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, store_zip#92 ASC NULLS FIRST] +Arguments: hashpartitioning(item_sk#90, store_name#91, store_zip#92, 5), ENSURE_REQUIREMENTS, [plan_id=14] -(113) ColumnarToRow [codegen id : 20] +(109) Sort [codegen id : 20] Input [17]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105] +Arguments: [item_sk#90 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, store_zip#92 ASC NULLS FIRST], false, 0 -(114) Scan parquet spark_catalog.default.store_sales +(110) Scan parquet spark_catalog.default.store_sales Output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Batched: true Location: InMemoryFileIndex [] @@ -715,380 +696,371 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#117), dynamicpruningexpression(ss_s PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_ticket_number), IsNotNull(ss_store_sk), IsNotNull(ss_customer_sk), IsNotNull(ss_cdemo_sk), IsNotNull(ss_promo_sk), IsNotNull(ss_hdemo_sk), IsNotNull(ss_addr_sk)] ReadSchema: struct -(115) CometFilter +(111) CometFilter Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Condition : (((((((isnotnull(ss_item_sk#106) AND isnotnull(ss_ticket_number#113)) AND isnotnull(ss_store_sk#111)) AND isnotnull(ss_customer_sk#107)) AND isnotnull(ss_cdemo_sk#108)) AND isnotnull(ss_promo_sk#112)) AND isnotnull(ss_hdemo_sk#109)) AND isnotnull(ss_addr_sk#110)) -(116) CometBroadcastExchange +(112) CometBroadcastExchange Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Arguments: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(117) Scan parquet spark_catalog.default.store_returns +(113) Scan parquet spark_catalog.default.store_returns Output [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] Batched: true Location [not included in comparison]/{warehouse_dir}/store_returns] PushedFilters: [IsNotNull(sr_item_sk), IsNotNull(sr_ticket_number)] ReadSchema: struct -(118) CometFilter +(114) CometFilter Input [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] Condition : (isnotnull(sr_item_sk#119) AND isnotnull(sr_ticket_number#120)) -(119) CometProject +(115) CometProject Input [3]: [sr_item_sk#119, sr_ticket_number#120, sr_returned_date_sk#121] Arguments: [sr_item_sk#119, sr_ticket_number#120], [sr_item_sk#119, sr_ticket_number#120] -(120) CometBroadcastHashJoin +(116) CometBroadcastHashJoin Left output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Right output [2]: [sr_item_sk#119, sr_ticket_number#120] Arguments: [ss_item_sk#106, ss_ticket_number#113], [sr_item_sk#119, sr_ticket_number#120], Inner, BuildLeft -(121) CometProject +(117) CometProject Input [14]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_ticket_number#113, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, sr_item_sk#119, sr_ticket_number#120] Arguments: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(122) CometColumnarExchange +(118) CometExchange Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -Arguments: hashpartitioning(ss_item_sk#106, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] +Arguments: hashpartitioning(ss_item_sk#106, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] -(123) CometSort +(119) CometSort Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Arguments: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117], [ss_item_sk#106 ASC NULLS FIRST] -(124) ColumnarToRow [codegen id : 21] +(120) ColumnarToRow [codegen id : 21] Input [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] -(125) ReusedExchange [Reuses operator id: 27] +(121) ReusedExchange [Reuses operator id: 26] Output [4]: [cs_item_sk#122, sum#123, sum#124, isEmpty#125] -(126) ColumnarToRow [codegen id : 23] -Input [4]: [cs_item_sk#122, sum#123, sum#124, isEmpty#125] - -(127) HashAggregate [codegen id : 23] +(122) HashAggregate [codegen id : 23] Input [4]: [cs_item_sk#122, sum#123, sum#124, isEmpty#125] Keys [1]: [cs_item_sk#122] Functions [2]: [sum(UnscaledValue(cs_ext_list_price#126)), sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))] Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_list_price#126))#33, sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))#34] Results [3]: [cs_item_sk#122, MakeDecimal(sum(UnscaledValue(cs_ext_list_price#126))#33,17,2) AS sale#130, sum(((cr_refunded_cash#127 + cr_reversed_charge#128) + cr_store_credit#129))#34 AS refund#131] -(128) Filter [codegen id : 23] +(123) Filter [codegen id : 23] Input [3]: [cs_item_sk#122, sale#130, refund#131] Condition : ((isnotnull(sale#130) AND isnotnull(refund#131)) AND (cast(sale#130 as decimal(21,2)) > (2 * refund#131))) -(129) Project [codegen id : 23] +(124) Project [codegen id : 23] Output [1]: [cs_item_sk#122] Input [3]: [cs_item_sk#122, sale#130, refund#131] -(130) Sort [codegen id : 23] +(125) Sort [codegen id : 23] Input [1]: [cs_item_sk#122] Arguments: [cs_item_sk#122 ASC NULLS FIRST], false, 0 -(131) SortMergeJoin [codegen id : 39] +(126) SortMergeJoin [codegen id : 39] Left keys [1]: [ss_item_sk#106] Right keys [1]: [cs_item_sk#122] Join type: Inner Join condition: None -(132) Project [codegen id : 39] +(127) Project [codegen id : 39] Output [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117] Input [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, cs_item_sk#122] -(133) ReusedExchange [Reuses operator id: 197] +(128) ReusedExchange [Reuses operator id: 190] Output [2]: [d_date_sk#132, d_year#133] -(134) BroadcastHashJoin [codegen id : 39] +(129) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_sold_date_sk#117] Right keys [1]: [d_date_sk#132] Join type: Inner Join condition: None -(135) Project [codegen id : 39] +(130) Project [codegen id : 39] Output [11]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133] Input [13]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, ss_sold_date_sk#117, d_date_sk#132, d_year#133] -(136) ReusedExchange [Reuses operator id: 41] +(131) ReusedExchange [Reuses operator id: 39] Output [3]: [s_store_sk#134, s_store_name#135, s_zip#136] -(137) BroadcastHashJoin [codegen id : 39] +(132) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_store_sk#111] Right keys [1]: [s_store_sk#134] Join type: Inner Join condition: None -(138) Project [codegen id : 39] +(133) Project [codegen id : 39] Output [12]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136] Input [14]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_store_sk#111, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_sk#134, s_store_name#135, s_zip#136] -(139) ReusedExchange [Reuses operator id: 47] +(134) ReusedExchange [Reuses operator id: 45] Output [6]: [c_customer_sk#137, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] -(140) BroadcastHashJoin [codegen id : 39] +(135) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_customer_sk#107] Right keys [1]: [c_customer_sk#137] Join type: Inner Join condition: None -(141) Project [codegen id : 39] +(136) Project [codegen id : 39] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] Input [18]: [ss_item_sk#106, ss_customer_sk#107, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_customer_sk#137, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142] -(142) ReusedExchange [Reuses operator id: 53] +(137) ReusedExchange [Reuses operator id: 51] Output [2]: [d_date_sk#143, d_year#144] -(143) BroadcastHashJoin [codegen id : 39] +(138) BroadcastHashJoin [codegen id : 39] Left keys [1]: [c_first_sales_date_sk#142] Right keys [1]: [d_date_sk#143] Join type: Inner Join condition: None -(144) Project [codegen id : 39] +(139) Project [codegen id : 39] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, d_year#144] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, c_first_sales_date_sk#142, d_date_sk#143, d_year#144] -(145) ReusedExchange [Reuses operator id: 53] +(140) ReusedExchange [Reuses operator id: 51] Output [2]: [d_date_sk#145, d_year#146] -(146) BroadcastHashJoin [codegen id : 39] +(141) BroadcastHashJoin [codegen id : 39] Left keys [1]: [c_first_shipto_date_sk#141] Right keys [1]: [d_date_sk#145] Join type: Inner Join condition: None -(147) Project [codegen id : 39] +(142) Project [codegen id : 39] Output [16]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, c_first_shipto_date_sk#141, d_year#144, d_date_sk#145, d_year#146] -(148) ReusedExchange [Reuses operator id: 62] +(143) ReusedExchange [Reuses operator id: 60] Output [2]: [cd_demo_sk#147, cd_marital_status#148] -(149) BroadcastHashJoin [codegen id : 39] +(144) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_cdemo_sk#108] Right keys [1]: [cd_demo_sk#147] Join type: Inner Join condition: None -(150) Project [codegen id : 39] +(145) Project [codegen id : 39] Output [16]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_marital_status#148] Input [18]: [ss_item_sk#106, ss_cdemo_sk#108, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_demo_sk#147, cd_marital_status#148] -(151) ReusedExchange [Reuses operator id: 62] +(146) ReusedExchange [Reuses operator id: 60] Output [2]: [cd_demo_sk#149, cd_marital_status#150] -(152) BroadcastHashJoin [codegen id : 39] +(147) BroadcastHashJoin [codegen id : 39] Left keys [1]: [c_current_cdemo_sk#138] Right keys [1]: [cd_demo_sk#149] Join type: Inner Join condition: NOT (cd_marital_status#148 = cd_marital_status#150) -(153) Project [codegen id : 39] +(148) Project [codegen id : 39] Output [14]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [18]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_cdemo_sk#138, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, cd_marital_status#148, cd_demo_sk#149, cd_marital_status#150] -(154) ReusedExchange [Reuses operator id: 71] +(149) ReusedExchange [Reuses operator id: 69] Output [1]: [p_promo_sk#151] -(155) BroadcastHashJoin [codegen id : 39] +(150) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_promo_sk#112] Right keys [1]: [p_promo_sk#151] Join type: Inner Join condition: None -(156) Project [codegen id : 39] +(151) Project [codegen id : 39] Output [13]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146] Input [15]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_promo_sk#112, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, p_promo_sk#151] -(157) ReusedExchange [Reuses operator id: 77] +(152) ReusedExchange [Reuses operator id: 75] Output [2]: [hd_demo_sk#152, hd_income_band_sk#153] -(158) BroadcastHashJoin [codegen id : 39] +(153) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_hdemo_sk#109] Right keys [1]: [hd_demo_sk#152] Join type: Inner Join condition: None -(159) Project [codegen id : 39] +(154) Project [codegen id : 39] Output [13]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153] Input [15]: [ss_item_sk#106, ss_hdemo_sk#109, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_demo_sk#152, hd_income_band_sk#153] -(160) ReusedExchange [Reuses operator id: 77] +(155) ReusedExchange [Reuses operator id: 75] Output [2]: [hd_demo_sk#154, hd_income_band_sk#155] -(161) BroadcastHashJoin [codegen id : 39] +(156) BroadcastHashJoin [codegen id : 39] Left keys [1]: [c_current_hdemo_sk#139] Right keys [1]: [hd_demo_sk#154] Join type: Inner Join condition: None -(162) Project [codegen id : 39] +(157) Project [codegen id : 39] Output [13]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155] Input [15]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_hdemo_sk#139, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_demo_sk#154, hd_income_band_sk#155] -(163) ReusedExchange [Reuses operator id: 86] +(158) ReusedExchange [Reuses operator id: 84] Output [5]: [ca_address_sk#156, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] -(164) BroadcastHashJoin [codegen id : 39] +(159) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_addr_sk#110] Right keys [1]: [ca_address_sk#156] Join type: Inner Join condition: None -(165) Project [codegen id : 39] +(160) Project [codegen id : 39] Output [16]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] Input [18]: [ss_item_sk#106, ss_addr_sk#110, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_address_sk#156, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160] -(166) ReusedExchange [Reuses operator id: 86] +(161) ReusedExchange [Reuses operator id: 84] Output [5]: [ca_address_sk#161, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] -(167) BroadcastHashJoin [codegen id : 39] +(162) BroadcastHashJoin [codegen id : 39] Left keys [1]: [c_current_addr_sk#140] Right keys [1]: [ca_address_sk#161] Join type: Inner Join condition: None -(168) Project [codegen id : 39] +(163) Project [codegen id : 39] Output [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [21]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, c_current_addr_sk#140, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_address_sk#161, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] -(169) ReusedExchange [Reuses operator id: 95] +(164) ReusedExchange [Reuses operator id: 93] Output [1]: [ib_income_band_sk#166] -(170) BroadcastHashJoin [codegen id : 39] +(165) BroadcastHashJoin [codegen id : 39] Left keys [1]: [hd_income_band_sk#153] Right keys [1]: [ib_income_band_sk#166] Join type: Inner Join condition: None -(171) Project [codegen id : 39] +(166) Project [codegen id : 39] Output [18]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [20]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#153, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, ib_income_band_sk#166] -(172) ReusedExchange [Reuses operator id: 95] +(167) ReusedExchange [Reuses operator id: 93] Output [1]: [ib_income_band_sk#167] -(173) BroadcastHashJoin [codegen id : 39] +(168) BroadcastHashJoin [codegen id : 39] Left keys [1]: [hd_income_band_sk#155] Right keys [1]: [ib_income_band_sk#167] Join type: Inner Join condition: None -(174) Project [codegen id : 39] +(169) Project [codegen id : 39] Output [17]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165] Input [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, hd_income_band_sk#155, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, ib_income_band_sk#167] -(175) ReusedExchange [Reuses operator id: 105] +(170) ReusedExchange [Reuses operator id: 103] Output [2]: [i_item_sk#168, i_product_name#169] -(176) BroadcastHashJoin [codegen id : 39] +(171) BroadcastHashJoin [codegen id : 39] Left keys [1]: [ss_item_sk#106] Right keys [1]: [i_item_sk#168] Join type: Inner Join condition: None -(177) Project [codegen id : 39] +(172) Project [codegen id : 39] Output [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, d_year#144, d_year#146, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] Input [19]: [ss_item_sk#106, ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, s_store_name#135, s_zip#136, d_year#144, d_year#146, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] -(178) HashAggregate [codegen id : 39] +(173) HashAggregate [codegen id : 39] Input [18]: [ss_wholesale_cost#114, ss_list_price#115, ss_coupon_amt#116, d_year#133, d_year#144, d_year#146, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, i_item_sk#168, i_product_name#169] Keys [15]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146] Functions [4]: [partial_count(1), partial_sum(UnscaledValue(ss_wholesale_cost#114)), partial_sum(UnscaledValue(ss_list_price#115)), partial_sum(UnscaledValue(ss_coupon_amt#116))] Aggregate Attributes [4]: [count#77, sum#170, sum#171, sum#172] Results [19]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146, count#81, sum#173, sum#174, sum#175] -(179) HashAggregate [codegen id : 39] +(174) HashAggregate [codegen id : 39] Input [19]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146, count#81, sum#173, sum#174, sum#175] Keys [15]: [i_product_name#169, i_item_sk#168, s_store_name#135, s_zip#136, ca_street_number#157, ca_street_name#158, ca_city#159, ca_zip#160, ca_street_number#162, ca_street_name#163, ca_city#164, ca_zip#165, d_year#133, d_year#144, d_year#146] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#114)), sum(UnscaledValue(ss_list_price#115)), sum(UnscaledValue(ss_coupon_amt#116))] Aggregate Attributes [4]: [count(1)#85, sum(UnscaledValue(ss_wholesale_cost#114))#86, sum(UnscaledValue(ss_list_price#115))#87, sum(UnscaledValue(ss_coupon_amt#116))#88] Results [8]: [i_item_sk#168 AS item_sk#176, s_store_name#135 AS store_name#177, s_zip#136 AS store_zip#178, d_year#133 AS syear#179, count(1)#85 AS cnt#180, MakeDecimal(sum(UnscaledValue(ss_wholesale_cost#114))#86,17,2) AS s1#181, MakeDecimal(sum(UnscaledValue(ss_list_price#115))#87,17,2) AS s2#182, MakeDecimal(sum(UnscaledValue(ss_coupon_amt#116))#88,17,2) AS s3#183] -(180) RowToColumnar -Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] - -(181) CometColumnarExchange -Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] -Arguments: hashpartitioning(item_sk#176, store_name#177, store_zip#178, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=16] - -(182) CometSort +(175) Exchange Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] -Arguments: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183], [item_sk#176 ASC NULLS FIRST, store_name#177 ASC NULLS FIRST, store_zip#178 ASC NULLS FIRST] +Arguments: hashpartitioning(item_sk#176, store_name#177, store_zip#178, 5), ENSURE_REQUIREMENTS, [plan_id=16] -(183) ColumnarToRow [codegen id : 40] +(176) Sort [codegen id : 40] Input [8]: [item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] +Arguments: [item_sk#176 ASC NULLS FIRST, store_name#177 ASC NULLS FIRST, store_zip#178 ASC NULLS FIRST], false, 0 -(184) SortMergeJoin [codegen id : 41] +(177) SortMergeJoin [codegen id : 41] Left keys [3]: [item_sk#90, store_name#91, store_zip#92] Right keys [3]: [item_sk#176, store_name#177, store_zip#178] Join type: Inner Join condition: (cnt#180 <= cnt#102) -(185) Project [codegen id : 41] +(178) Project [codegen id : 41] Output [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] Input [25]: [product_name#89, item_sk#90, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, item_sk#176, store_name#177, store_zip#178, syear#179, cnt#180, s1#181, s2#182, s3#183] -(186) RowToColumnar +(179) RowToColumnar Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] -(187) CometColumnarExchange +(180) CometColumnarExchange Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] Arguments: rangepartitioning(product_name#89 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, cnt#180 ASC NULLS FIRST, s1#103 ASC NULLS FIRST, s1#181 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=17] -(188) CometSort +(181) CometSort Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] Arguments: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180], [product_name#89 ASC NULLS FIRST, store_name#91 ASC NULLS FIRST, cnt#180 ASC NULLS FIRST, s1#103 ASC NULLS FIRST, s1#181 ASC NULLS FIRST] -(189) ColumnarToRow [codegen id : 42] +(182) ColumnarToRow [codegen id : 42] Input [21]: [product_name#89, store_name#91, store_zip#92, b_street_number#93, b_streen_name#94, b_city#95, b_zip#96, c_street_number#97, c_street_name#98, c_city#99, c_zip#100, syear#101, cnt#102, s1#103, s2#104, s3#105, s1#181, s2#182, s3#183, syear#179, cnt#180] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#12 IN dynamicpruning#13 -BroadcastExchange (193) -+- * ColumnarToRow (192) - +- CometFilter (191) - +- CometScan parquet spark_catalog.default.date_dim (190) +BroadcastExchange (186) ++- * ColumnarToRow (185) + +- CometFilter (184) + +- CometScan parquet spark_catalog.default.date_dim (183) -(190) Scan parquet spark_catalog.default.date_dim +(183) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#37, d_year#38] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,1999), IsNotNull(d_date_sk)] ReadSchema: struct -(191) CometFilter +(184) CometFilter Input [2]: [d_date_sk#37, d_year#38] Condition : ((isnotnull(d_year#38) AND (d_year#38 = 1999)) AND isnotnull(d_date_sk#37)) -(192) ColumnarToRow [codegen id : 1] +(185) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#37, d_year#38] -(193) BroadcastExchange +(186) BroadcastExchange Input [2]: [d_date_sk#37, d_year#38] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=18] -Subquery:2 Hosting operator id = 114 Hosting Expression = ss_sold_date_sk#117 IN dynamicpruning#118 -BroadcastExchange (197) -+- * ColumnarToRow (196) - +- CometFilter (195) - +- CometScan parquet spark_catalog.default.date_dim (194) +Subquery:2 Hosting operator id = 110 Hosting Expression = ss_sold_date_sk#117 IN dynamicpruning#118 +BroadcastExchange (190) ++- * ColumnarToRow (189) + +- CometFilter (188) + +- CometScan parquet spark_catalog.default.date_dim (187) -(194) Scan parquet spark_catalog.default.date_dim +(187) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#132, d_year#133] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(195) CometFilter +(188) CometFilter Input [2]: [d_date_sk#132, d_year#133] Condition : ((isnotnull(d_year#133) AND (d_year#133 = 2000)) AND isnotnull(d_date_sk#132)) -(196) ColumnarToRow [codegen id : 1] +(189) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#132, d_year#133] -(197) BroadcastExchange +(190) BroadcastExchange Input [2]: [d_date_sk#132, d_year#133] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=19] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt index 09bc3d099..e69fb5d75 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q64/simplified.txt @@ -9,268 +9,261 @@ WholeStageCodegen (42) SortMergeJoin [item_sk,store_name,store_zip,item_sk,store_name,store_zip,cnt,cnt] InputAdapter WholeStageCodegen (20) - ColumnarToRow + Sort [item_sk,store_name,store_zip] InputAdapter - CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #2 - RowToColumnar - WholeStageCodegen (19) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + Exchange [item_sk,store_name,store_zip] #2 + WholeStageCodegen (19) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #3 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (3) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [cs_item_sk] #6 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] + InputAdapter + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #3 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) ColumnarToRow InputAdapter - CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometColumnarExchange [cs_item_sk,cs_order_number] #7 - CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] - CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] - CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometColumnarExchange [cr_item_sk,cr_order_number] #8 - CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter - ReusedExchange [d_date_sk,d_year] #5 + WholeStageCodegen (3) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + InputAdapter + Exchange [cs_item_sk] #6 + WholeStageCodegen (2) + HashAggregate [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] [sum,sum,isEmpty,sum,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] + CometExchange [cs_item_sk,cs_order_number] #7 + CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] + CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] + CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometExchange [cr_item_sk,cr_order_number] #8 + CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] + CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] InputAdapter - BroadcastExchange #9 - WholeStageCodegen (5) - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_store_name,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] + ReusedExchange [d_date_sk,d_year] #5 InputAdapter - BroadcastExchange #10 - WholeStageCodegen (6) + BroadcastExchange #9 + WholeStageCodegen (5) ColumnarToRow InputAdapter - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometFilter [s_store_sk,s_store_name,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_zip] InputAdapter - BroadcastExchange #11 - WholeStageCodegen (7) + BroadcastExchange #10 + WholeStageCodegen (6) ColumnarToRow InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] InputAdapter - ReusedExchange [d_date_sk,d_year] #11 + BroadcastExchange #11 + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter - BroadcastExchange #12 - WholeStageCodegen (9) - ColumnarToRow - InputAdapter - CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + ReusedExchange [d_date_sk,d_year] #11 InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 + BroadcastExchange #12 + WholeStageCodegen (9) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_marital_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] InputAdapter - BroadcastExchange #13 - WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] + ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter - BroadcastExchange #14 - WholeStageCodegen (12) + BroadcastExchange #13 + WholeStageCodegen (11) ColumnarToRow InputAdapter - CometFilter [hd_demo_sk,hd_income_band_sk] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] + CometFilter [p_promo_sk] + CometScan parquet spark_catalog.default.promotion [p_promo_sk] InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + BroadcastExchange #14 + WholeStageCodegen (12) + ColumnarToRow + InputAdapter + CometFilter [hd_demo_sk,hd_income_band_sk] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_income_band_sk] InputAdapter - BroadcastExchange #15 - WholeStageCodegen (14) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + BroadcastExchange #15 + WholeStageCodegen (14) + ColumnarToRow + InputAdapter + CometFilter [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] InputAdapter - BroadcastExchange #16 - WholeStageCodegen (16) - ColumnarToRow - InputAdapter - CometFilter [ib_income_band_sk] - CometScan parquet spark_catalog.default.income_band [ib_income_band_sk] + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter - ReusedExchange [ib_income_band_sk] #16 + BroadcastExchange #16 + WholeStageCodegen (16) + ColumnarToRow + InputAdapter + CometFilter [ib_income_band_sk] + CometScan parquet spark_catalog.default.income_band [ib_income_band_sk] InputAdapter - BroadcastExchange #17 - WholeStageCodegen (18) - ColumnarToRow - InputAdapter - CometProject [i_item_sk,i_product_name] - CometFilter [i_item_sk,i_current_price,i_color,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + BroadcastExchange #17 + WholeStageCodegen (18) + ColumnarToRow + InputAdapter + CometProject [i_item_sk,i_product_name] + CometFilter [i_item_sk,i_current_price,i_color,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_color,i_product_name] InputAdapter WholeStageCodegen (40) - ColumnarToRow + Sort [item_sk,store_name,store_zip] InputAdapter - CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #18 - RowToColumnar - WholeStageCodegen (39) - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] - HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] - Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] - BroadcastHashJoin [ss_item_sk,i_item_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + Exchange [item_sk,store_name,store_zip] #18 + WholeStageCodegen (39) + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum] [count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt)),item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,count,sum,sum,sum] + HashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,ss_wholesale_cost,ss_list_price,ss_coupon_amt] [count,sum,sum,sum,count,sum,sum,sum] + Project [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] + BroadcastHashJoin [ss_item_sk,i_item_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [hd_income_band_sk,ib_income_band_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] - BroadcastHashJoin [ss_addr_sk,ca_address_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] - BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] - BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [ss_promo_sk,p_promo_sk] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] - Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] - BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] - BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] - BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] - Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] - BroadcastHashJoin [ss_customer_sk,c_customer_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] - BroadcastHashJoin [ss_store_sk,s_store_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] - BroadcastHashJoin [ss_sold_date_sk,d_date_sk] - Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SortMergeJoin [ss_item_sk,cs_item_sk] - InputAdapter - WholeStageCodegen (21) - ColumnarToRow - InputAdapter - CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #19 - CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #20 - CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #21 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - InputAdapter - WholeStageCodegen (23) - Sort [cs_item_sk] - Project [cs_item_sk] - Filter [sale,refund] - HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk,ca_street_number,ca_street_name,ca_city,ca_zip] + BroadcastHashJoin [ss_addr_sk,ca_address_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_addr_sk,d_year,d_year,hd_income_band_sk,hd_income_band_sk] + BroadcastHashJoin [c_current_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,hd_income_band_sk] + BroadcastHashJoin [ss_hdemo_sk,hd_demo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [ss_promo_sk,p_promo_sk] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_marital_status,cd_marital_status] + Project [ss_item_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year,cd_marital_status] + BroadcastHashJoin [ss_cdemo_sk,cd_demo_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,d_year,d_year] + BroadcastHashJoin [c_first_shipto_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,d_year] + BroadcastHashJoin [c_first_sales_date_sk,d_date_sk] + Project [ss_item_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] + BroadcastHashJoin [ss_customer_sk,c_customer_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,s_store_name,s_zip] + BroadcastHashJoin [ss_store_sk,s_store_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year] + BroadcastHashJoin [ss_sold_date_sk,d_date_sk] + Project [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SortMergeJoin [ss_item_sk,cs_item_sk] + InputAdapter + WholeStageCodegen (21) + ColumnarToRow + InputAdapter + CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometExchange [ss_item_sk] #19 + CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #20 + CometFilter [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #21 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] InputAdapter - ReusedExchange [d_date_sk,d_year] #21 + WholeStageCodegen (23) + Sort [cs_item_sk] + Project [cs_item_sk] + Filter [sale,refund] + HashAggregate [cs_item_sk,sum,sum,isEmpty] [sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit)),sale,refund,sum,sum,isEmpty] + InputAdapter + ReusedExchange [cs_item_sk,sum,sum,isEmpty] #6 InputAdapter - ReusedExchange [s_store_sk,s_store_name,s_zip] #9 + ReusedExchange [d_date_sk,d_year] #21 InputAdapter - ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 + ReusedExchange [s_store_sk,s_store_name,s_zip] #9 InputAdapter - ReusedExchange [d_date_sk,d_year] #11 + ReusedExchange [c_customer_sk,c_current_cdemo_sk,c_current_hdemo_sk,c_current_addr_sk,c_first_shipto_date_sk,c_first_sales_date_sk] #10 InputAdapter ReusedExchange [d_date_sk,d_year] #11 InputAdapter - ReusedExchange [cd_demo_sk,cd_marital_status] #12 + ReusedExchange [d_date_sk,d_year] #11 InputAdapter ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter - ReusedExchange [p_promo_sk] #13 + ReusedExchange [cd_demo_sk,cd_marital_status] #12 InputAdapter - ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 + ReusedExchange [p_promo_sk] #13 InputAdapter ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter - ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 + ReusedExchange [hd_demo_sk,hd_income_band_sk] #14 InputAdapter ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter - ReusedExchange [ib_income_band_sk] #16 + ReusedExchange [ca_address_sk,ca_street_number,ca_street_name,ca_city,ca_zip] #15 InputAdapter ReusedExchange [ib_income_band_sk] #16 InputAdapter - ReusedExchange [i_item_sk,i_product_name] #17 + ReusedExchange [ib_income_band_sk] #16 + InputAdapter + ReusedExchange [i_item_sk,i_product_name] #17 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt index 0c59ffbbc..25e832248 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/explain.txt @@ -1,103 +1,75 @@ == Physical Plan == -TakeOrderedAndProject (99) -+- * Filter (98) - +- Window (97) - +- WindowGroupLimit (96) - +- * ColumnarToRow (95) - +- CometSort (94) - +- CometColumnarExchange (93) - +- RowToColumnar (92) - +- WindowGroupLimit (91) - +- * Sort (90) - +- Union (89) - :- * HashAggregate (24) - : +- * ColumnarToRow (23) - : +- CometColumnarExchange (22) - : +- RowToColumnar (21) - : +- * HashAggregate (20) - : +- * ColumnarToRow (19) - : +- CometProject (18) - : +- CometBroadcastHashJoin (17) - : :- CometProject (13) - : : +- CometBroadcastHashJoin (12) - : : :- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- CometBroadcastExchange (11) - : : +- CometFilter (10) - : : +- CometScan parquet spark_catalog.default.store (9) - : +- CometBroadcastExchange (16) - : +- CometFilter (15) - : +- CometScan parquet spark_catalog.default.item (14) - :- * HashAggregate (32) - : +- * ColumnarToRow (31) - : +- CometColumnarExchange (30) - : +- RowToColumnar (29) - : +- * HashAggregate (28) - : +- * HashAggregate (27) - : +- * ColumnarToRow (26) - : +- ReusedExchange (25) - :- * HashAggregate (40) - : +- * ColumnarToRow (39) - : +- CometColumnarExchange (38) - : +- RowToColumnar (37) - : +- * HashAggregate (36) - : +- * HashAggregate (35) - : +- * ColumnarToRow (34) - : +- ReusedExchange (33) - :- * HashAggregate (48) - : +- * ColumnarToRow (47) - : +- CometColumnarExchange (46) - : +- RowToColumnar (45) - : +- * HashAggregate (44) - : +- * HashAggregate (43) - : +- * ColumnarToRow (42) - : +- ReusedExchange (41) - :- * HashAggregate (56) - : +- * ColumnarToRow (55) - : +- CometColumnarExchange (54) - : +- RowToColumnar (53) - : +- * HashAggregate (52) - : +- * HashAggregate (51) - : +- * ColumnarToRow (50) - : +- ReusedExchange (49) - :- * HashAggregate (64) - : +- * ColumnarToRow (63) - : +- CometColumnarExchange (62) - : +- RowToColumnar (61) - : +- * HashAggregate (60) - : +- * HashAggregate (59) - : +- * ColumnarToRow (58) - : +- ReusedExchange (57) - :- * HashAggregate (72) - : +- * ColumnarToRow (71) - : +- CometColumnarExchange (70) - : +- RowToColumnar (69) - : +- * HashAggregate (68) - : +- * HashAggregate (67) - : +- * ColumnarToRow (66) - : +- ReusedExchange (65) - :- * HashAggregate (80) - : +- * ColumnarToRow (79) - : +- CometColumnarExchange (78) - : +- RowToColumnar (77) - : +- * HashAggregate (76) - : +- * HashAggregate (75) - : +- * ColumnarToRow (74) - : +- ReusedExchange (73) - +- * HashAggregate (88) - +- * ColumnarToRow (87) - +- CometColumnarExchange (86) - +- RowToColumnar (85) - +- * HashAggregate (84) - +- * HashAggregate (83) - +- * ColumnarToRow (82) - +- ReusedExchange (81) +TakeOrderedAndProject (71) ++- * Filter (70) + +- Window (69) + +- WindowGroupLimit (68) + +- * Sort (67) + +- Exchange (66) + +- WindowGroupLimit (65) + +- * Sort (64) + +- Union (63) + :- * HashAggregate (22) + : +- Exchange (21) + : +- * HashAggregate (20) + : +- * ColumnarToRow (19) + : +- CometProject (18) + : +- CometBroadcastHashJoin (17) + : :- CometProject (13) + : : +- CometBroadcastHashJoin (12) + : : :- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- CometBroadcastExchange (11) + : : +- CometFilter (10) + : : +- CometScan parquet spark_catalog.default.store (9) + : +- CometBroadcastExchange (16) + : +- CometFilter (15) + : +- CometScan parquet spark_catalog.default.item (14) + :- * HashAggregate (27) + : +- Exchange (26) + : +- * HashAggregate (25) + : +- * HashAggregate (24) + : +- ReusedExchange (23) + :- * HashAggregate (32) + : +- Exchange (31) + : +- * HashAggregate (30) + : +- * HashAggregate (29) + : +- ReusedExchange (28) + :- * HashAggregate (37) + : +- Exchange (36) + : +- * HashAggregate (35) + : +- * HashAggregate (34) + : +- ReusedExchange (33) + :- * HashAggregate (42) + : +- Exchange (41) + : +- * HashAggregate (40) + : +- * HashAggregate (39) + : +- ReusedExchange (38) + :- * HashAggregate (47) + : +- Exchange (46) + : +- * HashAggregate (45) + : +- * HashAggregate (44) + : +- ReusedExchange (43) + :- * HashAggregate (52) + : +- Exchange (51) + : +- * HashAggregate (50) + : +- * HashAggregate (49) + : +- ReusedExchange (48) + :- * HashAggregate (57) + : +- Exchange (56) + : +- * HashAggregate (55) + : +- * HashAggregate (54) + : +- ReusedExchange (53) + +- * HashAggregate (62) + +- Exchange (61) + +- * HashAggregate (60) + +- * HashAggregate (59) + +- ReusedExchange (58) (1) Scan parquet spark_catalog.default.store_sales @@ -198,388 +170,304 @@ Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as d Aggregate Attributes [2]: [sum#19, isEmpty#20] Results [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sum#21, isEmpty#22] -(21) RowToColumnar +(21) Exchange Input [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sum#21, isEmpty#22] +Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(22) CometColumnarExchange -Input [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sum#21, isEmpty#22] -Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(23) ColumnarToRow [codegen id : 2] -Input [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sum#21, isEmpty#22] - -(24) HashAggregate [codegen id : 2] +(22) HashAggregate [codegen id : 2] Input [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sum#21, isEmpty#22] Keys [8]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13] Functions [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#23] Results [9]: [i_category#17 AS i_category#24, i_class#16 AS i_class#25, i_brand#15 AS i_brand#26, i_product_name#18 AS i_product_name#27, d_year#9 AS d_year#28, d_qoy#11 AS d_qoy#29, d_moy#10 AS d_moy#30, s_store_id#13 AS s_store_id#31, cast(sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))#23 as decimal(38,2)) AS sumsales#32] -(25) ReusedExchange [Reuses operator id: 22] +(23) ReusedExchange [Reuses operator id: 21] Output [10]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, s_store_id#40, sum#41, isEmpty#42] -(26) ColumnarToRow [codegen id : 4] -Input [10]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, s_store_id#40, sum#41, isEmpty#42] - -(27) HashAggregate [codegen id : 4] +(24) HashAggregate [codegen id : 4] Input [10]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, s_store_id#40, sum#41, isEmpty#42] Keys [8]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, s_store_id#40] Functions [1]: [sum(coalesce((ss_sales_price#43 * cast(ss_quantity#44 as decimal(10,0))), 0.00))] Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#43 * cast(ss_quantity#44 as decimal(10,0))), 0.00))#23] Results [8]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, sum(coalesce((ss_sales_price#43 * cast(ss_quantity#44 as decimal(10,0))), 0.00))#23 AS sumsales#45] -(28) HashAggregate [codegen id : 4] +(25) HashAggregate [codegen id : 4] Input [8]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, sumsales#45] Keys [7]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39] Functions [1]: [partial_sum(sumsales#45)] Aggregate Attributes [2]: [sum#46, isEmpty#47] Results [9]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, sum#48, isEmpty#49] -(29) RowToColumnar -Input [9]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, sum#48, isEmpty#49] - -(30) CometColumnarExchange +(26) Exchange Input [9]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, sum#48, isEmpty#49] -Arguments: hashpartitioning(i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(31) ColumnarToRow [codegen id : 5] -Input [9]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, sum#48, isEmpty#49] - -(32) HashAggregate [codegen id : 5] +(27) HashAggregate [codegen id : 5] Input [9]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, sum#48, isEmpty#49] Keys [7]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39] Functions [1]: [sum(sumsales#45)] Aggregate Attributes [1]: [sum(sumsales#45)#50] Results [9]: [i_category#33, i_class#34, i_brand#35, i_product_name#36, d_year#37, d_qoy#38, d_moy#39, null AS s_store_id#51, sum(sumsales#45)#50 AS sumsales#52] -(33) ReusedExchange [Reuses operator id: 22] +(28) ReusedExchange [Reuses operator id: 21] Output [10]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, d_moy#59, s_store_id#60, sum#61, isEmpty#62] -(34) ColumnarToRow [codegen id : 7] -Input [10]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, d_moy#59, s_store_id#60, sum#61, isEmpty#62] - -(35) HashAggregate [codegen id : 7] +(29) HashAggregate [codegen id : 7] Input [10]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, d_moy#59, s_store_id#60, sum#61, isEmpty#62] Keys [8]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, d_moy#59, s_store_id#60] Functions [1]: [sum(coalesce((ss_sales_price#63 * cast(ss_quantity#64 as decimal(10,0))), 0.00))] Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#63 * cast(ss_quantity#64 as decimal(10,0))), 0.00))#23] Results [7]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, sum(coalesce((ss_sales_price#63 * cast(ss_quantity#64 as decimal(10,0))), 0.00))#23 AS sumsales#65] -(36) HashAggregate [codegen id : 7] +(30) HashAggregate [codegen id : 7] Input [7]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, sumsales#65] Keys [6]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58] Functions [1]: [partial_sum(sumsales#65)] Aggregate Attributes [2]: [sum#66, isEmpty#67] Results [8]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, sum#68, isEmpty#69] -(37) RowToColumnar +(31) Exchange Input [8]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, sum#68, isEmpty#69] +Arguments: hashpartitioning(i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(38) CometColumnarExchange -Input [8]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, sum#68, isEmpty#69] -Arguments: hashpartitioning(i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(39) ColumnarToRow [codegen id : 8] -Input [8]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, sum#68, isEmpty#69] - -(40) HashAggregate [codegen id : 8] +(32) HashAggregate [codegen id : 8] Input [8]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, sum#68, isEmpty#69] Keys [6]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58] Functions [1]: [sum(sumsales#65)] Aggregate Attributes [1]: [sum(sumsales#65)#70] Results [9]: [i_category#53, i_class#54, i_brand#55, i_product_name#56, d_year#57, d_qoy#58, null AS d_moy#71, null AS s_store_id#72, sum(sumsales#65)#70 AS sumsales#73] -(41) ReusedExchange [Reuses operator id: 22] +(33) ReusedExchange [Reuses operator id: 21] Output [10]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, d_qoy#79, d_moy#80, s_store_id#81, sum#82, isEmpty#83] -(42) ColumnarToRow [codegen id : 10] -Input [10]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, d_qoy#79, d_moy#80, s_store_id#81, sum#82, isEmpty#83] - -(43) HashAggregate [codegen id : 10] +(34) HashAggregate [codegen id : 10] Input [10]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, d_qoy#79, d_moy#80, s_store_id#81, sum#82, isEmpty#83] Keys [8]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, d_qoy#79, d_moy#80, s_store_id#81] Functions [1]: [sum(coalesce((ss_sales_price#84 * cast(ss_quantity#85 as decimal(10,0))), 0.00))] Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#84 * cast(ss_quantity#85 as decimal(10,0))), 0.00))#23] Results [6]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sum(coalesce((ss_sales_price#84 * cast(ss_quantity#85 as decimal(10,0))), 0.00))#23 AS sumsales#86] -(44) HashAggregate [codegen id : 10] +(35) HashAggregate [codegen id : 10] Input [6]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sumsales#86] Keys [5]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78] Functions [1]: [partial_sum(sumsales#86)] Aggregate Attributes [2]: [sum#87, isEmpty#88] Results [7]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sum#89, isEmpty#90] -(45) RowToColumnar -Input [7]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sum#89, isEmpty#90] - -(46) CometColumnarExchange -Input [7]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sum#89, isEmpty#90] -Arguments: hashpartitioning(i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(47) ColumnarToRow [codegen id : 11] +(36) Exchange Input [7]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sum#89, isEmpty#90] +Arguments: hashpartitioning(i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(48) HashAggregate [codegen id : 11] +(37) HashAggregate [codegen id : 11] Input [7]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, sum#89, isEmpty#90] Keys [5]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78] Functions [1]: [sum(sumsales#86)] Aggregate Attributes [1]: [sum(sumsales#86)#91] Results [9]: [i_category#74, i_class#75, i_brand#76, i_product_name#77, d_year#78, null AS d_qoy#92, null AS d_moy#93, null AS s_store_id#94, sum(sumsales#86)#91 AS sumsales#95] -(49) ReusedExchange [Reuses operator id: 22] +(38) ReusedExchange [Reuses operator id: 21] Output [10]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, d_year#100, d_qoy#101, d_moy#102, s_store_id#103, sum#104, isEmpty#105] -(50) ColumnarToRow [codegen id : 13] -Input [10]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, d_year#100, d_qoy#101, d_moy#102, s_store_id#103, sum#104, isEmpty#105] - -(51) HashAggregate [codegen id : 13] +(39) HashAggregate [codegen id : 13] Input [10]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, d_year#100, d_qoy#101, d_moy#102, s_store_id#103, sum#104, isEmpty#105] Keys [8]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, d_year#100, d_qoy#101, d_moy#102, s_store_id#103] Functions [1]: [sum(coalesce((ss_sales_price#106 * cast(ss_quantity#107 as decimal(10,0))), 0.00))] Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#106 * cast(ss_quantity#107 as decimal(10,0))), 0.00))#23] Results [5]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sum(coalesce((ss_sales_price#106 * cast(ss_quantity#107 as decimal(10,0))), 0.00))#23 AS sumsales#108] -(52) HashAggregate [codegen id : 13] +(40) HashAggregate [codegen id : 13] Input [5]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sumsales#108] Keys [4]: [i_category#96, i_class#97, i_brand#98, i_product_name#99] Functions [1]: [partial_sum(sumsales#108)] Aggregate Attributes [2]: [sum#109, isEmpty#110] Results [6]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sum#111, isEmpty#112] -(53) RowToColumnar -Input [6]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sum#111, isEmpty#112] - -(54) CometColumnarExchange +(41) Exchange Input [6]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sum#111, isEmpty#112] -Arguments: hashpartitioning(i_category#96, i_class#97, i_brand#98, i_product_name#99, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(i_category#96, i_class#97, i_brand#98, i_product_name#99, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(55) ColumnarToRow [codegen id : 14] -Input [6]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sum#111, isEmpty#112] - -(56) HashAggregate [codegen id : 14] +(42) HashAggregate [codegen id : 14] Input [6]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, sum#111, isEmpty#112] Keys [4]: [i_category#96, i_class#97, i_brand#98, i_product_name#99] Functions [1]: [sum(sumsales#108)] Aggregate Attributes [1]: [sum(sumsales#108)#113] Results [9]: [i_category#96, i_class#97, i_brand#98, i_product_name#99, null AS d_year#114, null AS d_qoy#115, null AS d_moy#116, null AS s_store_id#117, sum(sumsales#108)#113 AS sumsales#118] -(57) ReusedExchange [Reuses operator id: 22] +(43) ReusedExchange [Reuses operator id: 21] Output [10]: [i_category#119, i_class#120, i_brand#121, i_product_name#122, d_year#123, d_qoy#124, d_moy#125, s_store_id#126, sum#127, isEmpty#128] -(58) ColumnarToRow [codegen id : 16] -Input [10]: [i_category#119, i_class#120, i_brand#121, i_product_name#122, d_year#123, d_qoy#124, d_moy#125, s_store_id#126, sum#127, isEmpty#128] - -(59) HashAggregate [codegen id : 16] +(44) HashAggregate [codegen id : 16] Input [10]: [i_category#119, i_class#120, i_brand#121, i_product_name#122, d_year#123, d_qoy#124, d_moy#125, s_store_id#126, sum#127, isEmpty#128] Keys [8]: [i_category#119, i_class#120, i_brand#121, i_product_name#122, d_year#123, d_qoy#124, d_moy#125, s_store_id#126] Functions [1]: [sum(coalesce((ss_sales_price#129 * cast(ss_quantity#130 as decimal(10,0))), 0.00))] Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#129 * cast(ss_quantity#130 as decimal(10,0))), 0.00))#23] Results [4]: [i_category#119, i_class#120, i_brand#121, sum(coalesce((ss_sales_price#129 * cast(ss_quantity#130 as decimal(10,0))), 0.00))#23 AS sumsales#131] -(60) HashAggregate [codegen id : 16] +(45) HashAggregate [codegen id : 16] Input [4]: [i_category#119, i_class#120, i_brand#121, sumsales#131] Keys [3]: [i_category#119, i_class#120, i_brand#121] Functions [1]: [partial_sum(sumsales#131)] Aggregate Attributes [2]: [sum#132, isEmpty#133] Results [5]: [i_category#119, i_class#120, i_brand#121, sum#134, isEmpty#135] -(61) RowToColumnar -Input [5]: [i_category#119, i_class#120, i_brand#121, sum#134, isEmpty#135] - -(62) CometColumnarExchange +(46) Exchange Input [5]: [i_category#119, i_class#120, i_brand#121, sum#134, isEmpty#135] -Arguments: hashpartitioning(i_category#119, i_class#120, i_brand#121, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(i_category#119, i_class#120, i_brand#121, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(63) ColumnarToRow [codegen id : 17] -Input [5]: [i_category#119, i_class#120, i_brand#121, sum#134, isEmpty#135] - -(64) HashAggregate [codegen id : 17] +(47) HashAggregate [codegen id : 17] Input [5]: [i_category#119, i_class#120, i_brand#121, sum#134, isEmpty#135] Keys [3]: [i_category#119, i_class#120, i_brand#121] Functions [1]: [sum(sumsales#131)] Aggregate Attributes [1]: [sum(sumsales#131)#136] Results [9]: [i_category#119, i_class#120, i_brand#121, null AS i_product_name#137, null AS d_year#138, null AS d_qoy#139, null AS d_moy#140, null AS s_store_id#141, sum(sumsales#131)#136 AS sumsales#142] -(65) ReusedExchange [Reuses operator id: 22] +(48) ReusedExchange [Reuses operator id: 21] Output [10]: [i_category#143, i_class#144, i_brand#145, i_product_name#146, d_year#147, d_qoy#148, d_moy#149, s_store_id#150, sum#151, isEmpty#152] -(66) ColumnarToRow [codegen id : 19] -Input [10]: [i_category#143, i_class#144, i_brand#145, i_product_name#146, d_year#147, d_qoy#148, d_moy#149, s_store_id#150, sum#151, isEmpty#152] - -(67) HashAggregate [codegen id : 19] +(49) HashAggregate [codegen id : 19] Input [10]: [i_category#143, i_class#144, i_brand#145, i_product_name#146, d_year#147, d_qoy#148, d_moy#149, s_store_id#150, sum#151, isEmpty#152] Keys [8]: [i_category#143, i_class#144, i_brand#145, i_product_name#146, d_year#147, d_qoy#148, d_moy#149, s_store_id#150] Functions [1]: [sum(coalesce((ss_sales_price#153 * cast(ss_quantity#154 as decimal(10,0))), 0.00))] Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#153 * cast(ss_quantity#154 as decimal(10,0))), 0.00))#23] Results [3]: [i_category#143, i_class#144, sum(coalesce((ss_sales_price#153 * cast(ss_quantity#154 as decimal(10,0))), 0.00))#23 AS sumsales#155] -(68) HashAggregate [codegen id : 19] +(50) HashAggregate [codegen id : 19] Input [3]: [i_category#143, i_class#144, sumsales#155] Keys [2]: [i_category#143, i_class#144] Functions [1]: [partial_sum(sumsales#155)] Aggregate Attributes [2]: [sum#156, isEmpty#157] Results [4]: [i_category#143, i_class#144, sum#158, isEmpty#159] -(69) RowToColumnar +(51) Exchange Input [4]: [i_category#143, i_class#144, sum#158, isEmpty#159] +Arguments: hashpartitioning(i_category#143, i_class#144, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(70) CometColumnarExchange -Input [4]: [i_category#143, i_class#144, sum#158, isEmpty#159] -Arguments: hashpartitioning(i_category#143, i_class#144, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(71) ColumnarToRow [codegen id : 20] -Input [4]: [i_category#143, i_class#144, sum#158, isEmpty#159] - -(72) HashAggregate [codegen id : 20] +(52) HashAggregate [codegen id : 20] Input [4]: [i_category#143, i_class#144, sum#158, isEmpty#159] Keys [2]: [i_category#143, i_class#144] Functions [1]: [sum(sumsales#155)] Aggregate Attributes [1]: [sum(sumsales#155)#160] Results [9]: [i_category#143, i_class#144, null AS i_brand#161, null AS i_product_name#162, null AS d_year#163, null AS d_qoy#164, null AS d_moy#165, null AS s_store_id#166, sum(sumsales#155)#160 AS sumsales#167] -(73) ReusedExchange [Reuses operator id: 22] +(53) ReusedExchange [Reuses operator id: 21] Output [10]: [i_category#168, i_class#169, i_brand#170, i_product_name#171, d_year#172, d_qoy#173, d_moy#174, s_store_id#175, sum#176, isEmpty#177] -(74) ColumnarToRow [codegen id : 22] -Input [10]: [i_category#168, i_class#169, i_brand#170, i_product_name#171, d_year#172, d_qoy#173, d_moy#174, s_store_id#175, sum#176, isEmpty#177] - -(75) HashAggregate [codegen id : 22] +(54) HashAggregate [codegen id : 22] Input [10]: [i_category#168, i_class#169, i_brand#170, i_product_name#171, d_year#172, d_qoy#173, d_moy#174, s_store_id#175, sum#176, isEmpty#177] Keys [8]: [i_category#168, i_class#169, i_brand#170, i_product_name#171, d_year#172, d_qoy#173, d_moy#174, s_store_id#175] Functions [1]: [sum(coalesce((ss_sales_price#178 * cast(ss_quantity#179 as decimal(10,0))), 0.00))] Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#178 * cast(ss_quantity#179 as decimal(10,0))), 0.00))#23] Results [2]: [i_category#168, sum(coalesce((ss_sales_price#178 * cast(ss_quantity#179 as decimal(10,0))), 0.00))#23 AS sumsales#180] -(76) HashAggregate [codegen id : 22] +(55) HashAggregate [codegen id : 22] Input [2]: [i_category#168, sumsales#180] Keys [1]: [i_category#168] Functions [1]: [partial_sum(sumsales#180)] Aggregate Attributes [2]: [sum#181, isEmpty#182] Results [3]: [i_category#168, sum#183, isEmpty#184] -(77) RowToColumnar -Input [3]: [i_category#168, sum#183, isEmpty#184] - -(78) CometColumnarExchange -Input [3]: [i_category#168, sum#183, isEmpty#184] -Arguments: hashpartitioning(i_category#168, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(79) ColumnarToRow [codegen id : 23] +(56) Exchange Input [3]: [i_category#168, sum#183, isEmpty#184] +Arguments: hashpartitioning(i_category#168, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(80) HashAggregate [codegen id : 23] +(57) HashAggregate [codegen id : 23] Input [3]: [i_category#168, sum#183, isEmpty#184] Keys [1]: [i_category#168] Functions [1]: [sum(sumsales#180)] Aggregate Attributes [1]: [sum(sumsales#180)#185] Results [9]: [i_category#168, null AS i_class#186, null AS i_brand#187, null AS i_product_name#188, null AS d_year#189, null AS d_qoy#190, null AS d_moy#191, null AS s_store_id#192, sum(sumsales#180)#185 AS sumsales#193] -(81) ReusedExchange [Reuses operator id: 22] +(58) ReusedExchange [Reuses operator id: 21] Output [10]: [i_category#194, i_class#195, i_brand#196, i_product_name#197, d_year#198, d_qoy#199, d_moy#200, s_store_id#201, sum#202, isEmpty#203] -(82) ColumnarToRow [codegen id : 25] -Input [10]: [i_category#194, i_class#195, i_brand#196, i_product_name#197, d_year#198, d_qoy#199, d_moy#200, s_store_id#201, sum#202, isEmpty#203] - -(83) HashAggregate [codegen id : 25] +(59) HashAggregate [codegen id : 25] Input [10]: [i_category#194, i_class#195, i_brand#196, i_product_name#197, d_year#198, d_qoy#199, d_moy#200, s_store_id#201, sum#202, isEmpty#203] Keys [8]: [i_category#194, i_class#195, i_brand#196, i_product_name#197, d_year#198, d_qoy#199, d_moy#200, s_store_id#201] Functions [1]: [sum(coalesce((ss_sales_price#204 * cast(ss_quantity#205 as decimal(10,0))), 0.00))] Aggregate Attributes [1]: [sum(coalesce((ss_sales_price#204 * cast(ss_quantity#205 as decimal(10,0))), 0.00))#23] Results [1]: [sum(coalesce((ss_sales_price#204 * cast(ss_quantity#205 as decimal(10,0))), 0.00))#23 AS sumsales#206] -(84) HashAggregate [codegen id : 25] +(60) HashAggregate [codegen id : 25] Input [1]: [sumsales#206] Keys: [] Functions [1]: [partial_sum(sumsales#206)] Aggregate Attributes [2]: [sum#207, isEmpty#208] Results [2]: [sum#209, isEmpty#210] -(85) RowToColumnar -Input [2]: [sum#209, isEmpty#210] - -(86) CometColumnarExchange +(61) Exchange Input [2]: [sum#209, isEmpty#210] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=9] -(87) ColumnarToRow [codegen id : 26] -Input [2]: [sum#209, isEmpty#210] - -(88) HashAggregate [codegen id : 26] +(62) HashAggregate [codegen id : 26] Input [2]: [sum#209, isEmpty#210] Keys: [] Functions [1]: [sum(sumsales#206)] Aggregate Attributes [1]: [sum(sumsales#206)#211] Results [9]: [null AS i_category#212, null AS i_class#213, null AS i_brand#214, null AS i_product_name#215, null AS d_year#216, null AS d_qoy#217, null AS d_moy#218, null AS s_store_id#219, sum(sumsales#206)#211 AS sumsales#220] -(89) Union +(63) Union -(90) Sort [codegen id : 27] +(64) Sort [codegen id : 27] Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32] Arguments: [i_category#24 ASC NULLS FIRST, sumsales#32 DESC NULLS LAST], false, 0 -(91) WindowGroupLimit +(65) WindowGroupLimit Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32] Arguments: [i_category#24], [sumsales#32 DESC NULLS LAST], rank(sumsales#32), 100, Partial -(92) RowToColumnar -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32] - -(93) CometColumnarExchange +(66) Exchange Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32] -Arguments: hashpartitioning(i_category#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(i_category#24, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(94) CometSort -Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32] -Arguments: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32], [i_category#24 ASC NULLS FIRST, sumsales#32 DESC NULLS LAST] - -(95) ColumnarToRow [codegen id : 28] +(67) Sort [codegen id : 28] Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32] +Arguments: [i_category#24 ASC NULLS FIRST, sumsales#32 DESC NULLS LAST], false, 0 -(96) WindowGroupLimit +(68) WindowGroupLimit Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32] Arguments: [i_category#24], [sumsales#32 DESC NULLS LAST], rank(sumsales#32), 100, Final -(97) Window +(69) Window Input [9]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32] Arguments: [rank(sumsales#32) windowspecdefinition(i_category#24, sumsales#32 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#221], [i_category#24], [sumsales#32 DESC NULLS LAST] -(98) Filter [codegen id : 29] +(70) Filter [codegen id : 29] Input [10]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32, rk#221] Condition : (rk#221 <= 100) -(99) TakeOrderedAndProject +(71) TakeOrderedAndProject Input [10]: [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32, rk#221] Arguments: 100, [i_category#24 ASC NULLS FIRST, i_class#25 ASC NULLS FIRST, i_brand#26 ASC NULLS FIRST, i_product_name#27 ASC NULLS FIRST, d_year#28 ASC NULLS FIRST, d_qoy#29 ASC NULLS FIRST, d_moy#30 ASC NULLS FIRST, s_store_id#31 ASC NULLS FIRST, sumsales#32 ASC NULLS FIRST, rk#221 ASC NULLS FIRST], [i_category#24, i_class#25, i_brand#26, i_product_name#27, d_year#28, d_qoy#29, d_moy#30, s_store_id#31, sumsales#32, rk#221] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (104) -+- * ColumnarToRow (103) - +- CometProject (102) - +- CometFilter (101) - +- CometScan parquet spark_catalog.default.date_dim (100) +BroadcastExchange (76) ++- * ColumnarToRow (75) + +- CometProject (74) + +- CometFilter (73) + +- CometScan parquet spark_catalog.default.date_dim (72) -(100) Scan parquet spark_catalog.default.date_dim +(72) Scan parquet spark_catalog.default.date_dim Output [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(101) CometFilter +(73) CometFilter Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Condition : (((isnotnull(d_month_seq#8) AND (d_month_seq#8 >= 1212)) AND (d_month_seq#8 <= 1223)) AND isnotnull(d_date_sk#7)) -(102) CometProject +(74) CometProject Input [5]: [d_date_sk#7, d_month_seq#8, d_year#9, d_moy#10, d_qoy#11] Arguments: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11], [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -(103) ColumnarToRow [codegen id : 1] +(75) ColumnarToRow [codegen id : 1] Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] -(104) BroadcastExchange +(76) BroadcastExchange Input [4]: [d_date_sk#7, d_year#9, d_moy#10, d_qoy#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=11] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt index 00b800f52..cc9627e89 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q67a/simplified.txt @@ -5,145 +5,117 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ Window [sumsales,i_category] WindowGroupLimit [i_category,sumsales] WholeStageCodegen (28) - ColumnarToRow + Sort [i_category,sumsales] InputAdapter - CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometColumnarExchange [i_category] #1 - RowToColumnar - WindowGroupLimit [i_category,sumsales] - WholeStageCodegen (27) - Sort [i_category,sumsales] - InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] - CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 - CometProject [d_date_sk,d_year,d_moy,d_qoy] - CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] - CometBroadcastExchange [s_store_sk,s_store_id] #5 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - WholeStageCodegen (5) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] [sum(sumsales),s_store_id,sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #7 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (8) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] [sum(sumsales),d_moy,s_store_id,sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #8 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (11) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] [sum(sumsales),d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year] #9 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (14) - HashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty] [sum(sumsales),d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class,i_brand,i_product_name] #10 - RowToColumnar - WholeStageCodegen (13) - HashAggregate [i_category,i_class,i_brand,i_product_name,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [i_category,i_class,i_brand,sum,isEmpty] [sum(sumsales),i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class,i_brand] #11 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [i_category,i_class,i_brand,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (20) - HashAggregate [i_category,i_class,sum,isEmpty] [sum(sumsales),i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class] #12 - RowToColumnar - WholeStageCodegen (19) - HashAggregate [i_category,i_class,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (23) - HashAggregate [i_category,sum,isEmpty] [sum(sumsales),i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category] #13 - RowToColumnar - WholeStageCodegen (22) - HashAggregate [i_category,sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 - WholeStageCodegen (26) - HashAggregate [sum,isEmpty] [sum(sumsales),i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #14 - RowToColumnar - WholeStageCodegen (25) - HashAggregate [sumsales] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + Exchange [i_category] #1 + WindowGroupLimit [i_category,sumsales] + WholeStageCodegen (27) + Sort [i_category,sumsales] + InputAdapter + Union + WholeStageCodegen (2) + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] + InputAdapter + Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,ss_sales_price,ss_quantity] [sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year,d_moy,d_qoy] + CometFilter [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_quantity,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [d_date_sk,d_year,d_moy,d_qoy] #4 + CometProject [d_date_sk,d_year,d_moy,d_qoy] + CometFilter [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq,d_year,d_moy,d_qoy] + CometBroadcastExchange [s_store_sk,s_store_id] #5 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #6 + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + WholeStageCodegen (5) + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty] [sum(sumsales),s_store_id,sumsales,sum,isEmpty] + InputAdapter + Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #7 + WholeStageCodegen (4) + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] + InputAdapter + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + WholeStageCodegen (8) + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty] [sum(sumsales),d_moy,s_store_id,sumsales,sum,isEmpty] + InputAdapter + Exchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #8 + WholeStageCodegen (7) + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] + InputAdapter + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + WholeStageCodegen (11) + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty] [sum(sumsales),d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] + InputAdapter + Exchange [i_category,i_class,i_brand,i_product_name,d_year] #9 + WholeStageCodegen (10) + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] + InputAdapter + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + WholeStageCodegen (14) + HashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty] [sum(sumsales),d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] + InputAdapter + Exchange [i_category,i_class,i_brand,i_product_name] #10 + WholeStageCodegen (13) + HashAggregate [i_category,i_class,i_brand,i_product_name,sumsales] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] + InputAdapter + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [i_category,i_class,i_brand,sum,isEmpty] [sum(sumsales),i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] + InputAdapter + Exchange [i_category,i_class,i_brand] #11 + WholeStageCodegen (16) + HashAggregate [i_category,i_class,i_brand,sumsales] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] + InputAdapter + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + WholeStageCodegen (20) + HashAggregate [i_category,i_class,sum,isEmpty] [sum(sumsales),i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] + InputAdapter + Exchange [i_category,i_class] #12 + WholeStageCodegen (19) + HashAggregate [i_category,i_class,sumsales] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] + InputAdapter + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + WholeStageCodegen (23) + HashAggregate [i_category,sum,isEmpty] [sum(sumsales),i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] + InputAdapter + Exchange [i_category] #13 + WholeStageCodegen (22) + HashAggregate [i_category,sumsales] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] + InputAdapter + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 + WholeStageCodegen (26) + HashAggregate [sum,isEmpty] [sum(sumsales),i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty] + InputAdapter + Exchange #14 + WholeStageCodegen (25) + HashAggregate [sumsales] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] [sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00)),sumsales,sum,isEmpty] + InputAdapter + ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt index 8d8b23eae..90e9a457b 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/explain.txt @@ -1,74 +1,62 @@ == Physical Plan == -TakeOrderedAndProject (70) -+- * Project (69) - +- Window (68) - +- * ColumnarToRow (67) - +- CometSort (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometColumnarExchange (63) - +- RowToColumnar (62) - +- * HashAggregate (61) - +- Union (60) - :- * HashAggregate (43) - : +- * ColumnarToRow (42) - : +- CometColumnarExchange (41) - : +- RowToColumnar (40) - : +- * HashAggregate (39) - : +- * Project (38) - : +- * BroadcastHashJoin Inner BuildRight (37) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- BroadcastExchange (36) - : +- * BroadcastHashJoin LeftSemi BuildRight (35) - : :- * ColumnarToRow (12) - : : +- CometFilter (11) - : : +- CometScan parquet spark_catalog.default.store (10) - : +- BroadcastExchange (34) - : +- * Project (33) - : +- * Filter (32) - : +- Window (31) - : +- WindowGroupLimit (30) - : +- * Sort (29) - : +- * HashAggregate (28) - : +- * ColumnarToRow (27) - : +- CometColumnarExchange (26) - : +- RowToColumnar (25) - : +- * HashAggregate (24) - : +- * ColumnarToRow (23) - : +- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store_sales (13) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.store (15) - : +- ReusedExchange (20) - :- * HashAggregate (51) - : +- * ColumnarToRow (50) - : +- CometColumnarExchange (49) - : +- RowToColumnar (48) - : +- * HashAggregate (47) - : +- * HashAggregate (46) - : +- * ColumnarToRow (45) - : +- ReusedExchange (44) - +- * HashAggregate (59) - +- * ColumnarToRow (58) - +- CometColumnarExchange (57) - +- RowToColumnar (56) - +- * HashAggregate (55) - +- * HashAggregate (54) - +- * ColumnarToRow (53) - +- ReusedExchange (52) +TakeOrderedAndProject (58) ++- * Project (57) + +- Window (56) + +- * Sort (55) + +- Exchange (54) + +- * HashAggregate (53) + +- Exchange (52) + +- * HashAggregate (51) + +- Union (50) + :- * HashAggregate (39) + : +- Exchange (38) + : +- * HashAggregate (37) + : +- * Project (36) + : +- * BroadcastHashJoin Inner BuildRight (35) + : :- * ColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- BroadcastExchange (34) + : +- * BroadcastHashJoin LeftSemi BuildRight (33) + : :- * ColumnarToRow (12) + : : +- CometFilter (11) + : : +- CometScan parquet spark_catalog.default.store (10) + : +- BroadcastExchange (32) + : +- * Project (31) + : +- * Filter (30) + : +- Window (29) + : +- WindowGroupLimit (28) + : +- * Sort (27) + : +- * HashAggregate (26) + : +- Exchange (25) + : +- * HashAggregate (24) + : +- * ColumnarToRow (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store_sales (13) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometScan parquet spark_catalog.default.store (15) + : +- ReusedExchange (20) + :- * HashAggregate (44) + : +- Exchange (43) + : +- * HashAggregate (42) + : +- * HashAggregate (41) + : +- ReusedExchange (40) + +- * HashAggregate (49) + +- Exchange (48) + +- * HashAggregate (47) + +- * HashAggregate (46) + +- ReusedExchange (45) (1) Scan parquet spark_catalog.default.store_sales @@ -186,238 +174,204 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] Aggregate Attributes [1]: [sum#17] Results [2]: [s_state#15, sum#18] -(25) RowToColumnar +(25) Exchange Input [2]: [s_state#15, sum#18] +Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(26) CometColumnarExchange -Input [2]: [s_state#15, sum#18] -Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(27) ColumnarToRow [codegen id : 2] -Input [2]: [s_state#15, sum#18] - -(28) HashAggregate [codegen id : 2] +(26) HashAggregate [codegen id : 2] Input [2]: [s_state#15, sum#18] Keys [1]: [s_state#15] Functions [1]: [sum(UnscaledValue(ss_net_profit#11))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#11))#19] Results [3]: [s_state#15, MakeDecimal(sum(UnscaledValue(ss_net_profit#11))#19,17,2) AS _w0#20, s_state#15] -(29) Sort [codegen id : 2] +(27) Sort [codegen id : 2] Input [3]: [s_state#15, _w0#20, s_state#15] Arguments: [s_state#15 ASC NULLS FIRST, _w0#20 DESC NULLS LAST], false, 0 -(30) WindowGroupLimit +(28) WindowGroupLimit Input [3]: [s_state#15, _w0#20, s_state#15] Arguments: [s_state#15], [_w0#20 DESC NULLS LAST], rank(_w0#20), 5, Final -(31) Window +(29) Window Input [3]: [s_state#15, _w0#20, s_state#15] Arguments: [rank(_w0#20) windowspecdefinition(s_state#15, _w0#20 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS ranking#21], [s_state#15], [_w0#20 DESC NULLS LAST] -(32) Filter [codegen id : 3] +(30) Filter [codegen id : 3] Input [4]: [s_state#15, _w0#20, s_state#15, ranking#21] Condition : (ranking#21 <= 5) -(33) Project [codegen id : 3] +(31) Project [codegen id : 3] Output [1]: [s_state#15] Input [4]: [s_state#15, _w0#20, s_state#15, ranking#21] -(34) BroadcastExchange +(32) BroadcastExchange Input [1]: [s_state#15] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=2] -(35) BroadcastHashJoin [codegen id : 4] +(33) BroadcastHashJoin [codegen id : 4] Left keys [1]: [s_state#9] Right keys [1]: [s_state#15] Join type: LeftSemi Join condition: None -(36) BroadcastExchange +(34) BroadcastExchange Input [3]: [s_store_sk#7, s_county#8, s_state#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=3] -(37) BroadcastHashJoin [codegen id : 5] +(35) BroadcastHashJoin [codegen id : 5] Left keys [1]: [ss_store_sk#1] Right keys [1]: [s_store_sk#7] Join type: Inner Join condition: None -(38) Project [codegen id : 5] +(36) Project [codegen id : 5] Output [3]: [ss_net_profit#2, s_county#8, s_state#9] Input [5]: [ss_store_sk#1, ss_net_profit#2, s_store_sk#7, s_county#8, s_state#9] -(39) HashAggregate [codegen id : 5] +(37) HashAggregate [codegen id : 5] Input [3]: [ss_net_profit#2, s_county#8, s_state#9] Keys [2]: [s_state#9, s_county#8] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum#22] Results [3]: [s_state#9, s_county#8, sum#23] -(40) RowToColumnar -Input [3]: [s_state#9, s_county#8, sum#23] - -(41) CometColumnarExchange -Input [3]: [s_state#9, s_county#8, sum#23] -Arguments: hashpartitioning(s_state#9, s_county#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(42) ColumnarToRow [codegen id : 6] +(38) Exchange Input [3]: [s_state#9, s_county#8, sum#23] +Arguments: hashpartitioning(s_state#9, s_county#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(43) HashAggregate [codegen id : 6] +(39) HashAggregate [codegen id : 6] Input [3]: [s_state#9, s_county#8, sum#23] Keys [2]: [s_state#9, s_county#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#24] Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#24,17,2) as decimal(27,2)) AS total_sum#25, s_state#9, s_county#8, 0 AS g_state#26, 0 AS g_county#27, 0 AS lochierarchy#28] -(44) ReusedExchange [Reuses operator id: 41] +(40) ReusedExchange [Reuses operator id: 38] Output [3]: [s_state#29, s_county#30, sum#31] -(45) ColumnarToRow [codegen id : 12] -Input [3]: [s_state#29, s_county#30, sum#31] - -(46) HashAggregate [codegen id : 12] +(41) HashAggregate [codegen id : 12] Input [3]: [s_state#29, s_county#30, sum#31] Keys [2]: [s_state#29, s_county#30] Functions [1]: [sum(UnscaledValue(ss_net_profit#32))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#32))#24] Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#32))#24,17,2) AS total_sum#33, s_state#29] -(47) HashAggregate [codegen id : 12] +(42) HashAggregate [codegen id : 12] Input [2]: [total_sum#33, s_state#29] Keys [1]: [s_state#29] Functions [1]: [partial_sum(total_sum#33)] Aggregate Attributes [2]: [sum#34, isEmpty#35] Results [3]: [s_state#29, sum#36, isEmpty#37] -(48) RowToColumnar -Input [3]: [s_state#29, sum#36, isEmpty#37] - -(49) CometColumnarExchange +(43) Exchange Input [3]: [s_state#29, sum#36, isEmpty#37] -Arguments: hashpartitioning(s_state#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(s_state#29, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(50) ColumnarToRow [codegen id : 13] -Input [3]: [s_state#29, sum#36, isEmpty#37] - -(51) HashAggregate [codegen id : 13] +(44) HashAggregate [codegen id : 13] Input [3]: [s_state#29, sum#36, isEmpty#37] Keys [1]: [s_state#29] Functions [1]: [sum(total_sum#33)] Aggregate Attributes [1]: [sum(total_sum#33)#38] Results [6]: [sum(total_sum#33)#38 AS total_sum#39, s_state#29, null AS s_county#40, 0 AS g_state#41, 1 AS g_county#42, 1 AS lochierarchy#43] -(52) ReusedExchange [Reuses operator id: 41] +(45) ReusedExchange [Reuses operator id: 38] Output [3]: [s_state#44, s_county#45, sum#46] -(53) ColumnarToRow [codegen id : 19] -Input [3]: [s_state#44, s_county#45, sum#46] - -(54) HashAggregate [codegen id : 19] +(46) HashAggregate [codegen id : 19] Input [3]: [s_state#44, s_county#45, sum#46] Keys [2]: [s_state#44, s_county#45] Functions [1]: [sum(UnscaledValue(ss_net_profit#47))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#47))#24] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#47))#24,17,2) AS total_sum#48] -(55) HashAggregate [codegen id : 19] +(47) HashAggregate [codegen id : 19] Input [1]: [total_sum#48] Keys: [] Functions [1]: [partial_sum(total_sum#48)] Aggregate Attributes [2]: [sum#49, isEmpty#50] Results [2]: [sum#51, isEmpty#52] -(56) RowToColumnar +(48) Exchange Input [2]: [sum#51, isEmpty#52] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(57) CometColumnarExchange -Input [2]: [sum#51, isEmpty#52] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(58) ColumnarToRow [codegen id : 20] -Input [2]: [sum#51, isEmpty#52] - -(59) HashAggregate [codegen id : 20] +(49) HashAggregate [codegen id : 20] Input [2]: [sum#51, isEmpty#52] Keys: [] Functions [1]: [sum(total_sum#48)] Aggregate Attributes [1]: [sum(total_sum#48)#53] Results [6]: [sum(total_sum#48)#53 AS total_sum#54, null AS s_state#55, null AS s_county#56, 1 AS g_state#57, 1 AS g_county#58, 2 AS lochierarchy#59] -(60) Union +(50) Union -(61) HashAggregate [codegen id : 21] +(51) HashAggregate [codegen id : 21] Input [6]: [total_sum#25, s_state#9, s_county#8, g_state#26, g_county#27, lochierarchy#28] Keys [6]: [total_sum#25, s_state#9, s_county#8, g_state#26, g_county#27, lochierarchy#28] Functions: [] Aggregate Attributes: [] Results [6]: [total_sum#25, s_state#9, s_county#8, g_state#26, g_county#27, lochierarchy#28] -(62) RowToColumnar -Input [6]: [total_sum#25, s_state#9, s_county#8, g_state#26, g_county#27, lochierarchy#28] - -(63) CometColumnarExchange +(52) Exchange Input [6]: [total_sum#25, s_state#9, s_county#8, g_state#26, g_county#27, lochierarchy#28] -Arguments: hashpartitioning(total_sum#25, s_state#9, s_county#8, g_state#26, g_county#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(total_sum#25, s_state#9, s_county#8, g_state#26, g_county#27, lochierarchy#28, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(64) CometHashAggregate +(53) HashAggregate [codegen id : 22] Input [6]: [total_sum#25, s_state#9, s_county#8, g_state#26, g_county#27, lochierarchy#28] Keys [6]: [total_sum#25, s_state#9, s_county#8, g_state#26, g_county#27, lochierarchy#28] Functions: [] +Aggregate Attributes: [] +Results [5]: [total_sum#25, s_state#9, s_county#8, lochierarchy#28, CASE WHEN (g_county#27 = 0) THEN s_state#9 END AS _w0#60] -(65) CometColumnarExchange -Input [5]: [total_sum#25, s_state#9, s_county#8, lochierarchy#28, _w0#60] -Arguments: hashpartitioning(lochierarchy#28, _w0#60, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(66) CometSort +(54) Exchange Input [5]: [total_sum#25, s_state#9, s_county#8, lochierarchy#28, _w0#60] -Arguments: [total_sum#25, s_state#9, s_county#8, lochierarchy#28, _w0#60], [lochierarchy#28 ASC NULLS FIRST, _w0#60 ASC NULLS FIRST, total_sum#25 DESC NULLS LAST] +Arguments: hashpartitioning(lochierarchy#28, _w0#60, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(67) ColumnarToRow [codegen id : 22] +(55) Sort [codegen id : 23] Input [5]: [total_sum#25, s_state#9, s_county#8, lochierarchy#28, _w0#60] +Arguments: [lochierarchy#28 ASC NULLS FIRST, _w0#60 ASC NULLS FIRST, total_sum#25 DESC NULLS LAST], false, 0 -(68) Window +(56) Window Input [5]: [total_sum#25, s_state#9, s_county#8, lochierarchy#28, _w0#60] Arguments: [rank(total_sum#25) windowspecdefinition(lochierarchy#28, _w0#60, total_sum#25 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#61], [lochierarchy#28, _w0#60], [total_sum#25 DESC NULLS LAST] -(69) Project [codegen id : 23] +(57) Project [codegen id : 24] Output [5]: [total_sum#25, s_state#9, s_county#8, lochierarchy#28, rank_within_parent#61] Input [6]: [total_sum#25, s_state#9, s_county#8, lochierarchy#28, _w0#60, rank_within_parent#61] -(70) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [5]: [total_sum#25, s_state#9, s_county#8, lochierarchy#28, rank_within_parent#61] Arguments: 100, [lochierarchy#28 DESC NULLS LAST, CASE WHEN (lochierarchy#28 = 0) THEN s_state#9 END ASC NULLS FIRST, rank_within_parent#61 ASC NULLS FIRST], [total_sum#25, s_state#9, s_county#8, lochierarchy#28, rank_within_parent#61] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (75) -+- * ColumnarToRow (74) - +- CometProject (73) - +- CometFilter (72) - +- CometScan parquet spark_catalog.default.date_dim (71) +BroadcastExchange (63) ++- * ColumnarToRow (62) + +- CometProject (61) + +- CometFilter (60) + +- CometScan parquet spark_catalog.default.date_dim (59) -(71) Scan parquet spark_catalog.default.date_dim +(59) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(72) CometFilter +(60) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) -(73) CometProject +(61) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(74) ColumnarToRow [codegen id : 1] +(62) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(75) BroadcastExchange +(63) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt index 816cb34fb..67f4841b4 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q70a/simplified.txt @@ -1,107 +1,97 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (23) + WholeStageCodegen (24) Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] InputAdapter Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (22) - ColumnarToRow + WholeStageCodegen (23) + Sort [lochierarchy,_w0,total_sum] InputAdapter - CometSort [total_sum,s_state,s_county,lochierarchy,_w0] - CometColumnarExchange [lochierarchy,_w0] #1 - CometHashAggregate [total_sum,s_state,s_county,lochierarchy,_w0,g_state,g_county] - CometColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 - RowToColumnar + Exchange [lochierarchy,_w0] #1 + WholeStageCodegen (22) + HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0] + InputAdapter + Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 WholeStageCodegen (21) HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] InputAdapter Union WholeStageCodegen (6) HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_state,s_county] #3 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_county,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (4) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WindowGroupLimit [s_state,_w0] - WholeStageCodegen (2) - Sort [s_state,_w0] - HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_state] #8 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_state,ss_net_profit] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #9 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #5 + InputAdapter + Exchange [s_state,s_county] #3 + WholeStageCodegen (5) + HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] + Project [ss_net_profit,s_county,s_state] + BroadcastHashJoin [ss_store_sk,s_store_sk] + ColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (4) + BroadcastHashJoin [s_state,s_state] + ColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WindowGroupLimit [s_state,_w0] + WholeStageCodegen (2) + Sort [s_state,_w0] + HashAggregate [sum] [sum(UnscaledValue(ss_net_profit)),_w0,s_state,sum] + InputAdapter + Exchange [s_state] #8 + WholeStageCodegen (1) + HashAggregate [s_state,ss_net_profit] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [s_store_sk,s_state] #9 + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #5 WholeStageCodegen (13) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_state] #10 - RowToColumnar - WholeStageCodegen (12) - HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 + InputAdapter + Exchange [s_state] #10 + WholeStageCodegen (12) + HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 WholeStageCodegen (20) HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #11 - RowToColumnar - WholeStageCodegen (19) - HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 + InputAdapter + Exchange #11 + WholeStageCodegen (19) + HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt index 53e1bb5a6..276e36240 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/explain.txt @@ -1,72 +1,71 @@ == Physical Plan == -* ColumnarToRow (68) -+- CometTakeOrderedAndProject (67) - +- CometHashAggregate (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometProject (63) - +- CometSortMergeJoin (62) - :- CometSort (56) - : +- CometColumnarExchange (55) - : +- RowToColumnar (54) - : +- * Project (53) - : +- * BroadcastHashJoin LeftOuter BuildRight (52) - : :- * Project (47) - : : +- * BroadcastHashJoin Inner BuildRight (46) - : : :- * ColumnarToRow (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometProject (35) - : : : : +- CometBroadcastHashJoin (34) - : : : : :- CometProject (29) - : : : : : +- CometBroadcastHashJoin (28) - : : : : : :- CometProject (23) - : : : : : : +- CometBroadcastHashJoin (22) - : : : : : : :- CometProject (17) - : : : : : : : +- CometBroadcastHashJoin (16) - : : : : : : : :- CometProject (12) - : : : : : : : : +- CometBroadcastHashJoin (11) - : : : : : : : : :- CometProject (7) - : : : : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- CometBroadcastExchange (5) - : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) - : : : : : : : : +- CometBroadcastExchange (10) - : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) - : : : : : : : +- CometBroadcastExchange (15) - : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan parquet spark_catalog.default.item (13) - : : : : : : +- CometBroadcastExchange (21) - : : : : : : +- CometProject (20) - : : : : : : +- CometFilter (19) - : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) - : : : : : +- CometBroadcastExchange (27) - : : : : : +- CometProject (26) - : : : : : +- CometFilter (25) - : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) - : : : : +- CometBroadcastExchange (33) - : : : : +- CometProject (32) - : : : : +- CometFilter (31) - : : : : +- CometScan parquet spark_catalog.default.date_dim (30) - : : : +- CometBroadcastExchange (38) - : : : +- CometFilter (37) - : : : +- CometScan parquet spark_catalog.default.date_dim (36) - : : +- BroadcastExchange (45) - : : +- * ColumnarToRow (44) - : : +- CometFilter (43) - : : +- CometScan parquet spark_catalog.default.date_dim (42) - : +- BroadcastExchange (51) - : +- * ColumnarToRow (50) - : +- CometFilter (49) - : +- CometScan parquet spark_catalog.default.promotion (48) - +- CometSort (61) - +- CometColumnarExchange (60) - +- CometProject (59) - +- CometFilter (58) - +- CometScan parquet spark_catalog.default.catalog_returns (57) +TakeOrderedAndProject (67) ++- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * SortMergeJoin LeftOuter (62) + :- * Sort (55) + : +- Exchange (54) + : +- * Project (53) + : +- * BroadcastHashJoin LeftOuter BuildRight (52) + : :- * Project (47) + : : +- * BroadcastHashJoin Inner BuildRight (46) + : : :- * ColumnarToRow (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometScan parquet spark_catalog.default.date_dim (36) + : : +- BroadcastExchange (45) + : : +- * ColumnarToRow (44) + : : +- CometFilter (43) + : : +- CometScan parquet spark_catalog.default.date_dim (42) + : +- BroadcastExchange (51) + : +- * ColumnarToRow (50) + : +- CometFilter (49) + : +- CometScan parquet spark_catalog.default.promotion (48) + +- * ColumnarToRow (61) + +- CometSort (60) + +- CometExchange (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.catalog_returns (56) (1) Scan parquet spark_catalog.default.catalog_sales @@ -321,99 +320,101 @@ Join condition: None Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -(54) RowToColumnar +(54) Exchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(55) CometColumnarExchange +(55) Sort [codegen id : 4] Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 -(56) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] - -(57) Scan parquet spark_catalog.default.catalog_returns +(56) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(58) CometFilter +(57) CometFilter Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) -(59) CometProject +(58) CometProject Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] -(60) CometColumnarExchange +(59) CometExchange Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(61) CometSort +(60) CometSort Input [2]: [cr_item_sk#31, cr_order_number#32] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] -(62) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +(61) ColumnarToRow [codegen id : 5] +Input [2]: [cr_item_sk#31, cr_order_number#32] -(63) CometProject +(62) SortMergeJoin [codegen id : 6] +Left keys [2]: [cs_item_sk#4, cs_order_number#6] +Right keys [2]: [cr_item_sk#31, cr_order_number#32] +Join type: LeftOuter +Join condition: None + +(63) Project [codegen id : 6] +Output [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(64) CometHashAggregate +(64) HashAggregate [codegen id : 6] Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#34] +Results [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] -(65) CometColumnarExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +(65) Exchange +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] +Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(66) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] +(66) HashAggregate [codegen id : 7] +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#36] +Results [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count(1)#36 AS no_promo#37, count(1)#36 AS promo#38, count(1)#36 AS total_cnt#39] -(67) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -(68) ColumnarToRow [codegen id : 4] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +(67) TakeOrderedAndProject +Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] +Arguments: 100, [total_cnt#39 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (73) -+- * ColumnarToRow (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan parquet spark_catalog.default.date_dim (69) +BroadcastExchange (72) ++- * ColumnarToRow (71) + +- CometProject (70) + +- CometFilter (69) + +- CometScan parquet spark_catalog.default.date_dim (68) -(69) Scan parquet spark_catalog.default.date_dim +(68) Scan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(70) CometFilter +(69) CometFilter Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) -(71) CometProject +(70) CometProject Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] -(72) ColumnarToRow [codegen id : 1] +(71) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -(73) BroadcastExchange +(72) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt index 1022e1a90..1efc2e5d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q72/simplified.txt @@ -1,86 +1,92 @@ -WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] - CometColumnarExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] - CometProject [w_warehouse_name,i_item_desc,d_week_seq] - CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometColumnarExchange [cs_item_sk,cs_order_number] #2 - RowToColumnar - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - ColumnarToRow - InputAdapter - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 - CometFilter [d_date_sk,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] +TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] + WholeStageCodegen (7) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] + InputAdapter + Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + WholeStageCodegen (6) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] + Project [w_warehouse_name,i_item_desc,d_week_seq] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter + WholeStageCodegen (4) + Sort [cs_item_sk,cs_order_number] + InputAdapter + Exchange [cs_item_sk,cs_order_number] #2 + WholeStageCodegen (3) + Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_promo_sk,p_promo_sk] + Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] + ColumnarToRow + InputAdapter + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #7 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #8 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometFilter [d_date_sk,d_week_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #11 - WholeStageCodegen (1) + BroadcastExchange #12 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] - CometSort [cr_item_sk,cr_order_number] - CometColumnarExchange [cr_item_sk,cr_order_number] #13 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometFilter [p_promo_sk] + CometScan parquet spark_catalog.default.promotion [p_promo_sk] + InputAdapter + WholeStageCodegen (5) + ColumnarToRow + InputAdapter + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #13 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt index f63a9c4cc..0ae3bf650 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/explain.txt @@ -1,83 +1,75 @@ == Physical Plan == -TakeOrderedAndProject (79) -+- * Project (78) - +- * BroadcastHashJoin Inner BuildRight (77) - :- * Project (59) - : +- * BroadcastHashJoin Inner BuildRight (58) - : :- * BroadcastHashJoin Inner BuildRight (39) - : : :- * Filter (19) - : : : +- * HashAggregate (18) - : : : +- * ColumnarToRow (17) - : : : +- CometColumnarExchange (16) - : : : +- RowToColumnar (15) - : : : +- * HashAggregate (14) - : : : +- * ColumnarToRow (13) - : : : +- CometProject (12) - : : : +- CometBroadcastHashJoin (11) - : : : :- CometProject (7) - : : : : +- CometBroadcastHashJoin (6) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.store_sales (3) - : : : +- CometBroadcastExchange (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- BroadcastExchange (38) - : : +- * HashAggregate (37) - : : +- * ColumnarToRow (36) - : : +- CometColumnarExchange (35) - : : +- RowToColumnar (34) - : : +- * HashAggregate (33) - : : +- * ColumnarToRow (32) - : : +- CometProject (31) - : : +- CometBroadcastHashJoin (30) - : : :- CometProject (26) - : : : +- CometBroadcastHashJoin (25) - : : : :- CometFilter (21) - : : : : +- CometScan parquet spark_catalog.default.customer (20) - : : : +- CometBroadcastExchange (24) - : : : +- CometFilter (23) - : : : +- CometScan parquet spark_catalog.default.store_sales (22) - : : +- CometBroadcastExchange (29) - : : +- CometFilter (28) - : : +- CometScan parquet spark_catalog.default.date_dim (27) - : +- BroadcastExchange (57) - : +- * Filter (56) - : +- * HashAggregate (55) - : +- * ColumnarToRow (54) - : +- CometColumnarExchange (53) - : +- RowToColumnar (52) - : +- * HashAggregate (51) - : +- * ColumnarToRow (50) - : +- CometProject (49) - : +- CometBroadcastHashJoin (48) - : :- CometProject (46) - : : +- CometBroadcastHashJoin (45) - : : :- CometFilter (41) - : : : +- CometScan parquet spark_catalog.default.customer (40) - : : +- CometBroadcastExchange (44) - : : +- CometFilter (43) - : : +- CometScan parquet spark_catalog.default.web_sales (42) - : +- ReusedExchange (47) - +- BroadcastExchange (76) - +- * HashAggregate (75) - +- * ColumnarToRow (74) - +- CometColumnarExchange (73) - +- RowToColumnar (72) - +- * HashAggregate (71) - +- * ColumnarToRow (70) - +- CometProject (69) - +- CometBroadcastHashJoin (68) - :- CometProject (66) - : +- CometBroadcastHashJoin (65) - : :- CometFilter (61) - : : +- CometScan parquet spark_catalog.default.customer (60) - : +- CometBroadcastExchange (64) - : +- CometFilter (63) - : +- CometScan parquet spark_catalog.default.web_sales (62) - +- ReusedExchange (67) +TakeOrderedAndProject (71) ++- * Project (70) + +- * BroadcastHashJoin Inner BuildRight (69) + :- * Project (53) + : +- * BroadcastHashJoin Inner BuildRight (52) + : :- * BroadcastHashJoin Inner BuildRight (35) + : : :- * Filter (17) + : : : +- * HashAggregate (16) + : : : +- Exchange (15) + : : : +- * HashAggregate (14) + : : : +- * ColumnarToRow (13) + : : : +- CometProject (12) + : : : +- CometBroadcastHashJoin (11) + : : : :- CometProject (7) + : : : : +- CometBroadcastHashJoin (6) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.store_sales (3) + : : : +- CometBroadcastExchange (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- BroadcastExchange (34) + : : +- * HashAggregate (33) + : : +- Exchange (32) + : : +- * HashAggregate (31) + : : +- * ColumnarToRow (30) + : : +- CometProject (29) + : : +- CometBroadcastHashJoin (28) + : : :- CometProject (24) + : : : +- CometBroadcastHashJoin (23) + : : : :- CometFilter (19) + : : : : +- CometScan parquet spark_catalog.default.customer (18) + : : : +- CometBroadcastExchange (22) + : : : +- CometFilter (21) + : : : +- CometScan parquet spark_catalog.default.store_sales (20) + : : +- CometBroadcastExchange (27) + : : +- CometFilter (26) + : : +- CometScan parquet spark_catalog.default.date_dim (25) + : +- BroadcastExchange (51) + : +- * Filter (50) + : +- * HashAggregate (49) + : +- Exchange (48) + : +- * HashAggregate (47) + : +- * ColumnarToRow (46) + : +- CometProject (45) + : +- CometBroadcastHashJoin (44) + : :- CometProject (42) + : : +- CometBroadcastHashJoin (41) + : : :- CometFilter (37) + : : : +- CometScan parquet spark_catalog.default.customer (36) + : : +- CometBroadcastExchange (40) + : : +- CometFilter (39) + : : +- CometScan parquet spark_catalog.default.web_sales (38) + : +- ReusedExchange (43) + +- BroadcastExchange (68) + +- * HashAggregate (67) + +- Exchange (66) + +- * HashAggregate (65) + +- * ColumnarToRow (64) + +- CometProject (63) + +- CometBroadcastHashJoin (62) + :- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometFilter (55) + : : +- CometScan parquet spark_catalog.default.customer (54) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometScan parquet spark_catalog.default.web_sales (56) + +- ReusedExchange (61) (1) Scan parquet spark_catalog.default.customer @@ -150,39 +142,33 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] Aggregate Attributes [1]: [sum#11] Results [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#12] -(15) RowToColumnar +(15) Exchange Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#12] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(16) CometColumnarExchange -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#12] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(17) ColumnarToRow [codegen id : 8] -Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#12] - -(18) HashAggregate [codegen id : 8] +(16) HashAggregate [codegen id : 8] Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#12] Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10] Functions [1]: [sum(UnscaledValue(ss_net_paid#6))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#6))#13] Results [2]: [c_customer_id#2 AS customer_id#14, MakeDecimal(sum(UnscaledValue(ss_net_paid#6))#13,17,2) AS year_total#15] -(19) Filter [codegen id : 8] +(17) Filter [codegen id : 8] Input [2]: [customer_id#14, year_total#15] Condition : (isnotnull(year_total#15) AND (year_total#15 > 0.00)) -(20) Scan parquet spark_catalog.default.customer +(18) Scan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(21) CometFilter +(19) CometFilter Input [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Condition : (isnotnull(c_customer_sk#16) AND isnotnull(c_customer_id#17)) -(22) Scan parquet spark_catalog.default.store_sales +(20) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] Batched: true Location: InMemoryFileIndex [] @@ -190,96 +176,90 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#22), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_customer_sk)] ReadSchema: struct -(23) CometFilter +(21) CometFilter Input [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] Condition : isnotnull(ss_customer_sk#20) -(24) CometBroadcastExchange +(22) CometBroadcastExchange Input [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] Arguments: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] -(25) CometBroadcastHashJoin +(23) CometBroadcastHashJoin Left output [4]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19] Right output [3]: [ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] Arguments: [c_customer_sk#16], [ss_customer_sk#20], Inner, BuildRight -(26) CometProject +(24) CometProject Input [7]: [c_customer_sk#16, c_customer_id#17, c_first_name#18, c_last_name#19, ss_customer_sk#20, ss_net_paid#21, ss_sold_date_sk#22] Arguments: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, ss_sold_date_sk#22], [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, ss_sold_date_sk#22] -(27) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [2]: [d_date_sk#24, d_year#25] Condition : (((isnotnull(d_year#25) AND (d_year#25 = 2002)) AND d_year#25 IN (2001,2002)) AND isnotnull(d_date_sk#24)) -(29) CometBroadcastExchange +(27) CometBroadcastExchange Input [2]: [d_date_sk#24, d_year#25] Arguments: [d_date_sk#24, d_year#25] -(30) CometBroadcastHashJoin +(28) CometBroadcastHashJoin Left output [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, ss_sold_date_sk#22] Right output [2]: [d_date_sk#24, d_year#25] Arguments: [ss_sold_date_sk#22], [d_date_sk#24], Inner, BuildRight -(31) CometProject +(29) CometProject Input [7]: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, ss_sold_date_sk#22, d_date_sk#24, d_year#25] Arguments: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, d_year#25], [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, d_year#25] -(32) ColumnarToRow [codegen id : 2] +(30) ColumnarToRow [codegen id : 2] Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, d_year#25] -(33) HashAggregate [codegen id : 2] +(31) HashAggregate [codegen id : 2] Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, ss_net_paid#21, d_year#25] Keys [4]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#21))] Aggregate Attributes [1]: [sum#26] Results [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, sum#27] -(34) RowToColumnar -Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, sum#27] - -(35) CometColumnarExchange -Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, sum#27] -Arguments: hashpartitioning(c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(36) ColumnarToRow [codegen id : 3] +(32) Exchange Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, sum#27] +Arguments: hashpartitioning(c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(37) HashAggregate [codegen id : 3] +(33) HashAggregate [codegen id : 3] Input [5]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25, sum#27] Keys [4]: [c_customer_id#17, c_first_name#18, c_last_name#19, d_year#25] Functions [1]: [sum(UnscaledValue(ss_net_paid#21))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#21))#13] Results [4]: [c_customer_id#17 AS customer_id#28, c_first_name#18 AS customer_first_name#29, c_last_name#19 AS customer_last_name#30, MakeDecimal(sum(UnscaledValue(ss_net_paid#21))#13,17,2) AS year_total#31] -(38) BroadcastExchange +(34) BroadcastExchange Input [4]: [customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=3] -(39) BroadcastHashJoin [codegen id : 8] +(35) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#14] Right keys [1]: [customer_id#28] Join type: Inner Join condition: None -(40) Scan parquet spark_catalog.default.customer +(36) Scan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(41) CometFilter +(37) CometFilter Input [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Condition : (isnotnull(c_customer_sk#32) AND isnotnull(c_customer_id#33)) -(42) Scan parquet spark_catalog.default.web_sales +(38) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] Batched: true Location: InMemoryFileIndex [] @@ -287,92 +267,86 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#38), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(43) CometFilter +(39) CometFilter Input [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] Condition : isnotnull(ws_bill_customer_sk#36) -(44) CometBroadcastExchange +(40) CometBroadcastExchange Input [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] Arguments: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] -(45) CometBroadcastHashJoin +(41) CometBroadcastHashJoin Left output [4]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35] Right output [3]: [ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] Arguments: [c_customer_sk#32], [ws_bill_customer_sk#36], Inner, BuildRight -(46) CometProject +(42) CometProject Input [7]: [c_customer_sk#32, c_customer_id#33, c_first_name#34, c_last_name#35, ws_bill_customer_sk#36, ws_net_paid#37, ws_sold_date_sk#38] Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, ws_sold_date_sk#38], [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, ws_sold_date_sk#38] -(47) ReusedExchange [Reuses operator id: 10] +(43) ReusedExchange [Reuses operator id: 10] Output [2]: [d_date_sk#40, d_year#41] -(48) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, ws_sold_date_sk#38] Right output [2]: [d_date_sk#40, d_year#41] Arguments: [ws_sold_date_sk#38], [d_date_sk#40], Inner, BuildRight -(49) CometProject +(45) CometProject Input [7]: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, ws_sold_date_sk#38, d_date_sk#40, d_year#41] Arguments: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, d_year#41], [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, d_year#41] -(50) ColumnarToRow [codegen id : 4] +(46) ColumnarToRow [codegen id : 4] Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, d_year#41] -(51) HashAggregate [codegen id : 4] +(47) HashAggregate [codegen id : 4] Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, ws_net_paid#37, d_year#41] Keys [4]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#37))] Aggregate Attributes [1]: [sum#42] Results [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, sum#43] -(52) RowToColumnar -Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, sum#43] - -(53) CometColumnarExchange -Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, sum#43] -Arguments: hashpartitioning(c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(54) ColumnarToRow [codegen id : 5] +(48) Exchange Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, sum#43] +Arguments: hashpartitioning(c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(55) HashAggregate [codegen id : 5] +(49) HashAggregate [codegen id : 5] Input [5]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41, sum#43] Keys [4]: [c_customer_id#33, c_first_name#34, c_last_name#35, d_year#41] Functions [1]: [sum(UnscaledValue(ws_net_paid#37))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#37))#44] Results [2]: [c_customer_id#33 AS customer_id#45, MakeDecimal(sum(UnscaledValue(ws_net_paid#37))#44,17,2) AS year_total#46] -(56) Filter [codegen id : 5] +(50) Filter [codegen id : 5] Input [2]: [customer_id#45, year_total#46] Condition : (isnotnull(year_total#46) AND (year_total#46 > 0.00)) -(57) BroadcastExchange +(51) BroadcastExchange Input [2]: [customer_id#45, year_total#46] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=5] -(58) BroadcastHashJoin [codegen id : 8] +(52) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#14] Right keys [1]: [customer_id#45] Join type: Inner Join condition: None -(59) Project [codegen id : 8] +(53) Project [codegen id : 8] Output [7]: [customer_id#14, year_total#15, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#46] Input [8]: [customer_id#14, year_total#15, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, customer_id#45, year_total#46] -(60) Scan parquet spark_catalog.default.customer +(54) Scan parquet spark_catalog.default.customer Output [4]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50] Batched: true Location [not included in comparison]/{warehouse_dir}/customer] PushedFilters: [IsNotNull(c_customer_sk), IsNotNull(c_customer_id)] ReadSchema: struct -(61) CometFilter +(55) CometFilter Input [4]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50] Condition : (isnotnull(c_customer_sk#47) AND isnotnull(c_customer_id#48)) -(62) Scan parquet spark_catalog.default.web_sales +(56) Scan parquet spark_catalog.default.web_sales Output [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] Batched: true Location: InMemoryFileIndex [] @@ -380,134 +354,128 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#53), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_bill_customer_sk)] ReadSchema: struct -(63) CometFilter +(57) CometFilter Input [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] Condition : isnotnull(ws_bill_customer_sk#51) -(64) CometBroadcastExchange +(58) CometBroadcastExchange Input [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] Arguments: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] -(65) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [4]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50] Right output [3]: [ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] Arguments: [c_customer_sk#47], [ws_bill_customer_sk#51], Inner, BuildRight -(66) CometProject +(60) CometProject Input [7]: [c_customer_sk#47, c_customer_id#48, c_first_name#49, c_last_name#50, ws_bill_customer_sk#51, ws_net_paid#52, ws_sold_date_sk#53] Arguments: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, ws_sold_date_sk#53], [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, ws_sold_date_sk#53] -(67) ReusedExchange [Reuses operator id: 29] +(61) ReusedExchange [Reuses operator id: 27] Output [2]: [d_date_sk#55, d_year#56] -(68) CometBroadcastHashJoin +(62) CometBroadcastHashJoin Left output [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, ws_sold_date_sk#53] Right output [2]: [d_date_sk#55, d_year#56] Arguments: [ws_sold_date_sk#53], [d_date_sk#55], Inner, BuildRight -(69) CometProject +(63) CometProject Input [7]: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, ws_sold_date_sk#53, d_date_sk#55, d_year#56] Arguments: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, d_year#56], [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, d_year#56] -(70) ColumnarToRow [codegen id : 6] +(64) ColumnarToRow [codegen id : 6] Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, d_year#56] -(71) HashAggregate [codegen id : 6] +(65) HashAggregate [codegen id : 6] Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, ws_net_paid#52, d_year#56] Keys [4]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#52))] Aggregate Attributes [1]: [sum#57] Results [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, sum#58] -(72) RowToColumnar -Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, sum#58] - -(73) CometColumnarExchange -Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, sum#58] -Arguments: hashpartitioning(c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(74) ColumnarToRow [codegen id : 7] +(66) Exchange Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, sum#58] +Arguments: hashpartitioning(c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(75) HashAggregate [codegen id : 7] +(67) HashAggregate [codegen id : 7] Input [5]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56, sum#58] Keys [4]: [c_customer_id#48, c_first_name#49, c_last_name#50, d_year#56] Functions [1]: [sum(UnscaledValue(ws_net_paid#52))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#52))#44] Results [2]: [c_customer_id#48 AS customer_id#59, MakeDecimal(sum(UnscaledValue(ws_net_paid#52))#44,17,2) AS year_total#60] -(76) BroadcastExchange +(68) BroadcastExchange Input [2]: [customer_id#59, year_total#60] Arguments: HashedRelationBroadcastMode(List(input[0, string, true]),false), [plan_id=7] -(77) BroadcastHashJoin [codegen id : 8] +(69) BroadcastHashJoin [codegen id : 8] Left keys [1]: [customer_id#14] Right keys [1]: [customer_id#59] Join type: Inner Join condition: (CASE WHEN (year_total#46 > 0.00) THEN (year_total#60 / year_total#46) END > CASE WHEN (year_total#15 > 0.00) THEN (year_total#31 / year_total#15) END) -(78) Project [codegen id : 8] +(70) Project [codegen id : 8] Output [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] Input [9]: [customer_id#14, year_total#15, customer_id#28, customer_first_name#29, customer_last_name#30, year_total#31, year_total#46, customer_id#59, year_total#60] -(79) TakeOrderedAndProject +(71) TakeOrderedAndProject Input [3]: [customer_id#28, customer_first_name#29, customer_last_name#30] Arguments: 100, [customer_first_name#29 ASC NULLS FIRST, customer_id#28 ASC NULLS FIRST, customer_last_name#30 ASC NULLS FIRST], [customer_id#28, customer_first_name#29, customer_last_name#30] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (83) -+- * ColumnarToRow (82) - +- CometFilter (81) - +- CometScan parquet spark_catalog.default.date_dim (80) +BroadcastExchange (75) ++- * ColumnarToRow (74) + +- CometFilter (73) + +- CometScan parquet spark_catalog.default.date_dim (72) -(80) Scan parquet spark_catalog.default.date_dim +(72) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#9, d_year#10] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(81) CometFilter +(73) CometFilter Input [2]: [d_date_sk#9, d_year#10] Condition : (((isnotnull(d_year#10) AND (d_year#10 = 2001)) AND d_year#10 IN (2001,2002)) AND isnotnull(d_date_sk#9)) -(82) ColumnarToRow [codegen id : 1] +(74) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#9, d_year#10] -(83) BroadcastExchange +(75) BroadcastExchange Input [2]: [d_date_sk#9, d_year#10] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=8] -Subquery:2 Hosting operator id = 22 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#23 -BroadcastExchange (87) -+- * ColumnarToRow (86) - +- CometFilter (85) - +- CometScan parquet spark_catalog.default.date_dim (84) +Subquery:2 Hosting operator id = 20 Hosting Expression = ss_sold_date_sk#22 IN dynamicpruning#23 +BroadcastExchange (79) ++- * ColumnarToRow (78) + +- CometFilter (77) + +- CometScan parquet spark_catalog.default.date_dim (76) -(84) Scan parquet spark_catalog.default.date_dim +(76) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), In(d_year, [2001,2002]), IsNotNull(d_date_sk)] ReadSchema: struct -(85) CometFilter +(77) CometFilter Input [2]: [d_date_sk#24, d_year#25] Condition : (((isnotnull(d_year#25) AND (d_year#25 = 2002)) AND d_year#25 IN (2001,2002)) AND isnotnull(d_date_sk#24)) -(86) ColumnarToRow [codegen id : 1] +(78) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#24, d_year#25] -(87) BroadcastExchange +(79) BroadcastExchange Input [2]: [d_date_sk#24, d_year#25] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=9] -Subquery:3 Hosting operator id = 42 Hosting Expression = ws_sold_date_sk#38 IN dynamicpruning#8 +Subquery:3 Hosting operator id = 38 Hosting Expression = ws_sold_date_sk#38 IN dynamicpruning#8 -Subquery:4 Hosting operator id = 62 Hosting Expression = ws_sold_date_sk#53 IN dynamicpruning#23 +Subquery:4 Hosting operator id = 56 Hosting Expression = ws_sold_date_sk#53 IN dynamicpruning#23 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt index cf84139a1..4360a9555 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q74/simplified.txt @@ -7,11 +7,38 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] BroadcastHashJoin [customer_id,customer_id] Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 - RowToColumnar - WholeStageCodegen (1) + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + WholeStageCodegen (1) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [d_date_sk,d_year] #4 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (3) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + WholeStageCodegen (2) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] ColumnarToRow InputAdapter @@ -21,94 +48,59 @@ TakeOrderedAndProject [customer_first_name,customer_id,customer_last_name] CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #2 + CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #8 WholeStageCodegen (1) ColumnarToRow InputAdapter CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #4 + CometBroadcastExchange [d_date_sk,d_year] #9 CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (3) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ss_net_paid)),customer_id,customer_first_name,customer_last_name,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ss_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ss_customer_sk,ss_net_paid,ss_sold_date_sk] #7 - CometFilter [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_net_paid,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #8 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [d_date_sk,d_year] #9 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter BroadcastExchange #10 WholeStageCodegen (5) Filter [year_total] HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 - CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk,d_year] #4 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + WholeStageCodegen (4) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #12 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk,d_year] #4 InputAdapter BroadcastExchange #13 WholeStageCodegen (7) HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum] [sum(UnscaledValue(ws_net_paid)),customer_id,year_total,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] - CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] - CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] - CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] - CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 - CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [d_date_sk,d_year] #9 + InputAdapter + Exchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + WholeStageCodegen (6) + HashAggregate [c_customer_id,c_first_name,c_last_name,d_year,ws_net_paid] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] + CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] + CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk] + CometBroadcastHashJoin [c_customer_sk,c_customer_id,c_first_name,c_last_name,ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometFilter [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_customer_id,c_first_name,c_last_name] + CometBroadcastExchange [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] #15 + CometFilter [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_net_paid,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [d_date_sk,d_year] #9 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt index bcf6fc534..ac3e0c824 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/explain.txt @@ -1,137 +1,129 @@ == Physical Plan == -TakeOrderedAndProject (133) -+- * Project (132) - +- * SortMergeJoin Inner (131) - :- * ColumnarToRow (71) - : +- CometSort (70) - : +- CometColumnarExchange (69) - : +- RowToColumnar (68) - : +- * Filter (67) - : +- * HashAggregate (66) - : +- * ColumnarToRow (65) - : +- CometColumnarExchange (64) - : +- RowToColumnar (63) - : +- * HashAggregate (62) - : +- * ColumnarToRow (61) - : +- CometHashAggregate (60) - : +- CometColumnarExchange (59) - : +- CometHashAggregate (58) - : +- CometUnion (57) - : :- CometProject (22) - : : +- CometSortMergeJoin (21) - : : :- CometSort (15) - : : : +- CometColumnarExchange (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.item (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.date_dim (9) - : : +- CometSort (20) - : : +- CometColumnarExchange (19) - : : +- CometProject (18) - : : +- CometFilter (17) - : : +- CometScan parquet spark_catalog.default.catalog_returns (16) - : :- CometProject (39) - : : +- CometSortMergeJoin (38) - : : :- CometSort (32) - : : : +- CometColumnarExchange (31) - : : : +- CometProject (30) - : : : +- CometBroadcastHashJoin (29) - : : : :- CometProject (27) - : : : : +- CometBroadcastHashJoin (26) - : : : : :- CometFilter (24) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (23) - : : : : +- ReusedExchange (25) - : : : +- ReusedExchange (28) - : : +- CometSort (37) - : : +- CometColumnarExchange (36) - : : +- CometProject (35) - : : +- CometFilter (34) - : : +- CometScan parquet spark_catalog.default.store_returns (33) - : +- CometProject (56) - : +- CometSortMergeJoin (55) - : :- CometSort (49) - : : +- CometColumnarExchange (48) - : : +- CometProject (47) - : : +- CometBroadcastHashJoin (46) - : : :- CometProject (44) - : : : +- CometBroadcastHashJoin (43) - : : : :- CometFilter (41) - : : : : +- CometScan parquet spark_catalog.default.web_sales (40) - : : : +- ReusedExchange (42) - : : +- ReusedExchange (45) - : +- CometSort (54) - : +- CometColumnarExchange (53) - : +- CometProject (52) - : +- CometFilter (51) - : +- CometScan parquet spark_catalog.default.web_returns (50) - +- * ColumnarToRow (130) - +- CometSort (129) - +- CometColumnarExchange (128) - +- RowToColumnar (127) - +- * Filter (126) - +- * HashAggregate (125) - +- * ColumnarToRow (124) - +- CometColumnarExchange (123) - +- RowToColumnar (122) - +- * HashAggregate (121) - +- * ColumnarToRow (120) - +- CometHashAggregate (119) - +- CometColumnarExchange (118) - +- CometHashAggregate (117) - +- CometUnion (116) - :- CometProject (87) - : +- CometSortMergeJoin (86) - : :- CometSort (83) - : : +- CometColumnarExchange (82) - : : +- CometProject (81) - : : +- CometBroadcastHashJoin (80) - : : :- CometProject (76) - : : : +- CometBroadcastHashJoin (75) - : : : :- CometFilter (73) - : : : : +- CometScan parquet spark_catalog.default.catalog_sales (72) - : : : +- ReusedExchange (74) - : : +- CometBroadcastExchange (79) - : : +- CometFilter (78) - : : +- CometScan parquet spark_catalog.default.date_dim (77) - : +- CometSort (85) - : +- ReusedExchange (84) - :- CometProject (101) - : +- CometSortMergeJoin (100) - : :- CometSort (97) - : : +- CometColumnarExchange (96) - : : +- CometProject (95) - : : +- CometBroadcastHashJoin (94) - : : :- CometProject (92) - : : : +- CometBroadcastHashJoin (91) - : : : :- CometFilter (89) - : : : : +- CometScan parquet spark_catalog.default.store_sales (88) - : : : +- ReusedExchange (90) - : : +- ReusedExchange (93) - : +- CometSort (99) - : +- ReusedExchange (98) - +- CometProject (115) - +- CometSortMergeJoin (114) - :- CometSort (111) - : +- CometColumnarExchange (110) - : +- CometProject (109) - : +- CometBroadcastHashJoin (108) - : :- CometProject (106) - : : +- CometBroadcastHashJoin (105) - : : :- CometFilter (103) - : : : +- CometScan parquet spark_catalog.default.web_sales (102) - : : +- ReusedExchange (104) - : +- ReusedExchange (107) - +- CometSort (113) - +- ReusedExchange (112) +TakeOrderedAndProject (125) ++- * Project (124) + +- * SortMergeJoin Inner (123) + :- * Sort (67) + : +- Exchange (66) + : +- * Filter (65) + : +- * HashAggregate (64) + : +- Exchange (63) + : +- * HashAggregate (62) + : +- * ColumnarToRow (61) + : +- CometHashAggregate (60) + : +- CometExchange (59) + : +- CometHashAggregate (58) + : +- CometUnion (57) + : :- CometProject (22) + : : +- CometSortMergeJoin (21) + : : :- CometSort (15) + : : : +- CometExchange (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.item (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan parquet spark_catalog.default.date_dim (9) + : : +- CometSort (20) + : : +- CometExchange (19) + : : +- CometProject (18) + : : +- CometFilter (17) + : : +- CometScan parquet spark_catalog.default.catalog_returns (16) + : :- CometProject (39) + : : +- CometSortMergeJoin (38) + : : :- CometSort (32) + : : : +- CometExchange (31) + : : : +- CometProject (30) + : : : +- CometBroadcastHashJoin (29) + : : : :- CometProject (27) + : : : : +- CometBroadcastHashJoin (26) + : : : : :- CometFilter (24) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (23) + : : : : +- ReusedExchange (25) + : : : +- ReusedExchange (28) + : : +- CometSort (37) + : : +- CometExchange (36) + : : +- CometProject (35) + : : +- CometFilter (34) + : : +- CometScan parquet spark_catalog.default.store_returns (33) + : +- CometProject (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometProject (47) + : : +- CometBroadcastHashJoin (46) + : : :- CometProject (44) + : : : +- CometBroadcastHashJoin (43) + : : : :- CometFilter (41) + : : : : +- CometScan parquet spark_catalog.default.web_sales (40) + : : : +- ReusedExchange (42) + : : +- ReusedExchange (45) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometScan parquet spark_catalog.default.web_returns (50) + +- * Sort (122) + +- Exchange (121) + +- * Filter (120) + +- * HashAggregate (119) + +- Exchange (118) + +- * HashAggregate (117) + +- * ColumnarToRow (116) + +- CometHashAggregate (115) + +- CometExchange (114) + +- CometHashAggregate (113) + +- CometUnion (112) + :- CometProject (83) + : +- CometSortMergeJoin (82) + : :- CometSort (79) + : : +- CometExchange (78) + : : +- CometProject (77) + : : +- CometBroadcastHashJoin (76) + : : :- CometProject (72) + : : : +- CometBroadcastHashJoin (71) + : : : :- CometFilter (69) + : : : : +- CometScan parquet spark_catalog.default.catalog_sales (68) + : : : +- ReusedExchange (70) + : : +- CometBroadcastExchange (75) + : : +- CometFilter (74) + : : +- CometScan parquet spark_catalog.default.date_dim (73) + : +- CometSort (81) + : +- ReusedExchange (80) + :- CometProject (97) + : +- CometSortMergeJoin (96) + : :- CometSort (93) + : : +- CometExchange (92) + : : +- CometProject (91) + : : +- CometBroadcastHashJoin (90) + : : :- CometProject (88) + : : : +- CometBroadcastHashJoin (87) + : : : :- CometFilter (85) + : : : : +- CometScan parquet spark_catalog.default.store_sales (84) + : : : +- ReusedExchange (86) + : : +- ReusedExchange (89) + : +- CometSort (95) + : +- ReusedExchange (94) + +- CometProject (111) + +- CometSortMergeJoin (110) + :- CometSort (107) + : +- CometExchange (106) + : +- CometProject (105) + : +- CometBroadcastHashJoin (104) + : :- CometProject (102) + : : +- CometBroadcastHashJoin (101) + : : :- CometFilter (99) + : : : +- CometScan parquet spark_catalog.default.web_sales (98) + : : +- ReusedExchange (100) + : +- ReusedExchange (103) + +- CometSort (109) + +- ReusedExchange (108) (1) Scan parquet spark_catalog.default.catalog_sales @@ -198,9 +190,9 @@ Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -(14) CometColumnarExchange +(14) CometExchange Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] @@ -221,9 +213,9 @@ Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -(19) CometColumnarExchange +(19) CometExchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -274,9 +266,9 @@ Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -(31) CometColumnarExchange +(31) CometExchange Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] @@ -297,9 +289,9 @@ Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -(36) CometColumnarExchange +(36) CometExchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -350,9 +342,9 @@ Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -(48) CometColumnarExchange +(48) CometExchange Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] @@ -373,9 +365,9 @@ Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -(53) CometColumnarExchange +(53) CometExchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -400,9 +392,9 @@ Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_ Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] Functions: [] -(59) CometColumnarExchange +(59) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] @@ -419,42 +411,30 @@ Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#2 Aggregate Attributes [2]: [sum#62, sum#63] Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] -(63) RowToColumnar +(63) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(64) CometColumnarExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(65) ColumnarToRow [codegen id : 2] -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] - -(66) HashAggregate [codegen id : 2] +(64) HashAggregate [codegen id : 2] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#64, sum#65] Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Aggregate Attributes [2]: [sum(sales_cnt#20)#66, sum(UnscaledValue(sales_amt#21))#67] Results [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum(sales_cnt#20)#66 AS sales_cnt#68, MakeDecimal(sum(UnscaledValue(sales_amt#21))#67,18,2) AS sales_amt#69] -(67) Filter [codegen id : 2] +(65) Filter [codegen id : 2] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] Condition : isnotnull(sales_cnt#68) -(68) RowToColumnar -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] - -(69) CometColumnarExchange -Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(70) CometSort +(66) Exchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] -Arguments: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69], [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST] +Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(71) ColumnarToRow [codegen id : 3] +(67) Sort [codegen id : 3] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69] +Arguments: [i_brand_id#8 ASC NULLS FIRST, i_class_id#9 ASC NULLS FIRST, i_category_id#10 ASC NULLS FIRST, i_manufact_id#12 ASC NULLS FIRST], false, 0 -(72) Scan parquet spark_catalog.default.catalog_sales +(68) Scan parquet spark_catalog.default.catalog_sales Output [5]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74] Batched: true Location: InMemoryFileIndex [] @@ -462,71 +442,71 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#74), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk)] ReadSchema: struct -(73) CometFilter +(69) CometFilter Input [5]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74] Condition : isnotnull(cs_item_sk#70) -(74) ReusedExchange [Reuses operator id: 6] +(70) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] -(75) CometBroadcastHashJoin +(71) CometBroadcastHashJoin Left output [5]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74] Right output [5]: [i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Arguments: [cs_item_sk#70], [i_item_sk#76], Inner, BuildRight -(76) CometProject +(72) CometProject Input [10]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_item_sk#76, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Arguments: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80], [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] -(77) Scan parquet spark_catalog.default.date_dim +(73) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#81, d_year#82] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(78) CometFilter +(74) CometFilter Input [2]: [d_date_sk#81, d_year#82] Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) -(79) CometBroadcastExchange +(75) CometBroadcastExchange Input [2]: [d_date_sk#81, d_year#82] Arguments: [d_date_sk#81, d_year#82] -(80) CometBroadcastHashJoin +(76) CometBroadcastHashJoin Left output [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Right output [2]: [d_date_sk#81, d_year#82] Arguments: [cs_sold_date_sk#74], [d_date_sk#81], Inner, BuildRight -(81) CometProject +(77) CometProject Input [11]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, cs_sold_date_sk#74, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_date_sk#81, d_year#82] Arguments: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82], [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] -(82) CometColumnarExchange +(78) CometExchange Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] -Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(cs_order_number#71, cs_item_sk#70, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] -(83) CometSort +(79) CometSort Input [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] Arguments: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82], [cs_order_number#71 ASC NULLS FIRST, cs_item_sk#70 ASC NULLS FIRST] -(84) ReusedExchange [Reuses operator id: 19] +(80) ReusedExchange [Reuses operator id: 19] Output [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] -(85) CometSort +(81) CometSort Input [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] Arguments: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86], [cr_order_number#84 ASC NULLS FIRST, cr_item_sk#83 ASC NULLS FIRST] -(86) CometSortMergeJoin +(82) CometSortMergeJoin Left output [9]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82] Right output [4]: [cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] Arguments: [cs_order_number#71, cs_item_sk#70], [cr_order_number#84, cr_item_sk#83], LeftOuter -(87) CometProject +(83) CometProject Input [13]: [cs_item_sk#70, cs_order_number#71, cs_quantity#72, cs_ext_sales_price#73, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, d_year#82, cr_item_sk#83, cr_order_number#84, cr_return_quantity#85, cr_return_amount#86] Arguments: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88], [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, (cs_quantity#72 - coalesce(cr_return_quantity#85, 0)) AS sales_cnt#87, (cs_ext_sales_price#73 - coalesce(cr_return_amount#86, 0.00)) AS sales_amt#88] -(88) Scan parquet spark_catalog.default.store_sales +(84) Scan parquet spark_catalog.default.store_sales Output [5]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93] Batched: true Location: InMemoryFileIndex [] @@ -534,59 +514,59 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#93), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(89) CometFilter +(85) CometFilter Input [5]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93] Condition : isnotnull(ss_item_sk#89) -(90) ReusedExchange [Reuses operator id: 6] +(86) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] -(91) CometBroadcastHashJoin +(87) CometBroadcastHashJoin Left output [5]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93] Right output [5]: [i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] Arguments: [ss_item_sk#89], [i_item_sk#95], Inner, BuildRight -(92) CometProject +(88) CometProject Input [10]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93, i_item_sk#95, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] Arguments: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99], [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] -(93) ReusedExchange [Reuses operator id: 79] +(89) ReusedExchange [Reuses operator id: 75] Output [2]: [d_date_sk#100, d_year#101] -(94) CometBroadcastHashJoin +(90) CometBroadcastHashJoin Left output [9]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99] Right output [2]: [d_date_sk#100, d_year#101] Arguments: [ss_sold_date_sk#93], [d_date_sk#100], Inner, BuildRight -(95) CometProject +(91) CometProject Input [11]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, ss_sold_date_sk#93, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_date_sk#100, d_year#101] Arguments: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101], [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101] -(96) CometColumnarExchange +(92) CometExchange Input [9]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101] -Arguments: hashpartitioning(ss_ticket_number#90, ss_item_sk#89, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(ss_ticket_number#90, ss_item_sk#89, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] -(97) CometSort +(93) CometSort Input [9]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101] Arguments: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101], [ss_ticket_number#90 ASC NULLS FIRST, ss_item_sk#89 ASC NULLS FIRST] -(98) ReusedExchange [Reuses operator id: 36] +(94) ReusedExchange [Reuses operator id: 36] Output [4]: [sr_item_sk#102, sr_ticket_number#103, sr_return_quantity#104, sr_return_amt#105] -(99) CometSort +(95) CometSort Input [4]: [sr_item_sk#102, sr_ticket_number#103, sr_return_quantity#104, sr_return_amt#105] Arguments: [sr_item_sk#102, sr_ticket_number#103, sr_return_quantity#104, sr_return_amt#105], [sr_ticket_number#103 ASC NULLS FIRST, sr_item_sk#102 ASC NULLS FIRST] -(100) CometSortMergeJoin +(96) CometSortMergeJoin Left output [9]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101] Right output [4]: [sr_item_sk#102, sr_ticket_number#103, sr_return_quantity#104, sr_return_amt#105] Arguments: [ss_ticket_number#90, ss_item_sk#89], [sr_ticket_number#103, sr_item_sk#102], LeftOuter -(101) CometProject +(97) CometProject Input [13]: [ss_item_sk#89, ss_ticket_number#90, ss_quantity#91, ss_ext_sales_price#92, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, d_year#101, sr_item_sk#102, sr_ticket_number#103, sr_return_quantity#104, sr_return_amt#105] Arguments: [d_year#101, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, sales_cnt#106, sales_amt#107], [d_year#101, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, (ss_quantity#91 - coalesce(sr_return_quantity#104, 0)) AS sales_cnt#106, (ss_ext_sales_price#92 - coalesce(sr_return_amt#105, 0.00)) AS sales_amt#107] -(102) Scan parquet spark_catalog.default.web_sales +(98) Scan parquet spark_catalog.default.web_sales Output [5]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112] Batched: true Location: InMemoryFileIndex [] @@ -594,160 +574,148 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#112), dynamicpruningexpression(ws_s PushedFilters: [IsNotNull(ws_item_sk)] ReadSchema: struct -(103) CometFilter +(99) CometFilter Input [5]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112] Condition : isnotnull(ws_item_sk#108) -(104) ReusedExchange [Reuses operator id: 6] +(100) ReusedExchange [Reuses operator id: 6] Output [5]: [i_item_sk#114, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] -(105) CometBroadcastHashJoin +(101) CometBroadcastHashJoin Left output [5]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112] Right output [5]: [i_item_sk#114, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] Arguments: [ws_item_sk#108], [i_item_sk#114], Inner, BuildRight -(106) CometProject +(102) CometProject Input [10]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112, i_item_sk#114, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] Arguments: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118], [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] -(107) ReusedExchange [Reuses operator id: 79] +(103) ReusedExchange [Reuses operator id: 75] Output [2]: [d_date_sk#119, d_year#120] -(108) CometBroadcastHashJoin +(104) CometBroadcastHashJoin Left output [9]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118] Right output [2]: [d_date_sk#119, d_year#120] Arguments: [ws_sold_date_sk#112], [d_date_sk#119], Inner, BuildRight -(109) CometProject +(105) CometProject Input [11]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, ws_sold_date_sk#112, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_date_sk#119, d_year#120] Arguments: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120], [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120] -(110) CometColumnarExchange +(106) CometExchange Input [9]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120] -Arguments: hashpartitioning(ws_order_number#109, ws_item_sk#108, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: hashpartitioning(ws_order_number#109, ws_item_sk#108, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] -(111) CometSort +(107) CometSort Input [9]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120] Arguments: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120], [ws_order_number#109 ASC NULLS FIRST, ws_item_sk#108 ASC NULLS FIRST] -(112) ReusedExchange [Reuses operator id: 53] +(108) ReusedExchange [Reuses operator id: 53] Output [4]: [wr_item_sk#121, wr_order_number#122, wr_return_quantity#123, wr_return_amt#124] -(113) CometSort +(109) CometSort Input [4]: [wr_item_sk#121, wr_order_number#122, wr_return_quantity#123, wr_return_amt#124] Arguments: [wr_item_sk#121, wr_order_number#122, wr_return_quantity#123, wr_return_amt#124], [wr_order_number#122 ASC NULLS FIRST, wr_item_sk#121 ASC NULLS FIRST] -(114) CometSortMergeJoin +(110) CometSortMergeJoin Left output [9]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120] Right output [4]: [wr_item_sk#121, wr_order_number#122, wr_return_quantity#123, wr_return_amt#124] Arguments: [ws_order_number#109, ws_item_sk#108], [wr_order_number#122, wr_item_sk#121], LeftOuter -(115) CometProject +(111) CometProject Input [13]: [ws_item_sk#108, ws_order_number#109, ws_quantity#110, ws_ext_sales_price#111, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, d_year#120, wr_item_sk#121, wr_order_number#122, wr_return_quantity#123, wr_return_amt#124] Arguments: [d_year#120, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, sales_cnt#125, sales_amt#126], [d_year#120, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, (ws_quantity#110 - coalesce(wr_return_quantity#123, 0)) AS sales_cnt#125, (ws_ext_sales_price#111 - coalesce(wr_return_amt#124, 0.00)) AS sales_amt#126] -(116) CometUnion +(112) CometUnion Child 0 Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Child 1 Input [7]: [d_year#101, i_brand_id#96, i_class_id#97, i_category_id#98, i_manufact_id#99, sales_cnt#106, sales_amt#107] Child 2 Input [7]: [d_year#120, i_brand_id#115, i_class_id#116, i_category_id#117, i_manufact_id#118, sales_cnt#125, sales_amt#126] -(117) CometHashAggregate +(113) CometHashAggregate Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Functions: [] -(118) CometColumnarExchange +(114) CometExchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] -(119) CometHashAggregate +(115) CometHashAggregate Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Keys [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Functions: [] -(120) ColumnarToRow [codegen id : 4] +(116) ColumnarToRow [codegen id : 4] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] -(121) HashAggregate [codegen id : 4] +(117) HashAggregate [codegen id : 4] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#87, sales_amt#88] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [partial_sum(sales_cnt#87), partial_sum(UnscaledValue(sales_amt#88))] Aggregate Attributes [2]: [sum#127, sum#128] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#129, sum#130] -(122) RowToColumnar +(118) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#129, sum#130] +Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=14] -(123) CometColumnarExchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#129, sum#130] -Arguments: hashpartitioning(d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] - -(124) ColumnarToRow [codegen id : 5] -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#129, sum#130] - -(125) HashAggregate [codegen id : 5] +(119) HashAggregate [codegen id : 5] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum#129, sum#130] Keys [5]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Functions [2]: [sum(sales_cnt#87), sum(UnscaledValue(sales_amt#88))] Aggregate Attributes [2]: [sum(sales_cnt#87)#66, sum(UnscaledValue(sales_amt#88))#67] Results [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sum(sales_cnt#87)#66 AS sales_cnt#131, MakeDecimal(sum(UnscaledValue(sales_amt#88))#67,18,2) AS sales_amt#132] -(126) Filter [codegen id : 5] +(120) Filter [codegen id : 5] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] Condition : isnotnull(sales_cnt#131) -(127) RowToColumnar -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] - -(128) CometColumnarExchange -Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] -Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] - -(129) CometSort +(121) Exchange Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] -Arguments: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132], [i_brand_id#77 ASC NULLS FIRST, i_class_id#78 ASC NULLS FIRST, i_category_id#79 ASC NULLS FIRST, i_manufact_id#80 ASC NULLS FIRST] +Arguments: hashpartitioning(i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, 5), ENSURE_REQUIREMENTS, [plan_id=15] -(130) ColumnarToRow [codegen id : 6] +(122) Sort [codegen id : 6] Input [7]: [d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] +Arguments: [i_brand_id#77 ASC NULLS FIRST, i_class_id#78 ASC NULLS FIRST, i_category_id#79 ASC NULLS FIRST, i_manufact_id#80 ASC NULLS FIRST], false, 0 -(131) SortMergeJoin [codegen id : 7] +(123) SortMergeJoin [codegen id : 7] Left keys [4]: [i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Right keys [4]: [i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80] Join type: Inner Join condition: ((cast(sales_cnt#68 as decimal(17,2)) / cast(sales_cnt#131 as decimal(17,2))) < 0.90000000000000000000) -(132) Project [codegen id : 7] +(124) Project [codegen id : 7] Output [10]: [d_year#82 AS prev_year#133, d_year#14 AS year#134, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#131 AS prev_yr_cnt#135, sales_cnt#68 AS curr_yr_cnt#136, (sales_cnt#68 - sales_cnt#131) AS sales_cnt_diff#137, (sales_amt#69 - sales_amt#132) AS sales_amt_diff#138] Input [14]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#68, sales_amt#69, d_year#82, i_brand_id#77, i_class_id#78, i_category_id#79, i_manufact_id#80, sales_cnt#131, sales_amt#132] -(133) TakeOrderedAndProject +(125) TakeOrderedAndProject Input [10]: [prev_year#133, year#134, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#135, curr_yr_cnt#136, sales_cnt_diff#137, sales_amt_diff#138] Arguments: 100, [sales_cnt_diff#137 ASC NULLS FIRST, sales_amt_diff#138 ASC NULLS FIRST], [prev_year#133, year#134, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, prev_yr_cnt#135, curr_yr_cnt#136, sales_cnt_diff#137, sales_amt_diff#138] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#5 IN dynamicpruning#6 -BroadcastExchange (137) -+- * ColumnarToRow (136) - +- CometFilter (135) - +- CometScan parquet spark_catalog.default.date_dim (134) +BroadcastExchange (129) ++- * ColumnarToRow (128) + +- CometFilter (127) + +- CometScan parquet spark_catalog.default.date_dim (126) -(134) Scan parquet spark_catalog.default.date_dim +(126) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#13, d_year#14] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_date_sk)] ReadSchema: struct -(135) CometFilter +(127) CometFilter Input [2]: [d_date_sk#13, d_year#14] Condition : ((isnotnull(d_year#14) AND (d_year#14 = 2002)) AND isnotnull(d_date_sk#13)) -(136) ColumnarToRow [codegen id : 1] +(128) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#13, d_year#14] -(137) BroadcastExchange +(129) BroadcastExchange Input [2]: [d_date_sk#13, d_year#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=16] @@ -755,33 +723,33 @@ Subquery:2 Hosting operator id = 23 Hosting Expression = ss_sold_date_sk#26 IN d Subquery:3 Hosting operator id = 40 Hosting Expression = ws_sold_date_sk#46 IN dynamicpruning#6 -Subquery:4 Hosting operator id = 72 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#75 -BroadcastExchange (141) -+- * ColumnarToRow (140) - +- CometFilter (139) - +- CometScan parquet spark_catalog.default.date_dim (138) +Subquery:4 Hosting operator id = 68 Hosting Expression = cs_sold_date_sk#74 IN dynamicpruning#75 +BroadcastExchange (133) ++- * ColumnarToRow (132) + +- CometFilter (131) + +- CometScan parquet spark_catalog.default.date_dim (130) -(138) Scan parquet spark_catalog.default.date_dim +(130) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#81, d_year#82] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk)] ReadSchema: struct -(139) CometFilter +(131) CometFilter Input [2]: [d_date_sk#81, d_year#82] Condition : ((isnotnull(d_year#82) AND (d_year#82 = 2001)) AND isnotnull(d_date_sk#81)) -(140) ColumnarToRow [codegen id : 1] +(132) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#81, d_year#82] -(141) BroadcastExchange +(133) BroadcastExchange Input [2]: [d_date_sk#81, d_year#82] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=17] -Subquery:5 Hosting operator id = 88 Hosting Expression = ss_sold_date_sk#93 IN dynamicpruning#75 +Subquery:5 Hosting operator id = 84 Hosting Expression = ss_sold_date_sk#93 IN dynamicpruning#75 -Subquery:6 Hosting operator id = 102 Hosting Expression = ws_sold_date_sk#112 IN dynamicpruning#75 +Subquery:6 Hosting operator id = 98 Hosting Expression = ws_sold_date_sk#112 IN dynamicpruning#75 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt index 6fb16b648..a80e7c7ef 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q75/simplified.txt @@ -4,163 +4,155 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i SortMergeJoin [i_brand_id,i_class_id,i_category_id,i_manufact_id,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_cnt] InputAdapter WholeStageCodegen (3) - ColumnarToRow + Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] InputAdapter - CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 - RowToColumnar - WholeStageCodegen (2) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #4 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] - CometBroadcastExchange [d_date_sk,d_year] #7 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometColumnarExchange [cr_order_number,cr_item_sk] #8 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #9 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #10 - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #11 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #7 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometColumnarExchange [wr_order_number,wr_item_sk] #12 - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + WholeStageCodegen (2) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + InputAdapter + Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + WholeStageCodegen (1) + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #4 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometBroadcastExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + CometProject [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand_id,i_class_id,i_category_id,i_category,i_manufact_id] + CometBroadcastExchange [d_date_sk,d_year] #7 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometExchange [cr_order_number,cr_item_sk] #8 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #9 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + ReusedExchange [d_date_sk,d_year] #7 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometExchange [sr_ticket_number,sr_item_sk] #10 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #11 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + ReusedExchange [d_date_sk,d_year] #7 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometExchange [wr_order_number,wr_item_sk] #12 + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] InputAdapter WholeStageCodegen (6) - ColumnarToRow + Sort [i_brand_id,i_class_id,i_category_id,i_manufact_id] InputAdapter - CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 - RowToColumnar - WholeStageCodegen (5) - Filter [sales_cnt] - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 - CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #16 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #2 - BroadcastExchange #17 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - CometBroadcastExchange [d_date_sk,d_year] #18 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #8 - CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #19 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 - CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #10 - CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #20 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #2 - ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 - ReusedExchange [d_date_sk,d_year] #18 - CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #12 + Exchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + WholeStageCodegen (5) + Filter [sales_cnt] + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum] [sum(sales_cnt),sum(UnscaledValue(sales_amt)),sales_cnt,sales_amt,sum,sum] + InputAdapter + Exchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + WholeStageCodegen (4) + HashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] [sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [cs_order_number,cs_item_sk] #16 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #2 + BroadcastExchange #17 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + CometBroadcastExchange [d_date_sk,d_year] #18 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + ReusedExchange [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] #8 + CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ss_ticket_number,ss_item_sk] #19 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + ReusedExchange [d_date_sk,d_year] #18 + CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + ReusedExchange [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] #10 + CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] + CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometExchange [ws_order_number,ws_item_sk] #20 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #2 + ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 + ReusedExchange [d_date_sk,d_year] #18 + CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + ReusedExchange [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] #12 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt index b2c2d2797..b72c8bebb 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/explain.txt @@ -1,125 +1,103 @@ == Physical Plan == -* ColumnarToRow (121) -+- CometTakeOrderedAndProject (120) - +- CometHashAggregate (119) - +- CometColumnarExchange (118) - +- RowToColumnar (117) - +- * HashAggregate (116) - +- Union (115) - :- * HashAggregate (98) - : +- * ColumnarToRow (97) - : +- CometColumnarExchange (96) - : +- RowToColumnar (95) - : +- * HashAggregate (94) - : +- Union (93) - : :- * Project (36) - : : +- * BroadcastHashJoin LeftOuter BuildRight (35) - : : :- * HashAggregate (19) - : : : +- * ColumnarToRow (18) - : : : +- CometColumnarExchange (17) - : : : +- RowToColumnar (16) - : : : +- * HashAggregate (15) - : : : +- * ColumnarToRow (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.store (9) - : : +- BroadcastExchange (34) - : : +- * HashAggregate (33) - : : +- * ColumnarToRow (32) - : : +- CometColumnarExchange (31) - : : +- RowToColumnar (30) - : : +- * HashAggregate (29) - : : +- * ColumnarToRow (28) - : : +- CometProject (27) - : : +- CometBroadcastHashJoin (26) - : : :- CometProject (24) - : : : +- CometBroadcastHashJoin (23) - : : : :- CometFilter (21) - : : : : +- CometScan parquet spark_catalog.default.store_returns (20) - : : : +- ReusedExchange (22) - : : +- ReusedExchange (25) - : :- * Project (59) - : : +- * BroadcastNestedLoopJoin Inner BuildLeft (58) - : : :- BroadcastExchange (47) - : : : +- * HashAggregate (46) - : : : +- * ColumnarToRow (45) - : : : +- CometColumnarExchange (44) - : : : +- RowToColumnar (43) - : : : +- * HashAggregate (42) - : : : +- * ColumnarToRow (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometScan parquet spark_catalog.default.catalog_sales (37) - : : : +- ReusedExchange (38) - : : +- * HashAggregate (57) - : : +- * ColumnarToRow (56) - : : +- CometColumnarExchange (55) - : : +- RowToColumnar (54) - : : +- * HashAggregate (53) - : : +- * ColumnarToRow (52) - : : +- CometProject (51) - : : +- CometBroadcastHashJoin (50) - : : :- CometScan parquet spark_catalog.default.catalog_returns (48) - : : +- ReusedExchange (49) - : +- * Project (92) - : +- * BroadcastHashJoin LeftOuter BuildRight (91) - : :- * HashAggregate (75) - : : +- * ColumnarToRow (74) - : : +- CometColumnarExchange (73) - : : +- RowToColumnar (72) - : : +- * HashAggregate (71) - : : +- * ColumnarToRow (70) - : : +- CometProject (69) - : : +- CometBroadcastHashJoin (68) - : : :- CometProject (64) - : : : +- CometBroadcastHashJoin (63) - : : : :- CometFilter (61) - : : : : +- CometScan parquet spark_catalog.default.web_sales (60) - : : : +- ReusedExchange (62) - : : +- CometBroadcastExchange (67) - : : +- CometFilter (66) - : : +- CometScan parquet spark_catalog.default.web_page (65) - : +- BroadcastExchange (90) - : +- * HashAggregate (89) - : +- * ColumnarToRow (88) - : +- CometColumnarExchange (87) - : +- RowToColumnar (86) - : +- * HashAggregate (85) - : +- * ColumnarToRow (84) - : +- CometProject (83) - : +- CometBroadcastHashJoin (82) - : :- CometProject (80) - : : +- CometBroadcastHashJoin (79) - : : :- CometFilter (77) - : : : +- CometScan parquet spark_catalog.default.web_returns (76) - : : +- ReusedExchange (78) - : +- ReusedExchange (81) - :- * HashAggregate (106) - : +- * ColumnarToRow (105) - : +- CometColumnarExchange (104) - : +- RowToColumnar (103) - : +- * HashAggregate (102) - : +- * HashAggregate (101) - : +- * ColumnarToRow (100) - : +- ReusedExchange (99) - +- * HashAggregate (114) - +- * ColumnarToRow (113) - +- CometColumnarExchange (112) - +- RowToColumnar (111) - +- * HashAggregate (110) - +- * HashAggregate (109) - +- * ColumnarToRow (108) - +- ReusedExchange (107) +TakeOrderedAndProject (99) ++- * HashAggregate (98) + +- Exchange (97) + +- * HashAggregate (96) + +- Union (95) + :- * HashAggregate (84) + : +- Exchange (83) + : +- * HashAggregate (82) + : +- Union (81) + : :- * Project (32) + : : +- * BroadcastHashJoin LeftOuter BuildRight (31) + : : :- * HashAggregate (17) + : : : +- Exchange (16) + : : : +- * HashAggregate (15) + : : : +- * ColumnarToRow (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan parquet spark_catalog.default.store (9) + : : +- BroadcastExchange (30) + : : +- * HashAggregate (29) + : : +- Exchange (28) + : : +- * HashAggregate (27) + : : +- * ColumnarToRow (26) + : : +- CometProject (25) + : : +- CometBroadcastHashJoin (24) + : : :- CometProject (22) + : : : +- CometBroadcastHashJoin (21) + : : : :- CometFilter (19) + : : : : +- CometScan parquet spark_catalog.default.store_returns (18) + : : : +- ReusedExchange (20) + : : +- ReusedExchange (23) + : :- * Project (51) + : : +- * BroadcastNestedLoopJoin Inner BuildLeft (50) + : : :- BroadcastExchange (41) + : : : +- * HashAggregate (40) + : : : +- Exchange (39) + : : : +- * HashAggregate (38) + : : : +- * ColumnarToRow (37) + : : : +- CometProject (36) + : : : +- CometBroadcastHashJoin (35) + : : : :- CometScan parquet spark_catalog.default.catalog_sales (33) + : : : +- ReusedExchange (34) + : : +- * HashAggregate (49) + : : +- Exchange (48) + : : +- * HashAggregate (47) + : : +- * ColumnarToRow (46) + : : +- CometProject (45) + : : +- CometBroadcastHashJoin (44) + : : :- CometScan parquet spark_catalog.default.catalog_returns (42) + : : +- ReusedExchange (43) + : +- * Project (80) + : +- * BroadcastHashJoin LeftOuter BuildRight (79) + : :- * HashAggregate (65) + : : +- Exchange (64) + : : +- * HashAggregate (63) + : : +- * ColumnarToRow (62) + : : +- CometProject (61) + : : +- CometBroadcastHashJoin (60) + : : :- CometProject (56) + : : : +- CometBroadcastHashJoin (55) + : : : :- CometFilter (53) + : : : : +- CometScan parquet spark_catalog.default.web_sales (52) + : : : +- ReusedExchange (54) + : : +- CometBroadcastExchange (59) + : : +- CometFilter (58) + : : +- CometScan parquet spark_catalog.default.web_page (57) + : +- BroadcastExchange (78) + : +- * HashAggregate (77) + : +- Exchange (76) + : +- * HashAggregate (75) + : +- * ColumnarToRow (74) + : +- CometProject (73) + : +- CometBroadcastHashJoin (72) + : :- CometProject (70) + : : +- CometBroadcastHashJoin (69) + : : :- CometFilter (67) + : : : +- CometScan parquet spark_catalog.default.web_returns (66) + : : +- ReusedExchange (68) + : +- ReusedExchange (71) + :- * HashAggregate (89) + : +- Exchange (88) + : +- * HashAggregate (87) + : +- * HashAggregate (86) + : +- ReusedExchange (85) + +- * HashAggregate (94) + +- Exchange (93) + +- * HashAggregate (92) + +- * HashAggregate (91) + +- ReusedExchange (90) (1) Scan parquet spark_catalog.default.store_sales @@ -196,24 +174,18 @@ Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(Un Aggregate Attributes [2]: [sum#9, sum#10] Results [3]: [s_store_sk#8, sum#11, sum#12] -(16) RowToColumnar +(16) Exchange Input [3]: [s_store_sk#8, sum#11, sum#12] +Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [3]: [s_store_sk#8, sum#11, sum#12] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 4] -Input [3]: [s_store_sk#8, sum#11, sum#12] - -(19) HashAggregate [codegen id : 4] +(17) HashAggregate [codegen id : 4] Input [3]: [s_store_sk#8, sum#11, sum#12] Keys [1]: [s_store_sk#8] Functions [2]: [sum(UnscaledValue(ss_ext_sales_price#2)), sum(UnscaledValue(ss_net_profit#3))] Aggregate Attributes [2]: [sum(UnscaledValue(ss_ext_sales_price#2))#13, sum(UnscaledValue(ss_net_profit#3))#14] Results [3]: [s_store_sk#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#13,17,2) AS sales#15, MakeDecimal(sum(UnscaledValue(ss_net_profit#3))#14,17,2) AS profit#16] -(20) Scan parquet spark_catalog.default.store_returns +(18) Scan parquet spark_catalog.default.store_returns Output [4]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19, sr_returned_date_sk#20] Batched: true Location: InMemoryFileIndex [] @@ -221,180 +193,162 @@ PartitionFilters: [isnotnull(sr_returned_date_sk#20), dynamicpruningexpression(s PushedFilters: [IsNotNull(sr_store_sk)] ReadSchema: struct -(21) CometFilter +(19) CometFilter Input [4]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19, sr_returned_date_sk#20] Condition : isnotnull(sr_store_sk#17) -(22) ReusedExchange [Reuses operator id: 6] +(20) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#22] -(23) CometBroadcastHashJoin +(21) CometBroadcastHashJoin Left output [4]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19, sr_returned_date_sk#20] Right output [1]: [d_date_sk#22] Arguments: [sr_returned_date_sk#20], [d_date_sk#22], Inner, BuildRight -(24) CometProject +(22) CometProject Input [5]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19, sr_returned_date_sk#20, d_date_sk#22] Arguments: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19], [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19] -(25) ReusedExchange [Reuses operator id: 11] +(23) ReusedExchange [Reuses operator id: 11] Output [1]: [s_store_sk#23] -(26) CometBroadcastHashJoin +(24) CometBroadcastHashJoin Left output [3]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19] Right output [1]: [s_store_sk#23] Arguments: [sr_store_sk#17], [s_store_sk#23], Inner, BuildRight -(27) CometProject +(25) CometProject Input [4]: [sr_store_sk#17, sr_return_amt#18, sr_net_loss#19, s_store_sk#23] Arguments: [sr_return_amt#18, sr_net_loss#19, s_store_sk#23], [sr_return_amt#18, sr_net_loss#19, s_store_sk#23] -(28) ColumnarToRow [codegen id : 2] +(26) ColumnarToRow [codegen id : 2] Input [3]: [sr_return_amt#18, sr_net_loss#19, s_store_sk#23] -(29) HashAggregate [codegen id : 2] +(27) HashAggregate [codegen id : 2] Input [3]: [sr_return_amt#18, sr_net_loss#19, s_store_sk#23] Keys [1]: [s_store_sk#23] Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#18)), partial_sum(UnscaledValue(sr_net_loss#19))] Aggregate Attributes [2]: [sum#24, sum#25] Results [3]: [s_store_sk#23, sum#26, sum#27] -(30) RowToColumnar -Input [3]: [s_store_sk#23, sum#26, sum#27] - -(31) CometColumnarExchange +(28) Exchange Input [3]: [s_store_sk#23, sum#26, sum#27] -Arguments: hashpartitioning(s_store_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(s_store_sk#23, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(32) ColumnarToRow [codegen id : 3] -Input [3]: [s_store_sk#23, sum#26, sum#27] - -(33) HashAggregate [codegen id : 3] +(29) HashAggregate [codegen id : 3] Input [3]: [s_store_sk#23, sum#26, sum#27] Keys [1]: [s_store_sk#23] Functions [2]: [sum(UnscaledValue(sr_return_amt#18)), sum(UnscaledValue(sr_net_loss#19))] Aggregate Attributes [2]: [sum(UnscaledValue(sr_return_amt#18))#28, sum(UnscaledValue(sr_net_loss#19))#29] Results [3]: [s_store_sk#23, MakeDecimal(sum(UnscaledValue(sr_return_amt#18))#28,17,2) AS returns#30, MakeDecimal(sum(UnscaledValue(sr_net_loss#19))#29,17,2) AS profit_loss#31] -(34) BroadcastExchange +(30) BroadcastExchange Input [3]: [s_store_sk#23, returns#30, profit_loss#31] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] -(35) BroadcastHashJoin [codegen id : 4] +(31) BroadcastHashJoin [codegen id : 4] Left keys [1]: [s_store_sk#8] Right keys [1]: [s_store_sk#23] Join type: LeftOuter Join condition: None -(36) Project [codegen id : 4] +(32) Project [codegen id : 4] Output [5]: [store channel AS channel#32, s_store_sk#8 AS id#33, sales#15, coalesce(returns#30, 0.00) AS returns#34, (profit#16 - coalesce(profit_loss#31, 0.00)) AS profit#35] Input [6]: [s_store_sk#8, sales#15, profit#16, s_store_sk#23, returns#30, profit_loss#31] -(37) Scan parquet spark_catalog.default.catalog_sales +(33) Scan parquet spark_catalog.default.catalog_sales Output [4]: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38, cs_sold_date_sk#39] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cs_sold_date_sk#39), dynamicpruningexpression(cs_sold_date_sk#39 IN dynamicpruning#40)] ReadSchema: struct -(38) ReusedExchange [Reuses operator id: 6] +(34) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#41] -(39) CometBroadcastHashJoin +(35) CometBroadcastHashJoin Left output [4]: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38, cs_sold_date_sk#39] Right output [1]: [d_date_sk#41] Arguments: [cs_sold_date_sk#39], [d_date_sk#41], Inner, BuildRight -(40) CometProject +(36) CometProject Input [5]: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38, cs_sold_date_sk#39, d_date_sk#41] Arguments: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38], [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38] -(41) ColumnarToRow [codegen id : 5] +(37) ColumnarToRow [codegen id : 5] Input [3]: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38] -(42) HashAggregate [codegen id : 5] +(38) HashAggregate [codegen id : 5] Input [3]: [cs_call_center_sk#36, cs_ext_sales_price#37, cs_net_profit#38] Keys [1]: [cs_call_center_sk#36] Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#37)), partial_sum(UnscaledValue(cs_net_profit#38))] Aggregate Attributes [2]: [sum#42, sum#43] Results [3]: [cs_call_center_sk#36, sum#44, sum#45] -(43) RowToColumnar -Input [3]: [cs_call_center_sk#36, sum#44, sum#45] - -(44) CometColumnarExchange -Input [3]: [cs_call_center_sk#36, sum#44, sum#45] -Arguments: hashpartitioning(cs_call_center_sk#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(45) ColumnarToRow [codegen id : 6] +(39) Exchange Input [3]: [cs_call_center_sk#36, sum#44, sum#45] +Arguments: hashpartitioning(cs_call_center_sk#36, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(46) HashAggregate [codegen id : 6] +(40) HashAggregate [codegen id : 6] Input [3]: [cs_call_center_sk#36, sum#44, sum#45] Keys [1]: [cs_call_center_sk#36] Functions [2]: [sum(UnscaledValue(cs_ext_sales_price#37)), sum(UnscaledValue(cs_net_profit#38))] Aggregate Attributes [2]: [sum(UnscaledValue(cs_ext_sales_price#37))#46, sum(UnscaledValue(cs_net_profit#38))#47] Results [3]: [cs_call_center_sk#36, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#37))#46,17,2) AS sales#48, MakeDecimal(sum(UnscaledValue(cs_net_profit#38))#47,17,2) AS profit#49] -(47) BroadcastExchange +(41) BroadcastExchange Input [3]: [cs_call_center_sk#36, sales#48, profit#49] Arguments: IdentityBroadcastMode, [plan_id=5] -(48) Scan parquet spark_catalog.default.catalog_returns +(42) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location: InMemoryFileIndex [] PartitionFilters: [isnotnull(cr_returned_date_sk#52), dynamicpruningexpression(cr_returned_date_sk#52 IN dynamicpruning#53)] ReadSchema: struct -(49) ReusedExchange [Reuses operator id: 6] +(43) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#54] -(50) CometBroadcastHashJoin +(44) CometBroadcastHashJoin Left output [3]: [cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Right output [1]: [d_date_sk#54] Arguments: [cr_returned_date_sk#52], [d_date_sk#54], Inner, BuildRight -(51) CometProject +(45) CometProject Input [4]: [cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52, d_date_sk#54] Arguments: [cr_return_amount#50, cr_net_loss#51], [cr_return_amount#50, cr_net_loss#51] -(52) ColumnarToRow [codegen id : 7] +(46) ColumnarToRow [codegen id : 7] Input [2]: [cr_return_amount#50, cr_net_loss#51] -(53) HashAggregate [codegen id : 7] +(47) HashAggregate [codegen id : 7] Input [2]: [cr_return_amount#50, cr_net_loss#51] Keys: [] Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#50)), partial_sum(UnscaledValue(cr_net_loss#51))] Aggregate Attributes [2]: [sum#55, sum#56] Results [2]: [sum#57, sum#58] -(54) RowToColumnar +(48) Exchange Input [2]: [sum#57, sum#58] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(55) CometColumnarExchange -Input [2]: [sum#57, sum#58] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(56) ColumnarToRow -Input [2]: [sum#57, sum#58] - -(57) HashAggregate +(49) HashAggregate Input [2]: [sum#57, sum#58] Keys: [] Functions [2]: [sum(UnscaledValue(cr_return_amount#50)), sum(UnscaledValue(cr_net_loss#51))] Aggregate Attributes [2]: [sum(UnscaledValue(cr_return_amount#50))#59, sum(UnscaledValue(cr_net_loss#51))#60] Results [2]: [MakeDecimal(sum(UnscaledValue(cr_return_amount#50))#59,17,2) AS returns#61, MakeDecimal(sum(UnscaledValue(cr_net_loss#51))#60,17,2) AS profit_loss#62] -(58) BroadcastNestedLoopJoin [codegen id : 8] +(50) BroadcastNestedLoopJoin [codegen id : 8] Join type: Inner Join condition: None -(59) Project [codegen id : 8] +(51) Project [codegen id : 8] Output [5]: [catalog channel AS channel#63, cs_call_center_sk#36 AS id#64, sales#48, returns#61, (profit#49 - profit_loss#62) AS profit#65] Input [5]: [cs_call_center_sk#36, sales#48, profit#49, returns#61, profit_loss#62] -(60) Scan parquet spark_catalog.default.web_sales +(52) Scan parquet spark_catalog.default.web_sales Output [4]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68, ws_sold_date_sk#69] Batched: true Location: InMemoryFileIndex [] @@ -402,74 +356,68 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#69), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_page_sk)] ReadSchema: struct -(61) CometFilter +(53) CometFilter Input [4]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68, ws_sold_date_sk#69] Condition : isnotnull(ws_web_page_sk#66) -(62) ReusedExchange [Reuses operator id: 6] +(54) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#71] -(63) CometBroadcastHashJoin +(55) CometBroadcastHashJoin Left output [4]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68, ws_sold_date_sk#69] Right output [1]: [d_date_sk#71] Arguments: [ws_sold_date_sk#69], [d_date_sk#71], Inner, BuildRight -(64) CometProject +(56) CometProject Input [5]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68, ws_sold_date_sk#69, d_date_sk#71] Arguments: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68], [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68] -(65) Scan parquet spark_catalog.default.web_page +(57) Scan parquet spark_catalog.default.web_page Output [1]: [wp_web_page_sk#72] Batched: true Location [not included in comparison]/{warehouse_dir}/web_page] PushedFilters: [IsNotNull(wp_web_page_sk)] ReadSchema: struct -(66) CometFilter +(58) CometFilter Input [1]: [wp_web_page_sk#72] Condition : isnotnull(wp_web_page_sk#72) -(67) CometBroadcastExchange +(59) CometBroadcastExchange Input [1]: [wp_web_page_sk#72] Arguments: [wp_web_page_sk#72] -(68) CometBroadcastHashJoin +(60) CometBroadcastHashJoin Left output [3]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68] Right output [1]: [wp_web_page_sk#72] Arguments: [ws_web_page_sk#66], [wp_web_page_sk#72], Inner, BuildRight -(69) CometProject +(61) CometProject Input [4]: [ws_web_page_sk#66, ws_ext_sales_price#67, ws_net_profit#68, wp_web_page_sk#72] Arguments: [ws_ext_sales_price#67, ws_net_profit#68, wp_web_page_sk#72], [ws_ext_sales_price#67, ws_net_profit#68, wp_web_page_sk#72] -(70) ColumnarToRow [codegen id : 9] +(62) ColumnarToRow [codegen id : 9] Input [3]: [ws_ext_sales_price#67, ws_net_profit#68, wp_web_page_sk#72] -(71) HashAggregate [codegen id : 9] +(63) HashAggregate [codegen id : 9] Input [3]: [ws_ext_sales_price#67, ws_net_profit#68, wp_web_page_sk#72] Keys [1]: [wp_web_page_sk#72] Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#67)), partial_sum(UnscaledValue(ws_net_profit#68))] Aggregate Attributes [2]: [sum#73, sum#74] Results [3]: [wp_web_page_sk#72, sum#75, sum#76] -(72) RowToColumnar -Input [3]: [wp_web_page_sk#72, sum#75, sum#76] - -(73) CometColumnarExchange +(64) Exchange Input [3]: [wp_web_page_sk#72, sum#75, sum#76] -Arguments: hashpartitioning(wp_web_page_sk#72, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(wp_web_page_sk#72, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(74) ColumnarToRow [codegen id : 12] -Input [3]: [wp_web_page_sk#72, sum#75, sum#76] - -(75) HashAggregate [codegen id : 12] +(65) HashAggregate [codegen id : 12] Input [3]: [wp_web_page_sk#72, sum#75, sum#76] Keys [1]: [wp_web_page_sk#72] Functions [2]: [sum(UnscaledValue(ws_ext_sales_price#67)), sum(UnscaledValue(ws_net_profit#68))] Aggregate Attributes [2]: [sum(UnscaledValue(ws_ext_sales_price#67))#77, sum(UnscaledValue(ws_net_profit#68))#78] Results [3]: [wp_web_page_sk#72, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#67))#77,17,2) AS sales#79, MakeDecimal(sum(UnscaledValue(ws_net_profit#68))#78,17,2) AS profit#80] -(76) Scan parquet spark_catalog.default.web_returns +(66) Scan parquet spark_catalog.default.web_returns Output [4]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83, wr_returned_date_sk#84] Batched: true Location: InMemoryFileIndex [] @@ -477,243 +425,209 @@ PartitionFilters: [isnotnull(wr_returned_date_sk#84), dynamicpruningexpression(w PushedFilters: [IsNotNull(wr_web_page_sk)] ReadSchema: struct -(77) CometFilter +(67) CometFilter Input [4]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83, wr_returned_date_sk#84] Condition : isnotnull(wr_web_page_sk#81) -(78) ReusedExchange [Reuses operator id: 6] +(68) ReusedExchange [Reuses operator id: 6] Output [1]: [d_date_sk#86] -(79) CometBroadcastHashJoin +(69) CometBroadcastHashJoin Left output [4]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83, wr_returned_date_sk#84] Right output [1]: [d_date_sk#86] Arguments: [wr_returned_date_sk#84], [d_date_sk#86], Inner, BuildRight -(80) CometProject +(70) CometProject Input [5]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83, wr_returned_date_sk#84, d_date_sk#86] Arguments: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83], [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83] -(81) ReusedExchange [Reuses operator id: 67] +(71) ReusedExchange [Reuses operator id: 59] Output [1]: [wp_web_page_sk#87] -(82) CometBroadcastHashJoin +(72) CometBroadcastHashJoin Left output [3]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83] Right output [1]: [wp_web_page_sk#87] Arguments: [wr_web_page_sk#81], [wp_web_page_sk#87], Inner, BuildRight -(83) CometProject +(73) CometProject Input [4]: [wr_web_page_sk#81, wr_return_amt#82, wr_net_loss#83, wp_web_page_sk#87] Arguments: [wr_return_amt#82, wr_net_loss#83, wp_web_page_sk#87], [wr_return_amt#82, wr_net_loss#83, wp_web_page_sk#87] -(84) ColumnarToRow [codegen id : 10] +(74) ColumnarToRow [codegen id : 10] Input [3]: [wr_return_amt#82, wr_net_loss#83, wp_web_page_sk#87] -(85) HashAggregate [codegen id : 10] +(75) HashAggregate [codegen id : 10] Input [3]: [wr_return_amt#82, wr_net_loss#83, wp_web_page_sk#87] Keys [1]: [wp_web_page_sk#87] Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#82)), partial_sum(UnscaledValue(wr_net_loss#83))] Aggregate Attributes [2]: [sum#88, sum#89] Results [3]: [wp_web_page_sk#87, sum#90, sum#91] -(86) RowToColumnar -Input [3]: [wp_web_page_sk#87, sum#90, sum#91] - -(87) CometColumnarExchange -Input [3]: [wp_web_page_sk#87, sum#90, sum#91] -Arguments: hashpartitioning(wp_web_page_sk#87, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(88) ColumnarToRow [codegen id : 11] +(76) Exchange Input [3]: [wp_web_page_sk#87, sum#90, sum#91] +Arguments: hashpartitioning(wp_web_page_sk#87, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(89) HashAggregate [codegen id : 11] +(77) HashAggregate [codegen id : 11] Input [3]: [wp_web_page_sk#87, sum#90, sum#91] Keys [1]: [wp_web_page_sk#87] Functions [2]: [sum(UnscaledValue(wr_return_amt#82)), sum(UnscaledValue(wr_net_loss#83))] Aggregate Attributes [2]: [sum(UnscaledValue(wr_return_amt#82))#92, sum(UnscaledValue(wr_net_loss#83))#93] Results [3]: [wp_web_page_sk#87, MakeDecimal(sum(UnscaledValue(wr_return_amt#82))#92,17,2) AS returns#94, MakeDecimal(sum(UnscaledValue(wr_net_loss#83))#93,17,2) AS profit_loss#95] -(90) BroadcastExchange +(78) BroadcastExchange Input [3]: [wp_web_page_sk#87, returns#94, profit_loss#95] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] -(91) BroadcastHashJoin [codegen id : 12] +(79) BroadcastHashJoin [codegen id : 12] Left keys [1]: [wp_web_page_sk#72] Right keys [1]: [wp_web_page_sk#87] Join type: LeftOuter Join condition: None -(92) Project [codegen id : 12] +(80) Project [codegen id : 12] Output [5]: [web channel AS channel#96, wp_web_page_sk#72 AS id#97, sales#79, coalesce(returns#94, 0.00) AS returns#98, (profit#80 - coalesce(profit_loss#95, 0.00)) AS profit#99] Input [6]: [wp_web_page_sk#72, sales#79, profit#80, wp_web_page_sk#87, returns#94, profit_loss#95] -(93) Union +(81) Union -(94) HashAggregate [codegen id : 13] +(82) HashAggregate [codegen id : 13] Input [5]: [channel#32, id#33, sales#15, returns#34, profit#35] Keys [2]: [channel#32, id#33] Functions [3]: [partial_sum(sales#15), partial_sum(returns#34), partial_sum(profit#35)] Aggregate Attributes [6]: [sum#100, isEmpty#101, sum#102, isEmpty#103, sum#104, isEmpty#105] Results [8]: [channel#32, id#33, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -(95) RowToColumnar -Input [8]: [channel#32, id#33, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] - -(96) CometColumnarExchange +(83) Exchange Input [8]: [channel#32, id#33, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] -Arguments: hashpartitioning(channel#32, id#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(channel#32, id#33, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(97) ColumnarToRow [codegen id : 14] -Input [8]: [channel#32, id#33, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] - -(98) HashAggregate [codegen id : 14] +(84) HashAggregate [codegen id : 14] Input [8]: [channel#32, id#33, sum#106, isEmpty#107, sum#108, isEmpty#109, sum#110, isEmpty#111] Keys [2]: [channel#32, id#33] Functions [3]: [sum(sales#15), sum(returns#34), sum(profit#35)] Aggregate Attributes [3]: [sum(sales#15)#112, sum(returns#34)#113, sum(profit#35)#114] Results [5]: [channel#32, id#33, cast(sum(sales#15)#112 as decimal(37,2)) AS sales#115, cast(sum(returns#34)#113 as decimal(37,2)) AS returns#116, cast(sum(profit#35)#114 as decimal(38,2)) AS profit#117] -(99) ReusedExchange [Reuses operator id: 96] +(85) ReusedExchange [Reuses operator id: 83] Output [8]: [channel#118, id#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] -(100) ColumnarToRow [codegen id : 28] -Input [8]: [channel#118, id#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] - -(101) HashAggregate [codegen id : 28] +(86) HashAggregate [codegen id : 28] Input [8]: [channel#118, id#119, sum#120, isEmpty#121, sum#122, isEmpty#123, sum#124, isEmpty#125] Keys [2]: [channel#118, id#119] Functions [3]: [sum(sales#126), sum(returns#127), sum(profit#128)] Aggregate Attributes [3]: [sum(sales#126)#112, sum(returns#127)#113, sum(profit#128)#114] Results [4]: [channel#118, sum(sales#126)#112 AS sales#129, sum(returns#127)#113 AS returns#130, sum(profit#128)#114 AS profit#131] -(102) HashAggregate [codegen id : 28] +(87) HashAggregate [codegen id : 28] Input [4]: [channel#118, sales#129, returns#130, profit#131] Keys [1]: [channel#118] Functions [3]: [partial_sum(sales#129), partial_sum(returns#130), partial_sum(profit#131)] Aggregate Attributes [6]: [sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] Results [7]: [channel#118, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] -(103) RowToColumnar -Input [7]: [channel#118, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] - -(104) CometColumnarExchange +(88) Exchange Input [7]: [channel#118, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] -Arguments: hashpartitioning(channel#118, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(channel#118, 5), ENSURE_REQUIREMENTS, [plan_id=11] -(105) ColumnarToRow [codegen id : 29] -Input [7]: [channel#118, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] - -(106) HashAggregate [codegen id : 29] +(89) HashAggregate [codegen id : 29] Input [7]: [channel#118, sum#138, isEmpty#139, sum#140, isEmpty#141, sum#142, isEmpty#143] Keys [1]: [channel#118] Functions [3]: [sum(sales#129), sum(returns#130), sum(profit#131)] Aggregate Attributes [3]: [sum(sales#129)#144, sum(returns#130)#145, sum(profit#131)#146] Results [5]: [channel#118, null AS id#147, sum(sales#129)#144 AS sales#148, sum(returns#130)#145 AS returns#149, sum(profit#131)#146 AS profit#150] -(107) ReusedExchange [Reuses operator id: 96] +(90) ReusedExchange [Reuses operator id: 83] Output [8]: [channel#151, id#152, sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] -(108) ColumnarToRow [codegen id : 43] -Input [8]: [channel#151, id#152, sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] - -(109) HashAggregate [codegen id : 43] +(91) HashAggregate [codegen id : 43] Input [8]: [channel#151, id#152, sum#153, isEmpty#154, sum#155, isEmpty#156, sum#157, isEmpty#158] Keys [2]: [channel#151, id#152] Functions [3]: [sum(sales#159), sum(returns#160), sum(profit#161)] Aggregate Attributes [3]: [sum(sales#159)#112, sum(returns#160)#113, sum(profit#161)#114] Results [3]: [sum(sales#159)#112 AS sales#162, sum(returns#160)#113 AS returns#163, sum(profit#161)#114 AS profit#164] -(110) HashAggregate [codegen id : 43] +(92) HashAggregate [codegen id : 43] Input [3]: [sales#162, returns#163, profit#164] Keys: [] Functions [3]: [partial_sum(sales#162), partial_sum(returns#163), partial_sum(profit#164)] Aggregate Attributes [6]: [sum#165, isEmpty#166, sum#167, isEmpty#168, sum#169, isEmpty#170] Results [6]: [sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] -(111) RowToColumnar -Input [6]: [sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] - -(112) CometColumnarExchange +(93) Exchange Input [6]: [sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] -(113) ColumnarToRow [codegen id : 44] -Input [6]: [sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] - -(114) HashAggregate [codegen id : 44] +(94) HashAggregate [codegen id : 44] Input [6]: [sum#171, isEmpty#172, sum#173, isEmpty#174, sum#175, isEmpty#176] Keys: [] Functions [3]: [sum(sales#162), sum(returns#163), sum(profit#164)] Aggregate Attributes [3]: [sum(sales#162)#177, sum(returns#163)#178, sum(profit#164)#179] Results [5]: [null AS channel#180, null AS id#181, sum(sales#162)#177 AS sales#182, sum(returns#163)#178 AS returns#183, sum(profit#164)#179 AS profit#184] -(115) Union +(95) Union -(116) HashAggregate [codegen id : 45] +(96) HashAggregate [codegen id : 45] Input [5]: [channel#32, id#33, sales#115, returns#116, profit#117] Keys [5]: [channel#32, id#33, sales#115, returns#116, profit#117] Functions: [] Aggregate Attributes: [] Results [5]: [channel#32, id#33, sales#115, returns#116, profit#117] -(117) RowToColumnar +(97) Exchange Input [5]: [channel#32, id#33, sales#115, returns#116, profit#117] +Arguments: hashpartitioning(channel#32, id#33, sales#115, returns#116, profit#117, 5), ENSURE_REQUIREMENTS, [plan_id=13] -(118) CometColumnarExchange -Input [5]: [channel#32, id#33, sales#115, returns#116, profit#117] -Arguments: hashpartitioning(channel#32, id#33, sales#115, returns#116, profit#117, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] - -(119) CometHashAggregate +(98) HashAggregate [codegen id : 46] Input [5]: [channel#32, id#33, sales#115, returns#116, profit#117] Keys [5]: [channel#32, id#33, sales#115, returns#116, profit#117] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#32, id#33, sales#115, returns#116, profit#117] -(120) CometTakeOrderedAndProject -Input [5]: [channel#32, id#33, sales#115, returns#116, profit#117] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#32 ASC NULLS FIRST,id#33 ASC NULLS FIRST], output=[channel#32,id#33,sales#115,returns#116,profit#117]), [channel#32, id#33, sales#115, returns#116, profit#117], 100, [channel#32 ASC NULLS FIRST, id#33 ASC NULLS FIRST], [channel#32, id#33, sales#115, returns#116, profit#117] - -(121) ColumnarToRow [codegen id : 46] +(99) TakeOrderedAndProject Input [5]: [channel#32, id#33, sales#115, returns#116, profit#117] +Arguments: 100, [channel#32 ASC NULLS FIRST, id#33 ASC NULLS FIRST], [channel#32, id#33, sales#115, returns#116, profit#117] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (126) -+- * ColumnarToRow (125) - +- CometProject (124) - +- CometFilter (123) - +- CometScan parquet spark_catalog.default.date_dim (122) +BroadcastExchange (104) ++- * ColumnarToRow (103) + +- CometProject (102) + +- CometFilter (101) + +- CometScan parquet spark_catalog.default.date_dim (100) -(122) Scan parquet spark_catalog.default.date_dim +(100) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_date#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(123) CometFilter +(101) CometFilter Input [2]: [d_date_sk#6, d_date#7] Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6)) -(124) CometProject +(102) CometProject Input [2]: [d_date_sk#6, d_date#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(125) ColumnarToRow [codegen id : 1] +(103) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(126) BroadcastExchange +(104) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] -Subquery:2 Hosting operator id = 20 Hosting Expression = sr_returned_date_sk#20 IN dynamicpruning#5 +Subquery:2 Hosting operator id = 18 Hosting Expression = sr_returned_date_sk#20 IN dynamicpruning#5 -Subquery:3 Hosting operator id = 37 Hosting Expression = cs_sold_date_sk#39 IN dynamicpruning#5 +Subquery:3 Hosting operator id = 33 Hosting Expression = cs_sold_date_sk#39 IN dynamicpruning#5 -Subquery:4 Hosting operator id = 48 Hosting Expression = cr_returned_date_sk#52 IN dynamicpruning#5 +Subquery:4 Hosting operator id = 42 Hosting Expression = cr_returned_date_sk#52 IN dynamicpruning#5 -Subquery:5 Hosting operator id = 60 Hosting Expression = ws_sold_date_sk#69 IN dynamicpruning#5 +Subquery:5 Hosting operator id = 52 Hosting Expression = ws_sold_date_sk#69 IN dynamicpruning#5 -Subquery:6 Hosting operator id = 76 Hosting Expression = wr_returned_date_sk#84 IN dynamicpruning#5 +Subquery:6 Hosting operator id = 66 Hosting Expression = wr_returned_date_sk#84 IN dynamicpruning#5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt index df6c4bc88..db4e10247 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q77a/simplified.txt @@ -1,177 +1,155 @@ -WholeStageCodegen (46) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,id,sales,returns,profit] - CometHashAggregate [channel,id,sales,returns,profit] - CometColumnarExchange [channel,id,sales,returns,profit] #1 - RowToColumnar - WholeStageCodegen (45) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (14) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id] #2 - RowToColumnar - WholeStageCodegen (13) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (4) - Project [s_store_sk,sales,returns,profit,profit_loss] - BroadcastHashJoin [s_store_sk,s_store_sk] - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (46) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Exchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (45) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (14) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel,id] #2 + WholeStageCodegen (13) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (4) + Project [s_store_sk,sales,returns,profit,profit_loss] + BroadcastHashJoin [s_store_sk,s_store_sk] + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit)),sales,profit,sum,sum] + InputAdapter + Exchange [s_store_sk] #3 + WholeStageCodegen (1) + HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] ColumnarToRow InputAdapter - CometColumnarExchange [s_store_sk] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_store_sk,ss_ext_sales_price,ss_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter CometProject [d_date_sk] CometFilter [d_date_sk,d_date] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #6 - CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #6 + CometFilter [s_store_sk] + CometScan parquet spark_catalog.default.store [s_store_sk] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (3) + HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - HashAggregate [s_store_sk,sum,sum] [sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss)),returns,profit_loss,sum,sum] + Exchange [s_store_sk] #8 + WholeStageCodegen (2) + HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum] ColumnarToRow InputAdapter - CometColumnarExchange [s_store_sk] #8 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [s_store_sk,sr_return_amt,sr_net_loss] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [sr_return_amt,sr_net_loss,s_store_sk] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] - CometProject [sr_store_sk,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [s_store_sk] #6 - WholeStageCodegen (8) - Project [cs_call_center_sk,sales,returns,profit,profit_loss] - BroadcastNestedLoopJoin + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + ReusedExchange [s_store_sk] #6 + WholeStageCodegen (8) + Project [cs_call_center_sk,sales,returns,profit,profit_loss] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (6) + HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] InputAdapter - BroadcastExchange #9 - WholeStageCodegen (6) - HashAggregate [cs_call_center_sk,sum,sum] [sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit)),sales,profit,sum,sum] + Exchange [cs_call_center_sk] #10 + WholeStageCodegen (5) + HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] ColumnarToRow InputAdapter - CometColumnarExchange [cs_call_center_sk] #10 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + HashAggregate [sum,sum] [sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss)),returns,profit_loss,sum,sum] + InputAdapter + Exchange #11 + WholeStageCodegen (7) + HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] ColumnarToRow InputAdapter - CometColumnarExchange #11 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [cr_return_amount,cr_net_loss] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (12) - Project [wp_web_page_sk,sales,returns,profit,profit_loss] - BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (12) + Project [wp_web_page_sk,sales,returns,profit,profit_loss] + BroadcastHashJoin [wp_web_page_sk,wp_web_page_sk] + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit)),sales,profit,sum,sum] + InputAdapter + Exchange [wp_web_page_sk] #12 + WholeStageCodegen (9) + HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum] ColumnarToRow InputAdapter - CometColumnarExchange [wp_web_page_sk] #12 - RowToColumnar - WholeStageCodegen (9) - HashAggregate [wp_web_page_sk,ws_ext_sales_price,ws_net_profit] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] - CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [wp_web_page_sk] #13 - CometFilter [wp_web_page_sk] - CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [wp_web_page_sk] #13 + CometFilter [wp_web_page_sk] + CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] + InputAdapter + BroadcastExchange #14 + WholeStageCodegen (11) + HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] InputAdapter - BroadcastExchange #14 - WholeStageCodegen (11) - HashAggregate [wp_web_page_sk,sum,sum] [sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss)),returns,profit_loss,sum,sum] + Exchange [wp_web_page_sk] #15 + WholeStageCodegen (10) + HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum] ColumnarToRow InputAdapter - CometColumnarExchange [wp_web_page_sk] #15 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [wp_web_page_sk,wr_return_amt,wr_net_loss] [sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] - CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] - CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [wp_web_page_sk] #13 - WholeStageCodegen (29) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel] #16 - RowToColumnar - WholeStageCodegen (28) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (44) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #17 - RowToColumnar - WholeStageCodegen (43) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + ReusedExchange [wp_web_page_sk] #13 + WholeStageCodegen (29) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel] #16 + WholeStageCodegen (28) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (44) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange #17 + WholeStageCodegen (43) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt index 8b2278f5f..0aaafa331 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/explain.txt @@ -1,79 +1,73 @@ == Physical Plan == -TakeOrderedAndProject (75) -+- * Project (74) - +- * SortMergeJoin Inner (73) - :- * Project (49) - : +- * SortMergeJoin Inner (48) - : :- * Sort (24) - : : +- * HashAggregate (23) - : : +- * ColumnarToRow (22) - : : +- CometColumnarExchange (21) - : : +- RowToColumnar (20) - : : +- * HashAggregate (19) - : : +- * ColumnarToRow (18) - : : +- CometProject (17) - : : +- CometBroadcastHashJoin (16) - : : :- CometProject (12) - : : : +- CometFilter (11) - : : : +- CometSortMergeJoin (10) - : : : :- CometSort (4) - : : : : +- CometColumnarExchange (3) - : : : : +- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : +- CometSort (9) - : : : +- CometColumnarExchange (8) - : : : +- CometProject (7) - : : : +- CometFilter (6) - : : : +- CometScan parquet spark_catalog.default.store_returns (5) - : : +- CometBroadcastExchange (15) - : : +- CometFilter (14) - : : +- CometScan parquet spark_catalog.default.date_dim (13) - : +- * Sort (47) - : +- * Filter (46) - : +- * HashAggregate (45) - : +- * ColumnarToRow (44) - : +- CometColumnarExchange (43) - : +- RowToColumnar (42) - : +- * HashAggregate (41) - : +- * ColumnarToRow (40) - : +- CometProject (39) - : +- CometBroadcastHashJoin (38) - : :- CometProject (36) - : : +- CometFilter (35) - : : +- CometSortMergeJoin (34) - : : :- CometSort (28) - : : : +- CometColumnarExchange (27) - : : : +- CometFilter (26) - : : : +- CometScan parquet spark_catalog.default.web_sales (25) - : : +- CometSort (33) - : : +- CometColumnarExchange (32) - : : +- CometProject (31) - : : +- CometFilter (30) - : : +- CometScan parquet spark_catalog.default.web_returns (29) - : +- ReusedExchange (37) - +- * Sort (72) - +- * Filter (71) - +- * HashAggregate (70) - +- * ColumnarToRow (69) - +- CometColumnarExchange (68) - +- RowToColumnar (67) - +- * HashAggregate (66) - +- * ColumnarToRow (65) - +- CometProject (64) - +- CometBroadcastHashJoin (63) - :- CometProject (61) - : +- CometFilter (60) - : +- CometSortMergeJoin (59) - : :- CometSort (53) - : : +- CometColumnarExchange (52) - : : +- CometFilter (51) - : : +- CometScan parquet spark_catalog.default.catalog_sales (50) - : +- CometSort (58) - : +- CometColumnarExchange (57) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometScan parquet spark_catalog.default.catalog_returns (54) - +- ReusedExchange (62) +TakeOrderedAndProject (69) ++- * Project (68) + +- * SortMergeJoin Inner (67) + :- * Project (45) + : +- * SortMergeJoin Inner (44) + : :- * Sort (22) + : : +- * HashAggregate (21) + : : +- Exchange (20) + : : +- * HashAggregate (19) + : : +- * ColumnarToRow (18) + : : +- CometProject (17) + : : +- CometBroadcastHashJoin (16) + : : :- CometProject (12) + : : : +- CometFilter (11) + : : : +- CometSortMergeJoin (10) + : : : :- CometSort (4) + : : : : +- CometExchange (3) + : : : : +- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : +- CometSort (9) + : : : +- CometExchange (8) + : : : +- CometProject (7) + : : : +- CometFilter (6) + : : : +- CometScan parquet spark_catalog.default.store_returns (5) + : : +- CometBroadcastExchange (15) + : : +- CometFilter (14) + : : +- CometScan parquet spark_catalog.default.date_dim (13) + : +- * Sort (43) + : +- * Filter (42) + : +- * HashAggregate (41) + : +- Exchange (40) + : +- * HashAggregate (39) + : +- * ColumnarToRow (38) + : +- CometProject (37) + : +- CometBroadcastHashJoin (36) + : :- CometProject (34) + : : +- CometFilter (33) + : : +- CometSortMergeJoin (32) + : : :- CometSort (26) + : : : +- CometExchange (25) + : : : +- CometFilter (24) + : : : +- CometScan parquet spark_catalog.default.web_sales (23) + : : +- CometSort (31) + : : +- CometExchange (30) + : : +- CometProject (29) + : : +- CometFilter (28) + : : +- CometScan parquet spark_catalog.default.web_returns (27) + : +- ReusedExchange (35) + +- * Sort (66) + +- * Filter (65) + +- * HashAggregate (64) + +- Exchange (63) + +- * HashAggregate (62) + +- * ColumnarToRow (61) + +- CometProject (60) + +- CometBroadcastHashJoin (59) + :- CometProject (57) + : +- CometFilter (56) + : +- CometSortMergeJoin (55) + : :- CometSort (49) + : : +- CometExchange (48) + : : +- CometFilter (47) + : : +- CometScan parquet spark_catalog.default.catalog_sales (46) + : +- CometSort (54) + : +- CometExchange (53) + : +- CometProject (52) + : +- CometFilter (51) + : +- CometScan parquet spark_catalog.default.catalog_returns (50) + +- ReusedExchange (58) (1) Scan parquet spark_catalog.default.store_sales @@ -88,9 +82,9 @@ ReadSchema: struct -(26) CometFilter +(24) CometFilter Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] Condition : (isnotnull(ws_item_sk#27) AND isnotnull(ws_bill_customer_sk#28)) -(27) CometColumnarExchange +(25) CometExchange Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] -Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(ws_order_number#29, ws_item_sk#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(28) CometSort +(26) CometSort Input [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] Arguments: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33], [ws_order_number#29 ASC NULLS FIRST, ws_item_sk#27 ASC NULLS FIRST] -(29) Scan parquet spark_catalog.default.web_returns +(27) Scan parquet spark_catalog.default.web_returns Output [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_order_number), IsNotNull(wr_item_sk)] ReadSchema: struct -(30) CometFilter +(28) CometFilter Input [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] Condition : (isnotnull(wr_order_number#36) AND isnotnull(wr_item_sk#35)) -(31) CometProject +(29) CometProject Input [3]: [wr_item_sk#35, wr_order_number#36, wr_returned_date_sk#37] Arguments: [wr_item_sk#35, wr_order_number#36], [wr_item_sk#35, wr_order_number#36] -(32) CometColumnarExchange +(30) CometExchange Input [2]: [wr_item_sk#35, wr_order_number#36] -Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(wr_order_number#36, wr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(33) CometSort +(31) CometSort Input [2]: [wr_item_sk#35, wr_order_number#36] Arguments: [wr_item_sk#35, wr_order_number#36], [wr_order_number#36 ASC NULLS FIRST, wr_item_sk#35 ASC NULLS FIRST] -(34) CometSortMergeJoin +(32) CometSortMergeJoin Left output [7]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] Right output [2]: [wr_item_sk#35, wr_order_number#36] Arguments: [ws_order_number#29, ws_item_sk#27], [wr_order_number#36, wr_item_sk#35], LeftOuter -(35) CometFilter +(33) CometFilter Input [9]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33, wr_item_sk#35, wr_order_number#36] Condition : isnull(wr_order_number#36) -(36) CometProject +(34) CometProject Input [9]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_order_number#29, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33, wr_item_sk#35, wr_order_number#36] Arguments: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33], [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] -(37) ReusedExchange [Reuses operator id: 15] +(35) ReusedExchange [Reuses operator id: 15] Output [2]: [d_date_sk#38, d_year#39] -(38) CometBroadcastHashJoin +(36) CometBroadcastHashJoin Left output [6]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33] Right output [2]: [d_date_sk#38, d_year#39] Arguments: [ws_sold_date_sk#33], [d_date_sk#38], Inner, BuildRight -(39) CometProject +(37) CometProject Input [8]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, ws_sold_date_sk#33, d_date_sk#38, d_year#39] Arguments: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, d_year#39], [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, d_year#39] -(40) ColumnarToRow [codegen id : 3] +(38) ColumnarToRow [codegen id : 3] Input [6]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, d_year#39] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [6]: [ws_item_sk#27, ws_bill_customer_sk#28, ws_quantity#30, ws_wholesale_cost#31, ws_sales_price#32, d_year#39] Keys [3]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28] Functions [3]: [partial_sum(ws_quantity#30), partial_sum(UnscaledValue(ws_wholesale_cost#31)), partial_sum(UnscaledValue(ws_sales_price#32))] Aggregate Attributes [3]: [sum#40, sum#41, sum#42] Results [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] -(42) RowToColumnar -Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] - -(43) CometColumnarExchange -Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] -Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] +Arguments: hashpartitioning(d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [6]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28, sum#43, sum#44, sum#45] Keys [3]: [d_year#39, ws_item_sk#27, ws_bill_customer_sk#28] Functions [3]: [sum(ws_quantity#30), sum(UnscaledValue(ws_wholesale_cost#31)), sum(UnscaledValue(ws_sales_price#32))] Aggregate Attributes [3]: [sum(ws_quantity#30)#46, sum(UnscaledValue(ws_wholesale_cost#31))#47, sum(UnscaledValue(ws_sales_price#32))#48] Results [6]: [d_year#39 AS ws_sold_year#49, ws_item_sk#27, ws_bill_customer_sk#28 AS ws_customer_sk#50, sum(ws_quantity#30)#46 AS ws_qty#51, MakeDecimal(sum(UnscaledValue(ws_wholesale_cost#31))#47,17,2) AS ws_wc#52, MakeDecimal(sum(UnscaledValue(ws_sales_price#32))#48,17,2) AS ws_sp#53] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [6]: [ws_sold_year#49, ws_item_sk#27, ws_customer_sk#50, ws_qty#51, ws_wc#52, ws_sp#53] Condition : (coalesce(ws_qty#51, 0) > 0) -(47) Sort [codegen id : 4] +(43) Sort [codegen id : 4] Input [6]: [ws_sold_year#49, ws_item_sk#27, ws_customer_sk#50, ws_qty#51, ws_wc#52, ws_sp#53] Arguments: [ws_sold_year#49 ASC NULLS FIRST, ws_item_sk#27 ASC NULLS FIRST, ws_customer_sk#50 ASC NULLS FIRST], false, 0 -(48) SortMergeJoin [codegen id : 5] +(44) SortMergeJoin [codegen id : 5] Left keys [3]: [ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2] Right keys [3]: [ws_sold_year#49, ws_item_sk#27, ws_customer_sk#50] Join type: Inner Join condition: None -(49) Project [codegen id : 5] +(45) Project [codegen id : 5] Output [9]: [ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2, ss_qty#24, ss_wc#25, ss_sp#26, ws_qty#51, ws_wc#52, ws_sp#53] Input [12]: [ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2, ss_qty#24, ss_wc#25, ss_sp#26, ws_sold_year#49, ws_item_sk#27, ws_customer_sk#50, ws_qty#51, ws_wc#52, ws_sp#53] -(50) Scan parquet spark_catalog.default.catalog_sales +(46) Scan parquet spark_catalog.default.catalog_sales Output [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] Batched: true Location: InMemoryFileIndex [] @@ -308,144 +290,138 @@ PartitionFilters: [isnotnull(cs_sold_date_sk#60), dynamicpruningexpression(cs_so PushedFilters: [IsNotNull(cs_item_sk), IsNotNull(cs_bill_customer_sk)] ReadSchema: struct -(51) CometFilter +(47) CometFilter Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] Condition : (isnotnull(cs_item_sk#55) AND isnotnull(cs_bill_customer_sk#54)) -(52) CometColumnarExchange +(48) CometExchange Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] -Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(cs_order_number#56, cs_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(53) CometSort +(49) CometSort Input [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] Arguments: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60], [cs_order_number#56 ASC NULLS FIRST, cs_item_sk#55 ASC NULLS FIRST] -(54) Scan parquet spark_catalog.default.catalog_returns +(50) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_order_number), IsNotNull(cr_item_sk)] ReadSchema: struct -(55) CometFilter +(51) CometFilter Input [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] Condition : (isnotnull(cr_order_number#63) AND isnotnull(cr_item_sk#62)) -(56) CometProject +(52) CometProject Input [3]: [cr_item_sk#62, cr_order_number#63, cr_returned_date_sk#64] Arguments: [cr_item_sk#62, cr_order_number#63], [cr_item_sk#62, cr_order_number#63] -(57) CometColumnarExchange +(53) CometExchange Input [2]: [cr_item_sk#62, cr_order_number#63] -Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(cr_order_number#63, cr_item_sk#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(58) CometSort +(54) CometSort Input [2]: [cr_item_sk#62, cr_order_number#63] Arguments: [cr_item_sk#62, cr_order_number#63], [cr_order_number#63 ASC NULLS FIRST, cr_item_sk#62 ASC NULLS FIRST] -(59) CometSortMergeJoin +(55) CometSortMergeJoin Left output [7]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] Right output [2]: [cr_item_sk#62, cr_order_number#63] Arguments: [cs_order_number#56, cs_item_sk#55], [cr_order_number#63, cr_item_sk#62], LeftOuter -(60) CometFilter +(56) CometFilter Input [9]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60, cr_item_sk#62, cr_order_number#63] Condition : isnull(cr_order_number#63) -(61) CometProject +(57) CometProject Input [9]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_order_number#56, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60, cr_item_sk#62, cr_order_number#63] Arguments: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60], [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] -(62) ReusedExchange [Reuses operator id: 15] +(58) ReusedExchange [Reuses operator id: 15] Output [2]: [d_date_sk#65, d_year#66] -(63) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [6]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60] Right output [2]: [d_date_sk#65, d_year#66] Arguments: [cs_sold_date_sk#60], [d_date_sk#65], Inner, BuildRight -(64) CometProject +(60) CometProject Input [8]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, cs_sold_date_sk#60, d_date_sk#65, d_year#66] Arguments: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, d_year#66], [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, d_year#66] -(65) ColumnarToRow [codegen id : 6] +(61) ColumnarToRow [codegen id : 6] Input [6]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, d_year#66] -(66) HashAggregate [codegen id : 6] +(62) HashAggregate [codegen id : 6] Input [6]: [cs_bill_customer_sk#54, cs_item_sk#55, cs_quantity#57, cs_wholesale_cost#58, cs_sales_price#59, d_year#66] Keys [3]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54] Functions [3]: [partial_sum(cs_quantity#57), partial_sum(UnscaledValue(cs_wholesale_cost#58)), partial_sum(UnscaledValue(cs_sales_price#59))] Aggregate Attributes [3]: [sum#67, sum#68, sum#69] Results [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] -(67) RowToColumnar -Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] - -(68) CometColumnarExchange -Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] -Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(69) ColumnarToRow [codegen id : 7] +(63) Exchange Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] +Arguments: hashpartitioning(d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(70) HashAggregate [codegen id : 7] +(64) HashAggregate [codegen id : 7] Input [6]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54, sum#70, sum#71, sum#72] Keys [3]: [d_year#66, cs_item_sk#55, cs_bill_customer_sk#54] Functions [3]: [sum(cs_quantity#57), sum(UnscaledValue(cs_wholesale_cost#58)), sum(UnscaledValue(cs_sales_price#59))] Aggregate Attributes [3]: [sum(cs_quantity#57)#73, sum(UnscaledValue(cs_wholesale_cost#58))#74, sum(UnscaledValue(cs_sales_price#59))#75] Results [6]: [d_year#66 AS cs_sold_year#76, cs_item_sk#55, cs_bill_customer_sk#54 AS cs_customer_sk#77, sum(cs_quantity#57)#73 AS cs_qty#78, MakeDecimal(sum(UnscaledValue(cs_wholesale_cost#58))#74,17,2) AS cs_wc#79, MakeDecimal(sum(UnscaledValue(cs_sales_price#59))#75,17,2) AS cs_sp#80] -(71) Filter [codegen id : 7] +(65) Filter [codegen id : 7] Input [6]: [cs_sold_year#76, cs_item_sk#55, cs_customer_sk#77, cs_qty#78, cs_wc#79, cs_sp#80] Condition : (coalesce(cs_qty#78, 0) > 0) -(72) Sort [codegen id : 7] +(66) Sort [codegen id : 7] Input [6]: [cs_sold_year#76, cs_item_sk#55, cs_customer_sk#77, cs_qty#78, cs_wc#79, cs_sp#80] Arguments: [cs_sold_year#76 ASC NULLS FIRST, cs_item_sk#55 ASC NULLS FIRST, cs_customer_sk#77 ASC NULLS FIRST], false, 0 -(73) SortMergeJoin [codegen id : 8] +(67) SortMergeJoin [codegen id : 8] Left keys [3]: [ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2] Right keys [3]: [cs_sold_year#76, cs_item_sk#55, cs_customer_sk#77] Join type: Inner Join condition: None -(74) Project [codegen id : 8] +(68) Project [codegen id : 8] Output [13]: [round((cast(ss_qty#24 as double) / cast(coalesce((ws_qty#51 + cs_qty#78), 1) as double)), 2) AS ratio#81, ss_qty#24 AS store_qty#82, ss_wc#25 AS store_wholesale_cost#83, ss_sp#26 AS store_sales_price#84, (coalesce(ws_qty#51, 0) + coalesce(cs_qty#78, 0)) AS other_chan_qty#85, (coalesce(ws_wc#52, 0.00) + coalesce(cs_wc#79, 0.00)) AS other_chan_wholesale_cost#86, (coalesce(ws_sp#53, 0.00) + coalesce(cs_sp#80, 0.00)) AS other_chan_sales_price#87, ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2, ss_qty#24, ss_wc#25, ss_sp#26] Input [15]: [ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2, ss_qty#24, ss_wc#25, ss_sp#26, ws_qty#51, ws_wc#52, ws_sp#53, cs_sold_year#76, cs_item_sk#55, cs_customer_sk#77, cs_qty#78, cs_wc#79, cs_sp#80] -(75) TakeOrderedAndProject +(69) TakeOrderedAndProject Input [13]: [ratio#81, store_qty#82, store_wholesale_cost#83, store_sales_price#84, other_chan_qty#85, other_chan_wholesale_cost#86, other_chan_sales_price#87, ss_sold_year#23, ss_item_sk#1, ss_customer_sk#2, ss_qty#24, ss_wc#25, ss_sp#26] Arguments: 100, [ss_sold_year#23 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST, ss_customer_sk#2 ASC NULLS FIRST, ss_qty#24 DESC NULLS LAST, ss_wc#25 DESC NULLS LAST, ss_sp#26 DESC NULLS LAST, other_chan_qty#85 ASC NULLS FIRST, other_chan_wholesale_cost#86 ASC NULLS FIRST, other_chan_sales_price#87 ASC NULLS FIRST, ratio#81 ASC NULLS FIRST], [ratio#81, store_qty#82, store_wholesale_cost#83, store_sales_price#84, other_chan_qty#85, other_chan_wholesale_cost#86, other_chan_sales_price#87] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (79) -+- * ColumnarToRow (78) - +- CometFilter (77) - +- CometScan parquet spark_catalog.default.date_dim (76) +BroadcastExchange (73) ++- * ColumnarToRow (72) + +- CometFilter (71) + +- CometScan parquet spark_catalog.default.date_dim (70) -(76) Scan parquet spark_catalog.default.date_dim +(70) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#12, d_year#13] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2000), IsNotNull(d_date_sk)] ReadSchema: struct -(77) CometFilter +(71) CometFilter Input [2]: [d_date_sk#12, d_year#13] Condition : ((isnotnull(d_year#13) AND (d_year#13 = 2000)) AND isnotnull(d_date_sk#12)) -(78) ColumnarToRow [codegen id : 1] +(72) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#12, d_year#13] -(79) BroadcastExchange +(73) BroadcastExchange Input [2]: [d_date_sk#12, d_year#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=10] -Subquery:2 Hosting operator id = 25 Hosting Expression = ws_sold_date_sk#33 IN dynamicpruning#8 +Subquery:2 Hosting operator id = 23 Hosting Expression = ws_sold_date_sk#33 IN dynamicpruning#8 -Subquery:3 Hosting operator id = 50 Hosting Expression = cs_sold_date_sk#60 IN dynamicpruning#8 +Subquery:3 Hosting operator id = 46 Hosting Expression = cs_sold_date_sk#60 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt index e903d5b7f..9e94433e6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q78/simplified.txt @@ -10,93 +10,87 @@ TakeOrderedAndProject [ss_sold_year,ss_item_sk,ss_customer_sk,ss_qty,ss_wc,ss_sp WholeStageCodegen (2) Sort [ss_sold_year,ss_item_sk,ss_customer_sk] HashAggregate [d_year,ss_item_sk,ss_customer_sk,sum,sum,sum] [sum(ss_quantity),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_sales_price)),ss_sold_year,ss_qty,ss_wc,ss_sp,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,ss_item_sk,ss_customer_sk] #1 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] - CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #2 - CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #4 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk,d_year] #5 - CometFilter [d_date_sk,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + InputAdapter + Exchange [d_year,ss_item_sk,ss_customer_sk] #1 + WholeStageCodegen (1) + HashAggregate [d_year,ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price] [sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,d_year] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,d_date_sk,d_year] + CometProject [ss_item_sk,ss_customer_sk,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometExchange [ss_ticket_number,ss_item_sk] #2 + CometFilter [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_ticket_number,ss_quantity,ss_wholesale_cost,ss_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #4 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk,d_year] #5 + CometFilter [d_date_sk,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] InputAdapter WholeStageCodegen (4) Sort [ws_sold_year,ws_item_sk,ws_customer_sk] Filter [ws_qty] HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,sum,sum,sum] [sum(ws_quantity),sum(UnscaledValue(ws_wholesale_cost)),sum(UnscaledValue(ws_sales_price)),ws_sold_year,ws_customer_sk,ws_qty,ws_wc,ws_sp,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,ws_item_sk,ws_bill_customer_sk] #6 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] - CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] - CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] - CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] - CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometColumnarExchange [ws_order_number,ws_item_sk] #7 - CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [wr_item_sk,wr_order_number] - CometColumnarExchange [wr_order_number,wr_item_sk] #8 - CometProject [wr_item_sk,wr_order_number] - CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + InputAdapter + Exchange [d_year,ws_item_sk,ws_bill_customer_sk] #6 + WholeStageCodegen (3) + HashAggregate [d_year,ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price] [sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,d_year] + CometBroadcastHashJoin [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,d_date_sk,d_year] + CometProject [ws_item_sk,ws_bill_customer_sk,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSortMergeJoin [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk,wr_item_sk,wr_order_number] + CometSort [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometExchange [ws_order_number,ws_item_sk] #7 + CometFilter [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_bill_customer_sk,ws_order_number,ws_quantity,ws_wholesale_cost,ws_sales_price,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [wr_item_sk,wr_order_number] + CometExchange [wr_order_number,wr_item_sk] #8 + CometProject [wr_item_sk,wr_order_number] + CometFilter [wr_item_sk,wr_order_number,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #5 InputAdapter WholeStageCodegen (7) Sort [cs_sold_year,cs_item_sk,cs_customer_sk] Filter [cs_qty] HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,sum,sum,sum] [sum(cs_quantity),sum(UnscaledValue(cs_wholesale_cost)),sum(UnscaledValue(cs_sales_price)),cs_sold_year,cs_customer_sk,cs_qty,cs_wc,cs_sp,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [d_year,cs_item_sk,cs_bill_customer_sk] #9 - RowToColumnar - WholeStageCodegen (6) - HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] - CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] - CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] - CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometColumnarExchange [cs_order_number,cs_item_sk] #10 - CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [cr_item_sk,cr_order_number] - CometColumnarExchange [cr_order_number,cr_item_sk] #11 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] - ReusedExchange [d_date_sk,d_year] #5 + InputAdapter + Exchange [d_year,cs_item_sk,cs_bill_customer_sk] #9 + WholeStageCodegen (6) + HashAggregate [d_year,cs_item_sk,cs_bill_customer_sk,cs_quantity,cs_wholesale_cost,cs_sales_price] [sum,sum,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,d_year] + CometBroadcastHashJoin [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,d_date_sk,d_year] + CometProject [cs_bill_customer_sk,cs_item_sk,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSortMergeJoin [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk,cr_item_sk,cr_order_number] + CometSort [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometExchange [cs_order_number,cs_item_sk] #10 + CometFilter [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_bill_customer_sk,cs_item_sk,cs_order_number,cs_quantity,cs_wholesale_cost,cs_sales_price,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_order_number,cr_item_sk] #11 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + ReusedExchange [d_date_sk,d_year] #5 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt index f52b07ad6..547802337 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/explain.txt @@ -1,135 +1,119 @@ == Physical Plan == -* ColumnarToRow (131) -+- CometTakeOrderedAndProject (130) - +- CometHashAggregate (129) - +- CometColumnarExchange (128) - +- RowToColumnar (127) - +- * HashAggregate (126) - +- Union (125) - :- * HashAggregate (108) - : +- * ColumnarToRow (107) - : +- CometColumnarExchange (106) - : +- RowToColumnar (105) - : +- * HashAggregate (104) - : +- Union (103) - : :- * HashAggregate (40) - : : +- * ColumnarToRow (39) - : : +- CometColumnarExchange (38) - : : +- RowToColumnar (37) - : : +- * HashAggregate (36) - : : +- * ColumnarToRow (35) - : : +- CometProject (34) - : : +- CometBroadcastHashJoin (33) - : : :- CometProject (28) - : : : +- CometBroadcastHashJoin (27) - : : : :- CometProject (22) - : : : : +- CometBroadcastHashJoin (21) - : : : : :- CometProject (17) - : : : : : +- CometBroadcastHashJoin (16) - : : : : : :- CometProject (11) - : : : : : : +- CometSortMergeJoin (10) - : : : : : : :- CometSort (4) - : : : : : : : +- CometColumnarExchange (3) - : : : : : : : +- CometFilter (2) - : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : : : +- CometSort (9) - : : : : : : +- CometColumnarExchange (8) - : : : : : : +- CometProject (7) - : : : : : : +- CometFilter (6) - : : : : : : +- CometScan parquet spark_catalog.default.store_returns (5) - : : : : : +- CometBroadcastExchange (15) - : : : : : +- CometProject (14) - : : : : : +- CometFilter (13) - : : : : : +- CometScan parquet spark_catalog.default.date_dim (12) - : : : : +- CometBroadcastExchange (20) - : : : : +- CometFilter (19) - : : : : +- CometScan parquet spark_catalog.default.store (18) - : : : +- CometBroadcastExchange (26) - : : : +- CometProject (25) - : : : +- CometFilter (24) - : : : +- CometScan parquet spark_catalog.default.item (23) - : : +- CometBroadcastExchange (32) - : : +- CometProject (31) - : : +- CometFilter (30) - : : +- CometScan parquet spark_catalog.default.promotion (29) - : :- * HashAggregate (71) - : : +- * ColumnarToRow (70) - : : +- CometColumnarExchange (69) - : : +- RowToColumnar (68) - : : +- * HashAggregate (67) - : : +- * ColumnarToRow (66) - : : +- CometProject (65) - : : +- CometBroadcastHashJoin (64) - : : :- CometProject (62) - : : : +- CometBroadcastHashJoin (61) - : : : :- CometProject (59) - : : : : +- CometBroadcastHashJoin (58) - : : : : :- CometProject (54) - : : : : : +- CometBroadcastHashJoin (53) - : : : : : :- CometProject (51) - : : : : : : +- CometSortMergeJoin (50) - : : : : : : :- CometSort (44) - : : : : : : : +- CometColumnarExchange (43) - : : : : : : : +- CometFilter (42) - : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (41) - : : : : : : +- CometSort (49) - : : : : : : +- CometColumnarExchange (48) - : : : : : : +- CometProject (47) - : : : : : : +- CometFilter (46) - : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (45) - : : : : : +- ReusedExchange (52) - : : : : +- CometBroadcastExchange (57) - : : : : +- CometFilter (56) - : : : : +- CometScan parquet spark_catalog.default.catalog_page (55) - : : : +- ReusedExchange (60) - : : +- ReusedExchange (63) - : +- * HashAggregate (102) - : +- * ColumnarToRow (101) - : +- CometColumnarExchange (100) - : +- RowToColumnar (99) - : +- * HashAggregate (98) - : +- * ColumnarToRow (97) - : +- CometProject (96) - : +- CometBroadcastHashJoin (95) - : :- CometProject (93) - : : +- CometBroadcastHashJoin (92) - : : :- CometProject (90) - : : : +- CometBroadcastHashJoin (89) - : : : :- CometProject (85) - : : : : +- CometBroadcastHashJoin (84) - : : : : :- CometProject (82) - : : : : : +- CometSortMergeJoin (81) - : : : : : :- CometSort (75) - : : : : : : +- CometColumnarExchange (74) - : : : : : : +- CometFilter (73) - : : : : : : +- CometScan parquet spark_catalog.default.web_sales (72) - : : : : : +- CometSort (80) - : : : : : +- CometColumnarExchange (79) - : : : : : +- CometProject (78) - : : : : : +- CometFilter (77) - : : : : : +- CometScan parquet spark_catalog.default.web_returns (76) - : : : : +- ReusedExchange (83) - : : : +- CometBroadcastExchange (88) - : : : +- CometFilter (87) - : : : +- CometScan parquet spark_catalog.default.web_site (86) - : : +- ReusedExchange (91) - : +- ReusedExchange (94) - :- * HashAggregate (116) - : +- * ColumnarToRow (115) - : +- CometColumnarExchange (114) - : +- RowToColumnar (113) - : +- * HashAggregate (112) - : +- * HashAggregate (111) - : +- * ColumnarToRow (110) - : +- ReusedExchange (109) - +- * HashAggregate (124) - +- * ColumnarToRow (123) - +- CometColumnarExchange (122) - +- RowToColumnar (121) - +- * HashAggregate (120) - +- * HashAggregate (119) - +- * ColumnarToRow (118) - +- ReusedExchange (117) +TakeOrderedAndProject (115) ++- * HashAggregate (114) + +- Exchange (113) + +- * HashAggregate (112) + +- Union (111) + :- * HashAggregate (100) + : +- Exchange (99) + : +- * HashAggregate (98) + : +- Union (97) + : :- * HashAggregate (38) + : : +- Exchange (37) + : : +- * HashAggregate (36) + : : +- * ColumnarToRow (35) + : : +- CometProject (34) + : : +- CometBroadcastHashJoin (33) + : : :- CometProject (28) + : : : +- CometBroadcastHashJoin (27) + : : : :- CometProject (22) + : : : : +- CometBroadcastHashJoin (21) + : : : : :- CometProject (17) + : : : : : +- CometBroadcastHashJoin (16) + : : : : : :- CometProject (11) + : : : : : : +- CometSortMergeJoin (10) + : : : : : : :- CometSort (4) + : : : : : : : +- CometExchange (3) + : : : : : : : +- CometFilter (2) + : : : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : : : +- CometSort (9) + : : : : : : +- CometExchange (8) + : : : : : : +- CometProject (7) + : : : : : : +- CometFilter (6) + : : : : : : +- CometScan parquet spark_catalog.default.store_returns (5) + : : : : : +- CometBroadcastExchange (15) + : : : : : +- CometProject (14) + : : : : : +- CometFilter (13) + : : : : : +- CometScan parquet spark_catalog.default.date_dim (12) + : : : : +- CometBroadcastExchange (20) + : : : : +- CometFilter (19) + : : : : +- CometScan parquet spark_catalog.default.store (18) + : : : +- CometBroadcastExchange (26) + : : : +- CometProject (25) + : : : +- CometFilter (24) + : : : +- CometScan parquet spark_catalog.default.item (23) + : : +- CometBroadcastExchange (32) + : : +- CometProject (31) + : : +- CometFilter (30) + : : +- CometScan parquet spark_catalog.default.promotion (29) + : :- * HashAggregate (67) + : : +- Exchange (66) + : : +- * HashAggregate (65) + : : +- * ColumnarToRow (64) + : : +- CometProject (63) + : : +- CometBroadcastHashJoin (62) + : : :- CometProject (60) + : : : +- CometBroadcastHashJoin (59) + : : : :- CometProject (57) + : : : : +- CometBroadcastHashJoin (56) + : : : : :- CometProject (52) + : : : : : +- CometBroadcastHashJoin (51) + : : : : : :- CometProject (49) + : : : : : : +- CometSortMergeJoin (48) + : : : : : : :- CometSort (42) + : : : : : : : +- CometExchange (41) + : : : : : : : +- CometFilter (40) + : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (39) + : : : : : : +- CometSort (47) + : : : : : : +- CometExchange (46) + : : : : : : +- CometProject (45) + : : : : : : +- CometFilter (44) + : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (43) + : : : : : +- ReusedExchange (50) + : : : : +- CometBroadcastExchange (55) + : : : : +- CometFilter (54) + : : : : +- CometScan parquet spark_catalog.default.catalog_page (53) + : : : +- ReusedExchange (58) + : : +- ReusedExchange (61) + : +- * HashAggregate (96) + : +- Exchange (95) + : +- * HashAggregate (94) + : +- * ColumnarToRow (93) + : +- CometProject (92) + : +- CometBroadcastHashJoin (91) + : :- CometProject (89) + : : +- CometBroadcastHashJoin (88) + : : :- CometProject (86) + : : : +- CometBroadcastHashJoin (85) + : : : :- CometProject (81) + : : : : +- CometBroadcastHashJoin (80) + : : : : :- CometProject (78) + : : : : : +- CometSortMergeJoin (77) + : : : : : :- CometSort (71) + : : : : : : +- CometExchange (70) + : : : : : : +- CometFilter (69) + : : : : : : +- CometScan parquet spark_catalog.default.web_sales (68) + : : : : : +- CometSort (76) + : : : : : +- CometExchange (75) + : : : : : +- CometProject (74) + : : : : : +- CometFilter (73) + : : : : : +- CometScan parquet spark_catalog.default.web_returns (72) + : : : : +- ReusedExchange (79) + : : : +- CometBroadcastExchange (84) + : : : +- CometFilter (83) + : : : +- CometScan parquet spark_catalog.default.web_site (82) + : : +- ReusedExchange (87) + : +- ReusedExchange (90) + :- * HashAggregate (105) + : +- Exchange (104) + : +- * HashAggregate (103) + : +- * HashAggregate (102) + : +- ReusedExchange (101) + +- * HashAggregate (110) + +- Exchange (109) + +- * HashAggregate (108) + +- * HashAggregate (107) + +- ReusedExchange (106) (1) Scan parquet spark_catalog.default.store_sales @@ -144,9 +128,9 @@ ReadSchema: struct -(42) CometFilter +(40) CometFilter Input [7]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Condition : ((isnotnull(cs_catalog_page_sk#40) AND isnotnull(cs_item_sk#41)) AND isnotnull(cs_promo_sk#42)) -(43) CometColumnarExchange +(41) CometExchange Input [7]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] -Arguments: hashpartitioning(cs_item_sk#41, cs_order_number#43, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_item_sk#41, cs_order_number#43, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(44) CometSort +(42) CometSort Input [7]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Arguments: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46], [cs_item_sk#41 ASC NULLS FIRST, cs_order_number#43 ASC NULLS FIRST] -(45) Scan parquet spark_catalog.default.catalog_returns +(43) Scan parquet spark_catalog.default.catalog_returns Output [5]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(46) CometFilter +(44) CometFilter Input [5]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Condition : (isnotnull(cr_item_sk#48) AND isnotnull(cr_order_number#49)) -(47) CometProject +(45) CometProject Input [5]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51, cr_returned_date_sk#52] Arguments: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51], [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51] -(48) CometColumnarExchange +(46) CometExchange Input [4]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51] -Arguments: hashpartitioning(cr_item_sk#48, cr_order_number#49, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(cr_item_sk#48, cr_order_number#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] -(49) CometSort +(47) CometSort Input [4]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51] Arguments: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51], [cr_item_sk#48 ASC NULLS FIRST, cr_order_number#49 ASC NULLS FIRST] -(50) CometSortMergeJoin +(48) CometSortMergeJoin Left output [7]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46] Right output [4]: [cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51] Arguments: [cs_item_sk#41, cs_order_number#43], [cr_item_sk#48, cr_order_number#49], LeftOuter -(51) CometProject +(49) CometProject Input [11]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_order_number#43, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46, cr_item_sk#48, cr_order_number#49, cr_return_amount#50, cr_net_loss#51] Arguments: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46, cr_return_amount#50, cr_net_loss#51], [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46, cr_return_amount#50, cr_net_loss#51] -(52) ReusedExchange [Reuses operator id: 15] +(50) ReusedExchange [Reuses operator id: 15] Output [1]: [d_date_sk#53] -(53) CometBroadcastHashJoin +(51) CometBroadcastHashJoin Left output [8]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46, cr_return_amount#50, cr_net_loss#51] Right output [1]: [d_date_sk#53] Arguments: [cs_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight -(54) CometProject +(52) CometProject Input [9]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cs_sold_date_sk#46, cr_return_amount#50, cr_net_loss#51, d_date_sk#53] Arguments: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51], [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51] -(55) Scan parquet spark_catalog.default.catalog_page +(53) Scan parquet spark_catalog.default.catalog_page Output [2]: [cp_catalog_page_sk#54, cp_catalog_page_id#55] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_page] PushedFilters: [IsNotNull(cp_catalog_page_sk)] ReadSchema: struct -(56) CometFilter +(54) CometFilter Input [2]: [cp_catalog_page_sk#54, cp_catalog_page_id#55] Condition : isnotnull(cp_catalog_page_sk#54) -(57) CometBroadcastExchange +(55) CometBroadcastExchange Input [2]: [cp_catalog_page_sk#54, cp_catalog_page_id#55] Arguments: [cp_catalog_page_sk#54, cp_catalog_page_id#55] -(58) CometBroadcastHashJoin +(56) CometBroadcastHashJoin Left output [7]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51] Right output [2]: [cp_catalog_page_sk#54, cp_catalog_page_id#55] Arguments: [cs_catalog_page_sk#40], [cp_catalog_page_sk#54], Inner, BuildRight -(59) CometProject +(57) CometProject Input [9]: [cs_catalog_page_sk#40, cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_sk#54, cp_catalog_page_id#55] Arguments: [cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55], [cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] -(60) ReusedExchange [Reuses operator id: 26] +(58) ReusedExchange [Reuses operator id: 26] Output [1]: [i_item_sk#56] -(61) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [7]: [cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] Right output [1]: [i_item_sk#56] Arguments: [cs_item_sk#41], [i_item_sk#56], Inner, BuildRight -(62) CometProject +(60) CometProject Input [8]: [cs_item_sk#41, cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55, i_item_sk#56] Arguments: [cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55], [cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] -(63) ReusedExchange [Reuses operator id: 32] +(61) ReusedExchange [Reuses operator id: 32] Output [1]: [p_promo_sk#57] -(64) CometBroadcastHashJoin +(62) CometBroadcastHashJoin Left output [6]: [cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] Right output [1]: [p_promo_sk#57] Arguments: [cs_promo_sk#42], [p_promo_sk#57], Inner, BuildRight -(65) CometProject +(63) CometProject Input [7]: [cs_promo_sk#42, cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55, p_promo_sk#57] Arguments: [cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55], [cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] -(66) ColumnarToRow [codegen id : 3] +(64) ColumnarToRow [codegen id : 3] Input [5]: [cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] -(67) HashAggregate [codegen id : 3] +(65) HashAggregate [codegen id : 3] Input [5]: [cs_ext_sales_price#44, cs_net_profit#45, cr_return_amount#50, cr_net_loss#51, cp_catalog_page_id#55] Keys [1]: [cp_catalog_page_id#55] Functions [3]: [partial_sum(UnscaledValue(cs_ext_sales_price#44)), partial_sum(coalesce(cast(cr_return_amount#50 as decimal(12,2)), 0.00)), partial_sum((cs_net_profit#45 - coalesce(cast(cr_net_loss#51 as decimal(12,2)), 0.00)))] Aggregate Attributes [5]: [sum#58, sum#59, isEmpty#60, sum#61, isEmpty#62] Results [6]: [cp_catalog_page_id#55, sum#63, sum#64, isEmpty#65, sum#66, isEmpty#67] -(68) RowToColumnar -Input [6]: [cp_catalog_page_id#55, sum#63, sum#64, isEmpty#65, sum#66, isEmpty#67] - -(69) CometColumnarExchange -Input [6]: [cp_catalog_page_id#55, sum#63, sum#64, isEmpty#65, sum#66, isEmpty#67] -Arguments: hashpartitioning(cp_catalog_page_id#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(70) ColumnarToRow [codegen id : 4] +(66) Exchange Input [6]: [cp_catalog_page_id#55, sum#63, sum#64, isEmpty#65, sum#66, isEmpty#67] +Arguments: hashpartitioning(cp_catalog_page_id#55, 5), ENSURE_REQUIREMENTS, [plan_id=6] -(71) HashAggregate [codegen id : 4] +(67) HashAggregate [codegen id : 4] Input [6]: [cp_catalog_page_id#55, sum#63, sum#64, isEmpty#65, sum#66, isEmpty#67] Keys [1]: [cp_catalog_page_id#55] Functions [3]: [sum(UnscaledValue(cs_ext_sales_price#44)), sum(coalesce(cast(cr_return_amount#50 as decimal(12,2)), 0.00)), sum((cs_net_profit#45 - coalesce(cast(cr_net_loss#51 as decimal(12,2)), 0.00)))] Aggregate Attributes [3]: [sum(UnscaledValue(cs_ext_sales_price#44))#68, sum(coalesce(cast(cr_return_amount#50 as decimal(12,2)), 0.00))#69, sum((cs_net_profit#45 - coalesce(cast(cr_net_loss#51 as decimal(12,2)), 0.00)))#70] Results [5]: [catalog channel AS channel#71, concat(catalog_page, cp_catalog_page_id#55) AS id#72, MakeDecimal(sum(UnscaledValue(cs_ext_sales_price#44))#68,17,2) AS sales#73, sum(coalesce(cast(cr_return_amount#50 as decimal(12,2)), 0.00))#69 AS returns#74, sum((cs_net_profit#45 - coalesce(cast(cr_net_loss#51 as decimal(12,2)), 0.00)))#70 AS profit#75] -(72) Scan parquet spark_catalog.default.web_sales +(68) Scan parquet spark_catalog.default.web_sales Output [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82] Batched: true Location: InMemoryFileIndex [] @@ -466,299 +438,265 @@ PartitionFilters: [isnotnull(ws_sold_date_sk#82), dynamicpruningexpression(ws_so PushedFilters: [IsNotNull(ws_web_site_sk), IsNotNull(ws_item_sk), IsNotNull(ws_promo_sk)] ReadSchema: struct -(73) CometFilter +(69) CometFilter Input [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82] Condition : ((isnotnull(ws_web_site_sk#77) AND isnotnull(ws_item_sk#76)) AND isnotnull(ws_promo_sk#78)) -(74) CometColumnarExchange +(70) CometExchange Input [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82] -Arguments: hashpartitioning(ws_item_sk#76, ws_order_number#79, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(ws_item_sk#76, ws_order_number#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(75) CometSort +(71) CometSort Input [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82] Arguments: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82], [ws_item_sk#76 ASC NULLS FIRST, ws_order_number#79 ASC NULLS FIRST] -(76) Scan parquet spark_catalog.default.web_returns +(72) Scan parquet spark_catalog.default.web_returns Output [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] Batched: true Location [not included in comparison]/{warehouse_dir}/web_returns] PushedFilters: [IsNotNull(wr_item_sk), IsNotNull(wr_order_number)] ReadSchema: struct -(77) CometFilter +(73) CometFilter Input [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] Condition : (isnotnull(wr_item_sk#84) AND isnotnull(wr_order_number#85)) -(78) CometProject +(74) CometProject Input [5]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87, wr_returned_date_sk#88] Arguments: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87], [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87] -(79) CometColumnarExchange +(75) CometExchange Input [4]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87] -Arguments: hashpartitioning(wr_item_sk#84, wr_order_number#85, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(wr_item_sk#84, wr_order_number#85, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] -(80) CometSort +(76) CometSort Input [4]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87] Arguments: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87], [wr_item_sk#84 ASC NULLS FIRST, wr_order_number#85 ASC NULLS FIRST] -(81) CometSortMergeJoin +(77) CometSortMergeJoin Left output [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82] Right output [4]: [wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87] Arguments: [ws_item_sk#76, ws_order_number#79], [wr_item_sk#84, wr_order_number#85], LeftOuter -(82) CometProject +(78) CometProject Input [11]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_order_number#79, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82, wr_item_sk#84, wr_order_number#85, wr_return_amt#86, wr_net_loss#87] Arguments: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82, wr_return_amt#86, wr_net_loss#87], [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82, wr_return_amt#86, wr_net_loss#87] -(83) ReusedExchange [Reuses operator id: 15] +(79) ReusedExchange [Reuses operator id: 15] Output [1]: [d_date_sk#89] -(84) CometBroadcastHashJoin +(80) CometBroadcastHashJoin Left output [8]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82, wr_return_amt#86, wr_net_loss#87] Right output [1]: [d_date_sk#89] Arguments: [ws_sold_date_sk#82], [d_date_sk#89], Inner, BuildRight -(85) CometProject +(81) CometProject Input [9]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, ws_sold_date_sk#82, wr_return_amt#86, wr_net_loss#87, d_date_sk#89] Arguments: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87], [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87] -(86) Scan parquet spark_catalog.default.web_site +(82) Scan parquet spark_catalog.default.web_site Output [2]: [web_site_sk#90, web_site_id#91] Batched: true Location [not included in comparison]/{warehouse_dir}/web_site] PushedFilters: [IsNotNull(web_site_sk)] ReadSchema: struct -(87) CometFilter +(83) CometFilter Input [2]: [web_site_sk#90, web_site_id#91] Condition : isnotnull(web_site_sk#90) -(88) CometBroadcastExchange +(84) CometBroadcastExchange Input [2]: [web_site_sk#90, web_site_id#91] Arguments: [web_site_sk#90, web_site_id#91] -(89) CometBroadcastHashJoin +(85) CometBroadcastHashJoin Left output [7]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87] Right output [2]: [web_site_sk#90, web_site_id#91] Arguments: [ws_web_site_sk#77], [web_site_sk#90], Inner, BuildRight -(90) CometProject +(86) CometProject Input [9]: [ws_item_sk#76, ws_web_site_sk#77, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_sk#90, web_site_id#91] Arguments: [ws_item_sk#76, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91], [ws_item_sk#76, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] -(91) ReusedExchange [Reuses operator id: 26] +(87) ReusedExchange [Reuses operator id: 26] Output [1]: [i_item_sk#92] -(92) CometBroadcastHashJoin +(88) CometBroadcastHashJoin Left output [7]: [ws_item_sk#76, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] Right output [1]: [i_item_sk#92] Arguments: [ws_item_sk#76], [i_item_sk#92], Inner, BuildRight -(93) CometProject +(89) CometProject Input [8]: [ws_item_sk#76, ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91, i_item_sk#92] Arguments: [ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91], [ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] -(94) ReusedExchange [Reuses operator id: 32] +(90) ReusedExchange [Reuses operator id: 32] Output [1]: [p_promo_sk#93] -(95) CometBroadcastHashJoin +(91) CometBroadcastHashJoin Left output [6]: [ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] Right output [1]: [p_promo_sk#93] Arguments: [ws_promo_sk#78], [p_promo_sk#93], Inner, BuildRight -(96) CometProject +(92) CometProject Input [7]: [ws_promo_sk#78, ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91, p_promo_sk#93] Arguments: [ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91], [ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] -(97) ColumnarToRow [codegen id : 5] +(93) ColumnarToRow [codegen id : 5] Input [5]: [ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] -(98) HashAggregate [codegen id : 5] +(94) HashAggregate [codegen id : 5] Input [5]: [ws_ext_sales_price#80, ws_net_profit#81, wr_return_amt#86, wr_net_loss#87, web_site_id#91] Keys [1]: [web_site_id#91] Functions [3]: [partial_sum(UnscaledValue(ws_ext_sales_price#80)), partial_sum(coalesce(cast(wr_return_amt#86 as decimal(12,2)), 0.00)), partial_sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#87 as decimal(12,2)), 0.00)))] Aggregate Attributes [5]: [sum#94, sum#95, isEmpty#96, sum#97, isEmpty#98] Results [6]: [web_site_id#91, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103] -(99) RowToColumnar +(95) Exchange Input [6]: [web_site_id#91, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103] +Arguments: hashpartitioning(web_site_id#91, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(100) CometColumnarExchange -Input [6]: [web_site_id#91, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103] -Arguments: hashpartitioning(web_site_id#91, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(101) ColumnarToRow [codegen id : 6] -Input [6]: [web_site_id#91, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103] - -(102) HashAggregate [codegen id : 6] +(96) HashAggregate [codegen id : 6] Input [6]: [web_site_id#91, sum#99, sum#100, isEmpty#101, sum#102, isEmpty#103] Keys [1]: [web_site_id#91] Functions [3]: [sum(UnscaledValue(ws_ext_sales_price#80)), sum(coalesce(cast(wr_return_amt#86 as decimal(12,2)), 0.00)), sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#87 as decimal(12,2)), 0.00)))] Aggregate Attributes [3]: [sum(UnscaledValue(ws_ext_sales_price#80))#104, sum(coalesce(cast(wr_return_amt#86 as decimal(12,2)), 0.00))#105, sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#87 as decimal(12,2)), 0.00)))#106] Results [5]: [web channel AS channel#107, concat(web_site, web_site_id#91) AS id#108, MakeDecimal(sum(UnscaledValue(ws_ext_sales_price#80))#104,17,2) AS sales#109, sum(coalesce(cast(wr_return_amt#86 as decimal(12,2)), 0.00))#105 AS returns#110, sum((ws_net_profit#81 - coalesce(cast(wr_net_loss#87 as decimal(12,2)), 0.00)))#106 AS profit#111] -(103) Union +(97) Union -(104) HashAggregate [codegen id : 7] +(98) HashAggregate [codegen id : 7] Input [5]: [channel#35, id#36, sales#37, returns#38, profit#39] Keys [2]: [channel#35, id#36] Functions [3]: [partial_sum(sales#37), partial_sum(returns#38), partial_sum(profit#39)] Aggregate Attributes [6]: [sum#112, isEmpty#113, sum#114, isEmpty#115, sum#116, isEmpty#117] Results [8]: [channel#35, id#36, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -(105) RowToColumnar -Input [8]: [channel#35, id#36, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] - -(106) CometColumnarExchange -Input [8]: [channel#35, id#36, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] -Arguments: hashpartitioning(channel#35, id#36, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] - -(107) ColumnarToRow [codegen id : 8] +(99) Exchange Input [8]: [channel#35, id#36, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] +Arguments: hashpartitioning(channel#35, id#36, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(108) HashAggregate [codegen id : 8] +(100) HashAggregate [codegen id : 8] Input [8]: [channel#35, id#36, sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] Keys [2]: [channel#35, id#36] Functions [3]: [sum(sales#37), sum(returns#38), sum(profit#39)] Aggregate Attributes [3]: [sum(sales#37)#124, sum(returns#38)#125, sum(profit#39)#126] Results [5]: [channel#35, id#36, cast(sum(sales#37)#124 as decimal(37,2)) AS sales#127, cast(sum(returns#38)#125 as decimal(38,2)) AS returns#128, cast(sum(profit#39)#126 as decimal(38,2)) AS profit#129] -(109) ReusedExchange [Reuses operator id: 106] +(101) ReusedExchange [Reuses operator id: 99] Output [8]: [channel#130, id#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] -(110) ColumnarToRow [codegen id : 16] -Input [8]: [channel#130, id#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] - -(111) HashAggregate [codegen id : 16] +(102) HashAggregate [codegen id : 16] Input [8]: [channel#130, id#131, sum#132, isEmpty#133, sum#134, isEmpty#135, sum#136, isEmpty#137] Keys [2]: [channel#130, id#131] Functions [3]: [sum(sales#138), sum(returns#139), sum(profit#140)] Aggregate Attributes [3]: [sum(sales#138)#124, sum(returns#139)#125, sum(profit#140)#126] Results [4]: [channel#130, sum(sales#138)#124 AS sales#141, sum(returns#139)#125 AS returns#142, sum(profit#140)#126 AS profit#143] -(112) HashAggregate [codegen id : 16] +(103) HashAggregate [codegen id : 16] Input [4]: [channel#130, sales#141, returns#142, profit#143] Keys [1]: [channel#130] Functions [3]: [partial_sum(sales#141), partial_sum(returns#142), partial_sum(profit#143)] Aggregate Attributes [6]: [sum#144, isEmpty#145, sum#146, isEmpty#147, sum#148, isEmpty#149] Results [7]: [channel#130, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] -(113) RowToColumnar -Input [7]: [channel#130, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] - -(114) CometColumnarExchange +(104) Exchange Input [7]: [channel#130, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] -Arguments: hashpartitioning(channel#130, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(channel#130, 5), ENSURE_REQUIREMENTS, [plan_id=11] -(115) ColumnarToRow [codegen id : 17] -Input [7]: [channel#130, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] - -(116) HashAggregate [codegen id : 17] +(105) HashAggregate [codegen id : 17] Input [7]: [channel#130, sum#150, isEmpty#151, sum#152, isEmpty#153, sum#154, isEmpty#155] Keys [1]: [channel#130] Functions [3]: [sum(sales#141), sum(returns#142), sum(profit#143)] Aggregate Attributes [3]: [sum(sales#141)#156, sum(returns#142)#157, sum(profit#143)#158] Results [5]: [channel#130, null AS id#159, sum(sales#141)#156 AS sales#160, sum(returns#142)#157 AS returns#161, sum(profit#143)#158 AS profit#162] -(117) ReusedExchange [Reuses operator id: 106] +(106) ReusedExchange [Reuses operator id: 99] Output [8]: [channel#163, id#164, sum#165, isEmpty#166, sum#167, isEmpty#168, sum#169, isEmpty#170] -(118) ColumnarToRow [codegen id : 25] -Input [8]: [channel#163, id#164, sum#165, isEmpty#166, sum#167, isEmpty#168, sum#169, isEmpty#170] - -(119) HashAggregate [codegen id : 25] +(107) HashAggregate [codegen id : 25] Input [8]: [channel#163, id#164, sum#165, isEmpty#166, sum#167, isEmpty#168, sum#169, isEmpty#170] Keys [2]: [channel#163, id#164] Functions [3]: [sum(sales#171), sum(returns#172), sum(profit#173)] Aggregate Attributes [3]: [sum(sales#171)#124, sum(returns#172)#125, sum(profit#173)#126] Results [3]: [sum(sales#171)#124 AS sales#174, sum(returns#172)#125 AS returns#175, sum(profit#173)#126 AS profit#176] -(120) HashAggregate [codegen id : 25] +(108) HashAggregate [codegen id : 25] Input [3]: [sales#174, returns#175, profit#176] Keys: [] Functions [3]: [partial_sum(sales#174), partial_sum(returns#175), partial_sum(profit#176)] Aggregate Attributes [6]: [sum#177, isEmpty#178, sum#179, isEmpty#180, sum#181, isEmpty#182] Results [6]: [sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] -(121) RowToColumnar +(109) Exchange Input [6]: [sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=12] -(122) CometColumnarExchange -Input [6]: [sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] - -(123) ColumnarToRow [codegen id : 26] -Input [6]: [sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] - -(124) HashAggregate [codegen id : 26] +(110) HashAggregate [codegen id : 26] Input [6]: [sum#183, isEmpty#184, sum#185, isEmpty#186, sum#187, isEmpty#188] Keys: [] Functions [3]: [sum(sales#174), sum(returns#175), sum(profit#176)] Aggregate Attributes [3]: [sum(sales#174)#189, sum(returns#175)#190, sum(profit#176)#191] Results [5]: [null AS channel#192, null AS id#193, sum(sales#174)#189 AS sales#194, sum(returns#175)#190 AS returns#195, sum(profit#176)#191 AS profit#196] -(125) Union +(111) Union -(126) HashAggregate [codegen id : 27] +(112) HashAggregate [codegen id : 27] Input [5]: [channel#35, id#36, sales#127, returns#128, profit#129] Keys [5]: [channel#35, id#36, sales#127, returns#128, profit#129] Functions: [] Aggregate Attributes: [] Results [5]: [channel#35, id#36, sales#127, returns#128, profit#129] -(127) RowToColumnar -Input [5]: [channel#35, id#36, sales#127, returns#128, profit#129] - -(128) CometColumnarExchange +(113) Exchange Input [5]: [channel#35, id#36, sales#127, returns#128, profit#129] -Arguments: hashpartitioning(channel#35, id#36, sales#127, returns#128, profit#129, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Arguments: hashpartitioning(channel#35, id#36, sales#127, returns#128, profit#129, 5), ENSURE_REQUIREMENTS, [plan_id=13] -(129) CometHashAggregate +(114) HashAggregate [codegen id : 28] Input [5]: [channel#35, id#36, sales#127, returns#128, profit#129] Keys [5]: [channel#35, id#36, sales#127, returns#128, profit#129] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#35, id#36, sales#127, returns#128, profit#129] -(130) CometTakeOrderedAndProject -Input [5]: [channel#35, id#36, sales#127, returns#128, profit#129] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#35 ASC NULLS FIRST,id#36 ASC NULLS FIRST], output=[channel#35,id#36,sales#127,returns#128,profit#129]), [channel#35, id#36, sales#127, returns#128, profit#129], 100, [channel#35 ASC NULLS FIRST, id#36 ASC NULLS FIRST], [channel#35, id#36, sales#127, returns#128, profit#129] - -(131) ColumnarToRow [codegen id : 28] +(115) TakeOrderedAndProject Input [5]: [channel#35, id#36, sales#127, returns#128, profit#129] +Arguments: 100, [channel#35 ASC NULLS FIRST, id#36 ASC NULLS FIRST], [channel#35, id#36, sales#127, returns#128, profit#129] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (136) -+- * ColumnarToRow (135) - +- CometProject (134) - +- CometFilter (133) - +- CometScan parquet spark_catalog.default.date_dim (132) +BroadcastExchange (120) ++- * ColumnarToRow (119) + +- CometProject (118) + +- CometFilter (117) + +- CometScan parquet spark_catalog.default.date_dim (116) -(132) Scan parquet spark_catalog.default.date_dim +(116) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#14, d_date#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(133) CometFilter +(117) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1998-08-04)) AND (d_date#15 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) -(134) CometProject +(118) CometProject Input [2]: [d_date_sk#14, d_date#15] Arguments: [d_date_sk#14], [d_date_sk#14] -(135) ColumnarToRow [codegen id : 1] +(119) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(136) BroadcastExchange +(120) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] -Subquery:2 Hosting operator id = 41 Hosting Expression = cs_sold_date_sk#46 IN dynamicpruning#8 +Subquery:2 Hosting operator id = 39 Hosting Expression = cs_sold_date_sk#46 IN dynamicpruning#8 -Subquery:3 Hosting operator id = 72 Hosting Expression = ws_sold_date_sk#82 IN dynamicpruning#8 +Subquery:3 Hosting operator id = 68 Hosting Expression = ws_sold_date_sk#82 IN dynamicpruning#8 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt index 8ad016ab7..75dbc06d7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q80a/simplified.txt @@ -1,169 +1,153 @@ -WholeStageCodegen (28) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,id,sales,returns,profit] - CometHashAggregate [channel,id,sales,returns,profit] - CometColumnarExchange [channel,id,sales,returns,profit] #1 - RowToColumnar - WholeStageCodegen (27) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (8) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id] #2 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (28) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Exchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (27) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (8) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel,id] #2 + WholeStageCodegen (7) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (2) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] InputAdapter - Union - WholeStageCodegen (2) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + Exchange [s_store_id] #3 + WholeStageCodegen (1) + HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] ColumnarToRow InputAdapter - CometColumnarExchange [s_store_id] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [s_store_id,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk,ss_ticket_number] #4 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometColumnarExchange [sr_item_sk,sr_ticket_number] #6 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #7 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #8 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #9 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #10 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (4) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #4 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #6 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #8 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #9 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #10 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (4) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [cp_catalog_page_id] #11 + WholeStageCodegen (3) + HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] ColumnarToRow InputAdapter - CometColumnarExchange [cp_catalog_page_id] #11 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [cp_catalog_page_id,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometColumnarExchange [cs_item_sk,cs_order_number] #12 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometColumnarExchange [cr_item_sk,cr_order_number] #13 - CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #14 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 - WholeStageCodegen (6) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #12 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #13 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #7 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #14 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #9 + ReusedExchange [p_promo_sk] #10 + WholeStageCodegen (6) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [web_site_id] #15 + WholeStageCodegen (5) + HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] ColumnarToRow InputAdapter - CometColumnarExchange [web_site_id] #15 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [web_site_id,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] [sum,sum,isEmpty,sum,isEmpty,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] - CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] - CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometColumnarExchange [ws_item_sk,ws_order_number] #16 - CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometColumnarExchange [wr_item_sk,wr_order_number] #17 - CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [web_site_sk,web_site_id] #18 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 - WholeStageCodegen (17) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel] #19 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (26) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #20 - RowToColumnar - WholeStageCodegen (25) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #16 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #17 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #7 + CometBroadcastExchange [web_site_sk,web_site_id] #18 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #9 + ReusedExchange [p_promo_sk] #10 + WholeStageCodegen (17) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel] #19 + WholeStageCodegen (16) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (26) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange #20 + WholeStageCodegen (25) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt index 9d890a1a9..efdca9dca 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/explain.txt @@ -1,50 +1,40 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * Project (45) - +- Window (44) - +- * ColumnarToRow (43) - +- CometSort (42) - +- CometColumnarExchange (41) - +- CometHashAggregate (40) - +- CometColumnarExchange (39) - +- RowToColumnar (38) - +- * HashAggregate (37) - +- Union (36) - :- * HashAggregate (19) - : +- * ColumnarToRow (18) - : +- CometColumnarExchange (17) - : +- RowToColumnar (16) - : +- * HashAggregate (15) - : +- * ColumnarToRow (14) - : +- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.item (9) - :- * HashAggregate (27) - : +- * ColumnarToRow (26) - : +- CometColumnarExchange (25) - : +- RowToColumnar (24) - : +- * HashAggregate (23) - : +- * HashAggregate (22) - : +- * ColumnarToRow (21) - : +- ReusedExchange (20) - +- * HashAggregate (35) - +- * ColumnarToRow (34) - +- CometColumnarExchange (33) - +- RowToColumnar (32) - +- * HashAggregate (31) - +- * HashAggregate (30) - +- * ColumnarToRow (29) - +- ReusedExchange (28) +TakeOrderedAndProject (36) ++- * Project (35) + +- Window (34) + +- * Sort (33) + +- Exchange (32) + +- * HashAggregate (31) + +- Exchange (30) + +- * HashAggregate (29) + +- Union (28) + :- * HashAggregate (17) + : +- Exchange (16) + : +- * HashAggregate (15) + : +- * ColumnarToRow (14) + : +- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.item (9) + :- * HashAggregate (22) + : +- Exchange (21) + : +- * HashAggregate (20) + : +- * HashAggregate (19) + : +- ReusedExchange (18) + +- * HashAggregate (27) + +- Exchange (26) + +- * HashAggregate (25) + +- * HashAggregate (24) + +- ReusedExchange (23) (1) Scan parquet spark_catalog.default.web_sales @@ -121,170 +111,142 @@ Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] Aggregate Attributes [1]: [sum#10] Results [3]: [i_category#9, i_class#8, sum#11] -(16) RowToColumnar +(16) Exchange Input [3]: [i_category#9, i_class#8, sum#11] +Arguments: hashpartitioning(i_category#9, i_class#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [3]: [i_category#9, i_class#8, sum#11] -Arguments: hashpartitioning(i_category#9, i_class#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 2] -Input [3]: [i_category#9, i_class#8, sum#11] - -(19) HashAggregate [codegen id : 2] +(17) HashAggregate [codegen id : 2] Input [3]: [i_category#9, i_class#8, sum#11] Keys [2]: [i_category#9, i_class#8] Functions [1]: [sum(UnscaledValue(ws_net_paid#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#2))#12] Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ws_net_paid#2))#12,17,2) as decimal(27,2)) AS total_sum#13, i_category#9, i_class#8, 0 AS g_category#14, 0 AS g_class#15, 0 AS lochierarchy#16] -(20) ReusedExchange [Reuses operator id: 17] +(18) ReusedExchange [Reuses operator id: 16] Output [3]: [i_category#17, i_class#18, sum#19] -(21) ColumnarToRow [codegen id : 4] -Input [3]: [i_category#17, i_class#18, sum#19] - -(22) HashAggregate [codegen id : 4] +(19) HashAggregate [codegen id : 4] Input [3]: [i_category#17, i_class#18, sum#19] Keys [2]: [i_category#17, i_class#18] Functions [1]: [sum(UnscaledValue(ws_net_paid#20))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#20))#12] Results [2]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#20))#12,17,2) AS total_sum#21, i_category#17] -(23) HashAggregate [codegen id : 4] +(20) HashAggregate [codegen id : 4] Input [2]: [total_sum#21, i_category#17] Keys [1]: [i_category#17] Functions [1]: [partial_sum(total_sum#21)] Aggregate Attributes [2]: [sum#22, isEmpty#23] Results [3]: [i_category#17, sum#24, isEmpty#25] -(24) RowToColumnar +(21) Exchange Input [3]: [i_category#17, sum#24, isEmpty#25] +Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(25) CometColumnarExchange -Input [3]: [i_category#17, sum#24, isEmpty#25] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(26) ColumnarToRow [codegen id : 5] -Input [3]: [i_category#17, sum#24, isEmpty#25] - -(27) HashAggregate [codegen id : 5] +(22) HashAggregate [codegen id : 5] Input [3]: [i_category#17, sum#24, isEmpty#25] Keys [1]: [i_category#17] Functions [1]: [sum(total_sum#21)] Aggregate Attributes [1]: [sum(total_sum#21)#26] Results [6]: [sum(total_sum#21)#26 AS total_sum#27, i_category#17, null AS i_class#28, 0 AS g_category#29, 1 AS g_class#30, 1 AS lochierarchy#31] -(28) ReusedExchange [Reuses operator id: 17] +(23) ReusedExchange [Reuses operator id: 16] Output [3]: [i_category#32, i_class#33, sum#34] -(29) ColumnarToRow [codegen id : 7] -Input [3]: [i_category#32, i_class#33, sum#34] - -(30) HashAggregate [codegen id : 7] +(24) HashAggregate [codegen id : 7] Input [3]: [i_category#32, i_class#33, sum#34] Keys [2]: [i_category#32, i_class#33] Functions [1]: [sum(UnscaledValue(ws_net_paid#35))] Aggregate Attributes [1]: [sum(UnscaledValue(ws_net_paid#35))#12] Results [1]: [MakeDecimal(sum(UnscaledValue(ws_net_paid#35))#12,17,2) AS total_sum#36] -(31) HashAggregate [codegen id : 7] +(25) HashAggregate [codegen id : 7] Input [1]: [total_sum#36] Keys: [] Functions [1]: [partial_sum(total_sum#36)] Aggregate Attributes [2]: [sum#37, isEmpty#38] Results [2]: [sum#39, isEmpty#40] -(32) RowToColumnar -Input [2]: [sum#39, isEmpty#40] - -(33) CometColumnarExchange -Input [2]: [sum#39, isEmpty#40] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] - -(34) ColumnarToRow [codegen id : 8] +(26) Exchange Input [2]: [sum#39, isEmpty#40] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=3] -(35) HashAggregate [codegen id : 8] +(27) HashAggregate [codegen id : 8] Input [2]: [sum#39, isEmpty#40] Keys: [] Functions [1]: [sum(total_sum#36)] Aggregate Attributes [1]: [sum(total_sum#36)#41] Results [6]: [sum(total_sum#36)#41 AS total_sum#42, null AS i_category#43, null AS i_class#44, 1 AS g_category#45, 1 AS g_class#46, 2 AS lochierarchy#47] -(36) Union +(28) Union -(37) HashAggregate [codegen id : 9] +(29) HashAggregate [codegen id : 9] Input [6]: [total_sum#13, i_category#9, i_class#8, g_category#14, g_class#15, lochierarchy#16] Keys [6]: [total_sum#13, i_category#9, i_class#8, g_category#14, g_class#15, lochierarchy#16] Functions: [] Aggregate Attributes: [] Results [6]: [total_sum#13, i_category#9, i_class#8, g_category#14, g_class#15, lochierarchy#16] -(38) RowToColumnar +(30) Exchange Input [6]: [total_sum#13, i_category#9, i_class#8, g_category#14, g_class#15, lochierarchy#16] +Arguments: hashpartitioning(total_sum#13, i_category#9, i_class#8, g_category#14, g_class#15, lochierarchy#16, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(39) CometColumnarExchange -Input [6]: [total_sum#13, i_category#9, i_class#8, g_category#14, g_class#15, lochierarchy#16] -Arguments: hashpartitioning(total_sum#13, i_category#9, i_class#8, g_category#14, g_class#15, lochierarchy#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(40) CometHashAggregate +(31) HashAggregate [codegen id : 10] Input [6]: [total_sum#13, i_category#9, i_class#8, g_category#14, g_class#15, lochierarchy#16] Keys [6]: [total_sum#13, i_category#9, i_class#8, g_category#14, g_class#15, lochierarchy#16] Functions: [] +Aggregate Attributes: [] +Results [5]: [total_sum#13, i_category#9, i_class#8, lochierarchy#16, CASE WHEN (g_class#15 = 0) THEN i_category#9 END AS _w0#48] -(41) CometColumnarExchange -Input [5]: [total_sum#13, i_category#9, i_class#8, lochierarchy#16, _w0#48] -Arguments: hashpartitioning(lochierarchy#16, _w0#48, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(42) CometSort +(32) Exchange Input [5]: [total_sum#13, i_category#9, i_class#8, lochierarchy#16, _w0#48] -Arguments: [total_sum#13, i_category#9, i_class#8, lochierarchy#16, _w0#48], [lochierarchy#16 ASC NULLS FIRST, _w0#48 ASC NULLS FIRST, total_sum#13 DESC NULLS LAST] +Arguments: hashpartitioning(lochierarchy#16, _w0#48, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) ColumnarToRow [codegen id : 10] +(33) Sort [codegen id : 11] Input [5]: [total_sum#13, i_category#9, i_class#8, lochierarchy#16, _w0#48] +Arguments: [lochierarchy#16 ASC NULLS FIRST, _w0#48 ASC NULLS FIRST, total_sum#13 DESC NULLS LAST], false, 0 -(44) Window +(34) Window Input [5]: [total_sum#13, i_category#9, i_class#8, lochierarchy#16, _w0#48] Arguments: [rank(total_sum#13) windowspecdefinition(lochierarchy#16, _w0#48, total_sum#13 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#49], [lochierarchy#16, _w0#48], [total_sum#13 DESC NULLS LAST] -(45) Project [codegen id : 11] +(35) Project [codegen id : 12] Output [5]: [total_sum#13, i_category#9, i_class#8, lochierarchy#16, rank_within_parent#49] Input [6]: [total_sum#13, i_category#9, i_class#8, lochierarchy#16, _w0#48, rank_within_parent#49] -(46) TakeOrderedAndProject +(36) TakeOrderedAndProject Input [5]: [total_sum#13, i_category#9, i_class#8, lochierarchy#16, rank_within_parent#49] Arguments: 100, [lochierarchy#16 DESC NULLS LAST, CASE WHEN (lochierarchy#16 = 0) THEN i_category#9 END ASC NULLS FIRST, rank_within_parent#49 ASC NULLS FIRST], [total_sum#13, i_category#9, i_class#8, lochierarchy#16, rank_within_parent#49] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (51) -+- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.date_dim (47) +BroadcastExchange (41) ++- * ColumnarToRow (40) + +- CometProject (39) + +- CometFilter (38) + +- CometScan parquet spark_catalog.default.date_dim (37) -(47) Scan parquet spark_catalog.default.date_dim +(37) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(48) CometFilter +(38) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) -(49) CometProject +(39) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(50) ColumnarToRow [codegen id : 1] +(40) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(51) BroadcastExchange +(41) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt index 66385580c..8996119f2 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q86a/simplified.txt @@ -1,72 +1,64 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_class] - WholeStageCodegen (11) + WholeStageCodegen (12) Project [total_sum,i_category,i_class,lochierarchy,rank_within_parent] InputAdapter Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (10) - ColumnarToRow + WholeStageCodegen (11) + Sort [lochierarchy,_w0,total_sum] InputAdapter - CometSort [total_sum,i_category,i_class,lochierarchy,_w0] - CometColumnarExchange [lochierarchy,_w0] #1 - CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,g_category,g_class] - CometColumnarExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 - RowToColumnar + Exchange [lochierarchy,_w0] #1 + WholeStageCodegen (10) + HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] [_w0] + InputAdapter + Exchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 WholeStageCodegen (9) HashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] InputAdapter Union WholeStageCodegen (2) HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,g_category,g_class,lochierarchy,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category,i_class] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_category,i_class,ws_net_paid] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ws_net_paid,i_class,i_category] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] - CometProject [ws_item_sk,ws_net_paid] - CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] - CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_class,i_category] #6 - CometFilter [i_item_sk,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] + InputAdapter + Exchange [i_category,i_class] #3 + WholeStageCodegen (1) + HashAggregate [i_category,i_class,ws_net_paid] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ws_net_paid,i_class,i_category] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] + CometProject [ws_item_sk,ws_net_paid] + CometBroadcastHashJoin [ws_item_sk,ws_net_paid,ws_sold_date_sk,d_date_sk] + CometFilter [ws_item_sk,ws_net_paid,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_net_paid,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_class,i_category] #6 + CometFilter [i_item_sk,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] WholeStageCodegen (5) HashAggregate [i_category,sum,isEmpty] [sum(total_sum),total_sum,i_class,g_category,g_class,lochierarchy,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_category] #7 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [i_category,total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,sum] #3 + InputAdapter + Exchange [i_category] #7 + WholeStageCodegen (4) + HashAggregate [i_category,total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] + InputAdapter + ReusedExchange [i_category,i_class,sum] #3 WholeStageCodegen (8) HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #8 - RowToColumnar - WholeStageCodegen (7) - HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [i_category,i_class,sum] #3 + InputAdapter + Exchange #8 + WholeStageCodegen (7) + HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [i_category,i_class,sum] [sum(UnscaledValue(ws_net_paid)),total_sum,sum] + InputAdapter + ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt index 5b3adf06d..6e0c2044f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/explain.txt @@ -1,33 +1,29 @@ == Physical Plan == -* ColumnarToRow (29) -+- CometSort (28) - +- CometColumnarExchange (27) - +- RowToColumnar (26) - +- * Project (25) - +- Window (24) - +- * ColumnarToRow (23) - +- CometSort (22) - +- CometColumnarExchange (21) - +- RowToColumnar (20) - +- * HashAggregate (19) - +- * ColumnarToRow (18) - +- CometColumnarExchange (17) - +- RowToColumnar (16) - +- * HashAggregate (15) - +- * ColumnarToRow (14) - +- CometProject (13) - +- CometBroadcastHashJoin (12) - :- CometProject (7) - : +- CometBroadcastHashJoin (6) - : :- CometFilter (2) - : : +- CometScan parquet spark_catalog.default.store_sales (1) - : +- CometBroadcastExchange (5) - : +- CometFilter (4) - : +- CometScan parquet spark_catalog.default.item (3) - +- CometBroadcastExchange (11) - +- CometProject (10) - +- CometFilter (9) - +- CometScan parquet spark_catalog.default.date_dim (8) +* ColumnarToRow (25) ++- CometSort (24) + +- CometColumnarExchange (23) + +- RowToColumnar (22) + +- * Project (21) + +- Window (20) + +- * Sort (19) + +- Exchange (18) + +- * HashAggregate (17) + +- Exchange (16) + +- * HashAggregate (15) + +- * ColumnarToRow (14) + +- CometProject (13) + +- CometBroadcastHashJoin (12) + :- CometProject (7) + : +- CometBroadcastHashJoin (6) + : :- CometFilter (2) + : : +- CometScan parquet spark_catalog.default.store_sales (1) + : +- CometBroadcastExchange (5) + : +- CometFilter (4) + : +- CometScan parquet spark_catalog.default.item (3) + +- CometBroadcastExchange (11) + +- CometProject (10) + +- CometFilter (9) + +- CometScan parquet spark_catalog.default.date_dim (8) (1) Scan parquet spark_catalog.default.store_sales @@ -104,88 +100,76 @@ Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] Aggregate Attributes [1]: [sum#13] Results [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -(16) RowToColumnar +(16) Exchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, [plan_id=1] -(17) CometColumnarExchange -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] - -(18) ColumnarToRow [codegen id : 2] -Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] - -(19) HashAggregate [codegen id : 2] +(17) HashAggregate [codegen id : 2] Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#14] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_ext_sales_price#2))#15] Results [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#15,17,2) AS itemrevenue#16, MakeDecimal(sum(UnscaledValue(ss_ext_sales_price#2))#15,17,2) AS _w0#17] -(20) RowToColumnar -Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] - -(21) CometColumnarExchange -Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(22) CometSort +(18) Exchange Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] -Arguments: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17], [i_class#9 ASC NULLS FIRST] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(23) ColumnarToRow [codegen id : 3] +(19) Sort [codegen id : 3] Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] +Arguments: [i_class#9 ASC NULLS FIRST], false, 0 -(24) Window +(20) Window Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17] Arguments: [sum(_w0#17) windowspecdefinition(i_class#9, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS _we0#18], [i_class#9] -(25) Project [codegen id : 4] +(21) Project [codegen id : 4] Output [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, ((_w0#17 * 100) / _we0#18) AS revenueratio#19] Input [8]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, _w0#17, _we0#18] -(26) RowToColumnar +(22) RowToColumnar Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] -(27) CometColumnarExchange +(23) CometColumnarExchange Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] Arguments: rangepartitioning(i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] -(28) CometSort +(24) CometSort Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] Arguments: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19], [i_category#10 ASC NULLS FIRST, i_class#9 ASC NULLS FIRST, i_item_id#6 ASC NULLS FIRST, i_item_desc#7 ASC NULLS FIRST, revenueratio#19 ASC NULLS FIRST] -(29) ColumnarToRow [codegen id : 5] +(25) ColumnarToRow [codegen id : 5] Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#16, revenueratio#19] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (34) -+- * ColumnarToRow (33) - +- CometProject (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.date_dim (30) +BroadcastExchange (30) ++- * ColumnarToRow (29) + +- CometProject (28) + +- CometFilter (27) + +- CometScan parquet spark_catalog.default.date_dim (26) -(30) Scan parquet spark_catalog.default.date_dim +(26) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#11, d_date#12] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1999-02-22), LessThanOrEqual(d_date,1999-03-24), IsNotNull(d_date_sk)] ReadSchema: struct -(31) CometFilter +(27) CometFilter Input [2]: [d_date_sk#11, d_date#12] Condition : (((isnotnull(d_date#12) AND (d_date#12 >= 1999-02-22)) AND (d_date#12 <= 1999-03-24)) AND isnotnull(d_date_sk#11)) -(32) CometProject +(28) CometProject Input [2]: [d_date_sk#11, d_date#12] Arguments: [d_date_sk#11], [d_date_sk#11] -(33) ColumnarToRow [codegen id : 1] +(29) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#11] -(34) BroadcastExchange +(30) BroadcastExchange Input [1]: [d_date_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=4] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt index dd87e6394..c85b14e14 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7-spark4_0/q98/simplified.txt @@ -9,39 +9,35 @@ WholeStageCodegen (5) InputAdapter Window [_w0,i_class] WholeStageCodegen (3) - ColumnarToRow + Sort [i_class] InputAdapter - CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometColumnarExchange [i_class] #2 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 - RowToColumnar - WholeStageCodegen (1) - HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] - ColumnarToRow - InputAdapter - CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] - CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 - CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + Exchange [i_class] #2 + WholeStageCodegen (2) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum] [sum(UnscaledValue(ss_ext_sales_price)),itemrevenue,_w0,sum] + InputAdapter + Exchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + WholeStageCodegen (1) + HashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,ss_ext_sales_price] [sum,sum] + ColumnarToRow + InputAdapter + CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] + CometProject [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastHashJoin [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk,i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometFilter [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ext_sales_price,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] #5 + CometFilter [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt index f2de73f90..fc449ff52 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (39) +- CometTakeOrderedAndProject (38) +- CometHashAggregate (37) - +- CometColumnarExchange (36) + +- CometExchange (36) +- CometHashAggregate (35) +- CometProject (34) +- CometBroadcastHashJoin (33) @@ -207,9 +207,9 @@ Input [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purch Keys [8]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30] Functions [1]: [partial_count(1)] -(36) CometColumnarExchange +(36) CometExchange Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#31] -Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (37) CometHashAggregate Input [9]: [cd_gender#23, cd_marital_status#24, cd_education_status#25, cd_purchase_estimate#26, cd_credit_rating#27, cd_dep_count#28, cd_dep_employed_count#29, cd_dep_college_count#30, count#31] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt index 243bdbb91..2e8f2022c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q10a/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6] CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cnt1,cd_purchase_estimate,cnt2,cd_credit_rating,cnt3,cd_dep_count,cnt4,cd_dep_employed_count,cnt5,cd_dep_college_count,cnt6,count,count(1)] - CometColumnarExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometExchange [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 CometHashAggregate [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count] CometProject [cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometBroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk,cd_gender,cd_marital_status,cd_education_status,cd_purchase_estimate,cd_credit_rating,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt index 0ae630c49..52d523ca7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/explain.txt @@ -8,7 +8,7 @@ : :- CometBroadcastHashJoin (33) : : :- CometFilter (16) : : : +- CometHashAggregate (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometHashAggregate (13) : : : +- CometProject (12) : : : +- CometBroadcastHashJoin (11) @@ -24,7 +24,7 @@ : : : +- CometScan parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometProject (28) : : +- CometBroadcastHashJoin (27) @@ -41,7 +41,7 @@ : +- CometBroadcastExchange (48) : +- CometFilter (47) : +- CometHashAggregate (46) - : +- CometColumnarExchange (45) + : +- CometExchange (45) : +- CometHashAggregate (44) : +- CometProject (43) : +- CometBroadcastHashJoin (42) @@ -55,7 +55,7 @@ : +- ReusedExchange (41) +- CometBroadcastExchange (64) +- CometHashAggregate (63) - +- CometColumnarExchange (62) + +- CometExchange (62) +- CometHashAggregate (61) +- CometProject (60) +- CometBroadcastHashJoin (59) @@ -134,9 +134,9 @@ Input [10]: [c_customer_id#2, c_first_name#3, c_last_name#4, c_preferred_cust_fl Keys [8]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#11 - ss_ext_discount_amt#10)))] -(14) CometColumnarExchange +(14) CometExchange Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [9]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#15, c_preferred_cust_flag#5, c_birth_country#6, c_login#7, c_email_address#8, sum#16] @@ -212,9 +212,9 @@ Input [10]: [c_customer_id#20, c_first_name#21, c_last_name#22, c_preferred_cust Keys [8]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26] Functions [1]: [partial_sum(UnscaledValue((ss_ext_list_price#29 - ss_ext_discount_amt#28)))] -(30) CometColumnarExchange +(30) CometExchange Input [9]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, sum#34] -Arguments: hashpartitioning(c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate Input [9]: [c_customer_id#20, c_first_name#21, c_last_name#22, d_year#33, c_preferred_cust_flag#23, c_birth_country#24, c_login#25, c_email_address#26, sum#34] @@ -283,9 +283,9 @@ Input [10]: [c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust Keys [8]: [c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_country#45, c_login#46, c_email_address#47, d_year#54] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#50 - ws_ext_discount_amt#49)))] -(45) CometColumnarExchange +(45) CometExchange Input [9]: [c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_country#45, c_login#46, c_email_address#47, d_year#54, sum#55] -Arguments: hashpartitioning(c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_country#45, c_login#46, c_email_address#47, d_year#54, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_country#45, c_login#46, c_email_address#47, d_year#54, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometHashAggregate Input [9]: [c_customer_id#41, c_first_name#42, c_last_name#43, c_preferred_cust_flag#44, c_birth_country#45, c_login#46, c_email_address#47, d_year#54, sum#55] @@ -362,9 +362,9 @@ Input [10]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust Keys [8]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#72] Functions [1]: [partial_sum(UnscaledValue((ws_ext_list_price#68 - ws_ext_discount_amt#67)))] -(62) CometColumnarExchange +(62) CometExchange Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#72, sum#73] -Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#72, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#72, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (63) CometHashAggregate Input [9]: [c_customer_id#59, c_first_name#60, c_last_name#61, c_preferred_cust_flag#62, c_birth_country#63, c_login#64, c_email_address#65, d_year#72, sum#73] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt index c4811d292..8743dd42e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q11/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #1 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -32,7 +32,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,customer_email_address,year_total,c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,sum(UnscaledValue((ss_ext_list_price - ss_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address] #6 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,sum,ss_ext_list_price,ss_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ss_ext_discount_amt,ss_ext_list_price,ss_sold_date_sk,d_date_sk,d_year] @@ -56,7 +56,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #11 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] @@ -71,7 +71,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,sum(UnscaledValue((ws_ext_list_price - ws_ext_discount_amt)))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year] #14 CometHashAggregate [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,d_year,sum,ws_ext_list_price,ws_ext_discount_amt] CometProject [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,c_preferred_cust_flag,c_birth_country,c_login,c_email_address,ws_ext_discount_amt,ws_ext_list_price,ws_sold_date_sk,d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt index fbe5feb3e..155728936 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (22) +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -92,18 +92,18 @@ Input [6]: [ws_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(ws_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ws_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt index e2df89eec..6f51be0c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q12/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c ColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometColumnarExchange [i_class] #1 + CometExchange [i_class] #1 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(ws_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ws_ext_sales_price] CometProject [ws_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ws_ext_sales_price,ws_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt index 90e7b6ca1..2a92752c3 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/explain.txt @@ -4,7 +4,7 @@ +- CometBroadcastHashJoin (83) :- CometFilter (64) : +- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometProject (60) : +- CometBroadcastHashJoin (59) @@ -21,7 +21,7 @@ : : : +- CometBroadcastExchange (43) : : : +- CometBroadcastHashJoin (42) : : : :- CometHashAggregate (32) - : : : : +- CometColumnarExchange (31) + : : : : +- CometExchange (31) : : : : +- CometHashAggregate (30) : : : : +- CometProject (29) : : : : +- CometBroadcastHashJoin (28) @@ -69,7 +69,7 @@ +- CometBroadcastExchange (82) +- CometFilter (81) +- CometHashAggregate (80) - +- CometColumnarExchange (79) + +- CometExchange (79) +- CometHashAggregate (78) +- CometProject (77) +- CometBroadcastHashJoin (76) @@ -235,9 +235,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -376,9 +376,9 @@ Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_cate Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(62) CometColumnarExchange +(62) CometExchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#47, isEmpty#48, count#49] @@ -454,9 +454,9 @@ Input [5]: [ss_quantity#56, ss_list_price#57, i_brand_id#61, i_class_id#62, i_ca Keys [3]: [i_brand_id#61, i_class_id#62, i_category_id#63] Functions [2]: [partial_sum((cast(ss_quantity#56 as decimal(10,0)) * ss_list_price#57)), partial_count(1)] -(79) CometColumnarExchange +(79) CometExchange Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#68, isEmpty#69, count#70] -Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_brand_id#61, i_class_id#62, i_category_id#63, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (80) CometHashAggregate Input [6]: [i_brand_id#61, i_class_id#62, i_category_id#63, sum#68, isEmpty#69, count#70] @@ -488,7 +488,7 @@ Input [12]: [channel#50, i_brand_id#40, i_class_id#41, i_category_id#42, sales#5 Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#53, [id=#54] * ColumnarToRow (102) +- CometHashAggregate (101) - +- CometColumnarExchange (100) + +- CometExchange (100) +- CometHashAggregate (99) +- CometUnion (98) :- CometProject (89) @@ -572,9 +572,9 @@ Input [2]: [quantity#79, list_price#80] Keys: [] Functions [1]: [partial_avg((cast(quantity#79 as decimal(10,0)) * list_price#80))] -(100) CometColumnarExchange +(100) CometExchange Input [2]: [sum#95, count#96] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (101) CometHashAggregate Input [2]: [sum#95, count#96] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt index a51d1d007..bb4be4d3a 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometColumnarExchange #14 + CometExchange #14 CometHashAggregate [sum,count,quantity,list_price] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] @@ -28,7 +28,7 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #3 ReusedExchange [d_date_sk] #10 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #1 + CometExchange [i_brand_id,i_class_id,i_category_id] #1 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -60,7 +60,7 @@ WholeStageCodegen (1) CometBroadcastExchange [brand_id,class_id,category_id] #4 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #5 + CometExchange [brand_id,class_id,category_id] #5 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -120,7 +120,7 @@ WholeStageCodegen (1) CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #4 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #16 + CometExchange [i_brand_id,i_class_id,i_category_id] #16 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt index 0a4f72baa..04ffdbb5f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/explain.txt @@ -2,16 +2,16 @@ * ColumnarToRow (122) +- CometTakeOrderedAndProject (121) +- CometHashAggregate (120) - +- CometColumnarExchange (119) + +- CometExchange (119) +- CometHashAggregate (118) +- CometUnion (117) :- CometHashAggregate (96) - : +- CometColumnarExchange (95) + : +- CometExchange (95) : +- CometHashAggregate (94) : +- CometUnion (93) : :- CometFilter (64) : : +- CometHashAggregate (63) - : : +- CometColumnarExchange (62) + : : +- CometExchange (62) : : +- CometHashAggregate (61) : : +- CometProject (60) : : +- CometBroadcastHashJoin (59) @@ -28,7 +28,7 @@ : : : : +- CometBroadcastExchange (43) : : : : +- CometBroadcastHashJoin (42) : : : : :- CometHashAggregate (32) - : : : : : +- CometColumnarExchange (31) + : : : : : +- CometExchange (31) : : : : : +- CometHashAggregate (30) : : : : : +- CometProject (29) : : : : : +- CometBroadcastHashJoin (28) @@ -75,7 +75,7 @@ : : +- CometScan parquet spark_catalog.default.date_dim (55) : :- CometFilter (78) : : +- CometHashAggregate (77) - : : +- CometColumnarExchange (76) + : : +- CometExchange (76) : : +- CometHashAggregate (75) : : +- CometProject (74) : : +- CometBroadcastHashJoin (73) @@ -89,7 +89,7 @@ : : +- ReusedExchange (72) : +- CometFilter (92) : +- CometHashAggregate (91) - : +- CometColumnarExchange (90) + : +- CometExchange (90) : +- CometHashAggregate (89) : +- CometProject (88) : +- CometBroadcastHashJoin (87) @@ -102,22 +102,22 @@ : : +- ReusedExchange (83) : +- ReusedExchange (86) :- CometHashAggregate (101) - : +- CometColumnarExchange (100) + : +- CometExchange (100) : +- CometHashAggregate (99) : +- CometHashAggregate (98) : +- ReusedExchange (97) :- CometHashAggregate (106) - : +- CometColumnarExchange (105) + : +- CometExchange (105) : +- CometHashAggregate (104) : +- CometHashAggregate (103) : +- ReusedExchange (102) :- CometHashAggregate (111) - : +- CometColumnarExchange (110) + : +- CometExchange (110) : +- CometHashAggregate (109) : +- CometHashAggregate (108) : +- ReusedExchange (107) +- CometHashAggregate (116) - +- CometColumnarExchange (115) + +- CometExchange (115) +- CometHashAggregate (114) +- CometHashAggregate (113) +- ReusedExchange (112) @@ -272,9 +272,9 @@ Input [3]: [brand_id#27, class_id#28, category_id#29] Keys [3]: [brand_id#27, class_id#28, category_id#29] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [3]: [brand_id#27, class_id#28, category_id#29] -Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(brand_id#27, class_id#28, category_id#29, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (32) CometHashAggregate Input [3]: [brand_id#27, class_id#28, category_id#29] @@ -413,9 +413,9 @@ Input [5]: [ss_quantity#2, ss_list_price#3, i_brand_id#40, i_class_id#41, i_cate Keys [3]: [i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum((cast(ss_quantity#2 as decimal(10,0)) * ss_list_price#3)), partial_count(1)] -(62) CometColumnarExchange +(62) CometExchange Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] -Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate Input [6]: [i_brand_id#40, i_class_id#41, i_category_id#42, sum#46, isEmpty#47, count#48] @@ -475,9 +475,9 @@ Input [5]: [cs_quantity#55, cs_list_price#56, i_brand_id#60, i_class_id#61, i_ca Keys [3]: [i_brand_id#60, i_class_id#61, i_category_id#62] Functions [2]: [partial_sum((cast(cs_quantity#55 as decimal(10,0)) * cs_list_price#56)), partial_count(1)] -(76) CometColumnarExchange +(76) CometExchange Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#64, isEmpty#65, count#66] -Arguments: hashpartitioning(i_brand_id#60, i_class_id#61, i_category_id#62, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_brand_id#60, i_class_id#61, i_category_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (77) CometHashAggregate Input [6]: [i_brand_id#60, i_class_id#61, i_category_id#62, sum#64, isEmpty#65, count#66] @@ -537,9 +537,9 @@ Input [5]: [ws_quantity#71, ws_list_price#72, i_brand_id#76, i_class_id#77, i_ca Keys [3]: [i_brand_id#76, i_class_id#77, i_category_id#78] Functions [2]: [partial_sum((cast(ws_quantity#71 as decimal(10,0)) * ws_list_price#72)), partial_count(1)] -(90) CometColumnarExchange +(90) CometExchange Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum#80, isEmpty#81, count#82] -Arguments: hashpartitioning(i_brand_id#76, i_class_id#77, i_category_id#78, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_brand_id#76, i_class_id#77, i_category_id#78, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (91) CometHashAggregate Input [6]: [i_brand_id#76, i_class_id#77, i_category_id#78, sum#80, isEmpty#81, count#82] @@ -560,9 +560,9 @@ Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sales#50 Keys [4]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42] Functions [2]: [partial_sum(sales#50), partial_sum(number_sales#51)] -(95) CometColumnarExchange +(95) CometExchange Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (96) CometHashAggregate Input [7]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum#86, isEmpty#87, sum#88] @@ -582,9 +582,9 @@ Input [5]: [channel#49, i_brand_id#40, i_class_id#41, sum_sales#89, number_sales Keys [3]: [channel#49, i_brand_id#40, i_class_id#41] Functions [2]: [partial_sum(sum_sales#89), partial_sum(number_sales#90)] -(100) CometColumnarExchange +(100) CometExchange Input [6]: [channel#49, i_brand_id#40, i_class_id#41, sum#91, isEmpty#92, sum#93] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (101) CometHashAggregate Input [6]: [channel#49, i_brand_id#40, i_class_id#41, sum#91, isEmpty#92, sum#93] @@ -604,9 +604,9 @@ Input [4]: [channel#49, i_brand_id#40, sum_sales#89, number_sales#90] Keys [2]: [channel#49, i_brand_id#40] Functions [2]: [partial_sum(sum_sales#89), partial_sum(number_sales#90)] -(105) CometColumnarExchange +(105) CometExchange Input [5]: [channel#49, i_brand_id#40, sum#94, isEmpty#95, sum#96] -Arguments: hashpartitioning(channel#49, i_brand_id#40, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(channel#49, i_brand_id#40, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (106) CometHashAggregate Input [5]: [channel#49, i_brand_id#40, sum#94, isEmpty#95, sum#96] @@ -626,9 +626,9 @@ Input [3]: [channel#49, sum_sales#89, number_sales#90] Keys [1]: [channel#49] Functions [2]: [partial_sum(sum_sales#89), partial_sum(number_sales#90)] -(110) CometColumnarExchange +(110) CometExchange Input [4]: [channel#49, sum#97, isEmpty#98, sum#99] -Arguments: hashpartitioning(channel#49, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(channel#49, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (111) CometHashAggregate Input [4]: [channel#49, sum#97, isEmpty#98, sum#99] @@ -648,9 +648,9 @@ Input [2]: [sum_sales#89, number_sales#90] Keys: [] Functions [2]: [partial_sum(sum_sales#89), partial_sum(number_sales#90)] -(115) CometColumnarExchange +(115) CometExchange Input [3]: [sum#100, isEmpty#101, sum#102] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (116) CometHashAggregate Input [3]: [sum#100, isEmpty#101, sum#102] @@ -669,9 +669,9 @@ Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sale Keys [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] Functions: [] -(119) CometColumnarExchange +(119) CometExchange Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] -Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (120) CometHashAggregate Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sales#89, number_sales#90] @@ -690,7 +690,7 @@ Input [6]: [channel#49, i_brand_id#40, i_class_id#41, i_category_id#42, sum_sale Subquery:1 Hosting operator id = 64 Hosting Expression = Subquery scalar-subquery#52, [id=#53] * ColumnarToRow (142) +- CometHashAggregate (141) - +- CometColumnarExchange (140) + +- CometExchange (140) +- CometHashAggregate (139) +- CometUnion (138) :- CometProject (126) @@ -793,9 +793,9 @@ Input [2]: [quantity#126, list_price#127] Keys: [] Functions [1]: [partial_avg((cast(quantity#126 as decimal(10,0)) * list_price#127))] -(140) CometColumnarExchange +(140) CometExchange Input [2]: [sum#143, count#144] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (141) CometHashAggregate Input [2]: [sum#143, count#144] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt index 01803108f..760d0963e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q14a/simplified.txt @@ -3,11 +3,11 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] - CometColumnarExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 + CometExchange [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] #1 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales] CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum_sales,number_sales,sum,isEmpty,sum,sum(sales),sum(number_salesL)] - CometColumnarExchange [channel,i_brand_id,i_class_id,i_category_id] #2 + CometExchange [channel,i_brand_id,i_class_id,i_category_id] #2 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sales,number_sales] CometUnion [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] @@ -16,7 +16,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [average_sales,sum,count,avg((cast(quantity as decimal(10,0)) * list_price))] - CometColumnarExchange #16 + CometExchange #16 CometHashAggregate [sum,count,quantity,list_price] CometUnion [quantity,list_price] CometProject [ss_quantity,ss_list_price] [quantity,list_price] @@ -45,7 +45,7 @@ WholeStageCodegen (1) ReusedSubquery [d_date_sk] #4 ReusedExchange [d_date_sk] #18 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ss_quantity as decimal(10,0)) * ss_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #3 + CometExchange [i_brand_id,i_class_id,i_category_id] #3 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ss_quantity,ss_list_price] CometProject [ss_quantity,ss_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ss_quantity,ss_list_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -70,7 +70,7 @@ WholeStageCodegen (1) CometBroadcastExchange [brand_id,class_id,category_id] #6 CometBroadcastHashJoin [brand_id,class_id,category_id,i_brand_id,i_class_id,i_category_id] CometHashAggregate [brand_id,class_id,category_id] - CometColumnarExchange [brand_id,class_id,category_id] #7 + CometExchange [brand_id,class_id,category_id] #7 CometHashAggregate [brand_id,class_id,category_id] CometProject [i_brand_id,i_class_id,i_category_id] [brand_id,class_id,category_id] CometBroadcastHashJoin [ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -128,7 +128,7 @@ WholeStageCodegen (1) CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(cs_quantity as decimal(10,0)) * cs_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #19 + CometExchange [i_brand_id,i_class_id,i_category_id] #19 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,cs_quantity,cs_list_price] CometProject [cs_quantity,cs_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [cs_quantity,cs_list_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -144,7 +144,7 @@ WholeStageCodegen (1) CometFilter [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales] ReusedSubquery [average_sales] #3 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sales,number_sales,sum,isEmpty,count,sum((cast(ws_quantity as decimal(10,0)) * ws_list_price)),count(1)] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id] #20 + CometExchange [i_brand_id,i_class_id,i_category_id] #20 CometHashAggregate [i_brand_id,i_class_id,i_category_id,sum,isEmpty,count,ws_quantity,ws_list_price] CometProject [ws_quantity,ws_list_price,i_brand_id,i_class_id,i_category_id] CometBroadcastHashJoin [ws_quantity,ws_list_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,d_date_sk] @@ -158,22 +158,22 @@ WholeStageCodegen (1) ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id] #14 ReusedExchange [d_date_sk] #15 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometColumnarExchange [channel,i_brand_id,i_class_id] #21 + CometExchange [channel,i_brand_id,i_class_id] #21 CometHashAggregate [channel,i_brand_id,i_class_id,sum,isEmpty,sum,sum_sales,number_sales] CometHashAggregate [channel,i_brand_id,i_class_id,sum_sales,number_sales,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometColumnarExchange [channel,i_brand_id] #22 + CometExchange [channel,i_brand_id] #22 CometHashAggregate [channel,i_brand_id,sum,isEmpty,sum,sum_sales,number_sales] CometHashAggregate [channel,i_brand_id,sum_sales,number_sales,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometColumnarExchange [channel] #23 + CometExchange [channel] #23 CometHashAggregate [channel,sum,isEmpty,sum,sum_sales,number_sales] CometHashAggregate [channel,sum_sales,number_sales,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 CometHashAggregate [channel,i_brand_id,i_class_id,i_category_id,sum(sum_sales),sum(number_sales),sum,isEmpty,sum,sum(sum_sales),sum(number_salesL)] - CometColumnarExchange #24 + CometExchange #24 CometHashAggregate [sum,isEmpty,sum,sum_sales,number_sales] CometHashAggregate [sum_sales,number_sales,channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum,sum(sales),sum(number_salesL)] ReusedExchange [channel,i_brand_id,i_class_id,i_category_id,sum,isEmpty,sum] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt index dab5b3d55..9990a2dfc 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (142) +- CometUnion (141) :- CometHashAggregate (38) - : +- CometColumnarExchange (37) + : +- CometExchange (37) : +- CometHashAggregate (36) : +- CometProject (35) : +- CometBroadcastHashJoin (34) @@ -41,7 +41,7 @@ : +- CometFilter (32) : +- CometScan parquet spark_catalog.default.item (31) :- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometProject (60) : +- CometBroadcastHashJoin (59) @@ -66,7 +66,7 @@ : : +- ReusedExchange (55) : +- ReusedExchange (58) :- CometHashAggregate (89) - : +- CometColumnarExchange (88) + : +- CometExchange (88) : +- CometHashAggregate (87) : +- CometProject (86) : +- CometBroadcastHashJoin (85) @@ -92,7 +92,7 @@ : : +- ReusedExchange (81) : +- ReusedExchange (84) :- CometHashAggregate (115) - : +- CometColumnarExchange (114) + : +- CometExchange (114) : +- CometHashAggregate (113) : +- CometProject (112) : +- CometBroadcastHashJoin (111) @@ -118,7 +118,7 @@ : : +- ReusedExchange (107) : +- ReusedExchange (110) +- CometHashAggregate (140) - +- CometColumnarExchange (139) + +- CometExchange (139) +- CometHashAggregate (138) +- CometProject (137) +- CometBroadcastHashJoin (136) @@ -317,9 +317,9 @@ Input [11]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, agg1#29, ag Keys [4]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22] Functions [7]: [partial_avg(agg1#29), partial_avg(agg2#30), partial_avg(agg3#31), partial_avg(agg4#32), partial_avg(agg5#33), partial_avg(agg6#34), partial_avg(agg7#35)] -(37) CometColumnarExchange +(37) CometExchange Input [18]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] -Arguments: hashpartitioning(i_item_id#28, ca_country#24, ca_state#23, ca_county#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#28, ca_country#24, ca_state#23, ca_county#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (38) CometHashAggregate Input [18]: [i_item_id#28, ca_country#24, ca_state#23, ca_county#22, sum#36, count#37, sum#38, count#39, sum#40, count#41, sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] @@ -427,9 +427,9 @@ Input [10]: [i_item_id#28, ca_country#24, ca_state#23, agg1#29, agg2#30, agg3#31 Keys [3]: [i_item_id#28, ca_country#24, ca_state#23] Functions [7]: [partial_avg(agg1#29), partial_avg(agg2#30), partial_avg(agg3#31), partial_avg(agg4#32), partial_avg(agg5#33), partial_avg(agg6#34), partial_avg(agg7#35)] -(62) CometColumnarExchange +(62) CometExchange Input [17]: [i_item_id#28, ca_country#24, ca_state#23, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62, sum#63, count#64] -Arguments: hashpartitioning(i_item_id#28, ca_country#24, ca_state#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#28, ca_country#24, ca_state#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (63) CometHashAggregate Input [17]: [i_item_id#28, ca_country#24, ca_state#23, sum#51, count#52, sum#53, count#54, sum#55, count#56, sum#57, count#58, sum#59, count#60, sum#61, count#62, sum#63, count#64] @@ -541,9 +541,9 @@ Input [9]: [i_item_id#28, ca_country#24, agg1#29, agg2#30, agg3#31, agg4#32, agg Keys [2]: [i_item_id#28, ca_country#24] Functions [7]: [partial_avg(agg1#29), partial_avg(agg2#30), partial_avg(agg3#31), partial_avg(agg4#32), partial_avg(agg5#33), partial_avg(agg6#34), partial_avg(agg7#35)] -(88) CometColumnarExchange +(88) CometExchange Input [16]: [i_item_id#28, ca_country#24, sum#66, count#67, sum#68, count#69, sum#70, count#71, sum#72, count#73, sum#74, count#75, sum#76, count#77, sum#78, count#79] -Arguments: hashpartitioning(i_item_id#28, ca_country#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_item_id#28, ca_country#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (89) CometHashAggregate Input [16]: [i_item_id#28, ca_country#24, sum#66, count#67, sum#68, count#69, sum#70, count#71, sum#72, count#73, sum#74, count#75, sum#76, count#77, sum#78, count#79] @@ -655,9 +655,9 @@ Input [8]: [i_item_id#28, agg1#29, agg2#30, agg3#31, agg4#32, agg5#33, agg6#34, Keys [1]: [i_item_id#28] Functions [7]: [partial_avg(agg1#29), partial_avg(agg2#30), partial_avg(agg3#31), partial_avg(agg4#32), partial_avg(agg5#33), partial_avg(agg6#34), partial_avg(agg7#35)] -(114) CometColumnarExchange +(114) CometExchange Input [15]: [i_item_id#28, sum#81, count#82, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94] -Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_item_id#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (115) CometHashAggregate Input [15]: [i_item_id#28, sum#81, count#82, sum#83, count#84, sum#85, count#86, sum#87, count#88, sum#89, count#90, sum#91, count#92, sum#93, count#94] @@ -765,9 +765,9 @@ Input [7]: [agg1#29, agg2#30, agg3#31, agg4#32, agg5#33, agg6#34, agg7#35] Keys: [] Functions [7]: [partial_avg(agg1#29), partial_avg(agg2#30), partial_avg(agg3#31), partial_avg(agg4#32), partial_avg(agg5#33), partial_avg(agg6#34), partial_avg(agg7#35)] -(139) CometColumnarExchange +(139) CometExchange Input [14]: [sum#96, count#97, sum#98, count#99, sum#100, count#101, sum#102, count#103, sum#104, count#105, sum#106, count#107, sum#108, count#109] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (140) CometHashAggregate Input [14]: [sum#96, count#97, sum#98, count#99, sum#100, count#101, sum#102, count#103, sum#104, count#105, sum#106, count#107, sum#108, count#109] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt index aef60ba9b..add76afad 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q18a/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometUnion [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometColumnarExchange [i_item_id,ca_country,ca_state,ca_county] #1 + CometExchange [i_item_id,ca_country,ca_state,ca_county] #1 CometHashAggregate [i_item_id,ca_country,ca_state,ca_county,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,ca_county,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_county,ca_state,ca_country,i_item_sk,i_item_id] @@ -50,7 +50,7 @@ WholeStageCodegen (1) CometFilter [i_item_sk,i_item_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometColumnarExchange [i_item_id,ca_country,ca_state] #9 + CometExchange [i_item_id,ca_country,ca_state] #9 CometHashAggregate [i_item_id,ca_country,ca_state,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,ca_state,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_state,ca_country,i_item_sk,i_item_id] @@ -76,7 +76,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk] #7 ReusedExchange [i_item_sk,i_item_id] #8 CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometColumnarExchange [i_item_id,ca_country] #11 + CometExchange [i_item_id,ca_country] #11 CometHashAggregate [i_item_id,ca_country,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,ca_country,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,ca_country,i_item_sk,i_item_id] @@ -103,7 +103,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk] #7 ReusedExchange [i_item_sk,i_item_id] #8 CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometColumnarExchange [i_item_id] #13 + CometExchange [i_item_id] #13 CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [i_item_id,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk,i_item_id] @@ -130,7 +130,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk] #7 ReusedExchange [i_item_sk,i_item_id] #8 CometHashAggregate [i_item_id,ca_country,ca_state,county,agg1,agg2,agg3,agg4,agg5,agg6,agg7,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(agg2),avg(agg3),avg(agg4),avg(agg5),avg(agg6),avg(agg7)] - CometColumnarExchange #15 + CometExchange #15 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometProject [cs_quantity,cs_list_price,cs_coupon_amt,cs_sales_price,cs_net_profit,c_birth_year,cd_dep_count] [agg1,agg2,agg3,agg4,agg5,agg6,agg7] CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_list_price,cs_sales_price,cs_coupon_amt,cs_net_profit,cd_dep_count,c_birth_year,i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt index e2ecb0b16..3de2b0499 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/explain.txt @@ -4,9 +4,9 @@ TakeOrderedAndProject (22) +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -92,18 +92,18 @@ Input [6]: [cs_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(cs_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(cs_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt index 7b17378dc..c4d5e48ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q20/simplified.txt @@ -7,9 +7,9 @@ TakeOrderedAndProject [i_category,i_class,i_item_id,i_item_desc,revenueratio,i_c ColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometColumnarExchange [i_class] #1 + CometExchange [i_class] #1 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(cs_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #2 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,cs_ext_sales_price] CometProject [cs_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [cs_ext_sales_price,cs_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt index 1975ac486..2810779ed 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/explain.txt @@ -1,30 +1,28 @@ == Physical Plan == -TakeOrderedAndProject (26) -+- * HashAggregate (25) - +- * ColumnarToRow (24) - +- CometColumnarExchange (23) - +- RowToColumnar (22) - +- * HashAggregate (21) - +- * Expand (20) - +- * Project (19) - +- * BroadcastNestedLoopJoin Inner BuildRight (18) - :- * ColumnarToRow (14) - : +- CometProject (13) - : +- CometBroadcastHashJoin (12) - : :- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.inventory (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- CometBroadcastExchange (11) - : +- CometFilter (10) - : +- CometScan parquet spark_catalog.default.item (9) - +- BroadcastExchange (17) - +- * ColumnarToRow (16) - +- CometScan parquet spark_catalog.default.warehouse (15) +TakeOrderedAndProject (24) ++- * HashAggregate (23) + +- Exchange (22) + +- * HashAggregate (21) + +- * Expand (20) + +- * Project (19) + +- * BroadcastNestedLoopJoin Inner BuildRight (18) + :- * ColumnarToRow (14) + : +- CometProject (13) + : +- CometBroadcastHashJoin (12) + : :- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.inventory (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- CometBroadcastExchange (11) + : +- CometFilter (10) + : +- CometScan parquet spark_catalog.default.item (9) + +- BroadcastExchange (17) + +- * ColumnarToRow (16) + +- CometScan parquet spark_catalog.default.warehouse (15) (1) Scan parquet spark_catalog.default.inventory @@ -126,56 +124,50 @@ Functions [1]: [partial_avg(inv_quantity_on_hand#2)] Aggregate Attributes [2]: [sum#17, count#18] Results [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] -(22) RowToColumnar +(22) Exchange Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] +Arguments: hashpartitioning(i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, [plan_id=2] -(23) CometColumnarExchange -Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] -Arguments: hashpartitioning(i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] - -(24) ColumnarToRow [codegen id : 3] -Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] - -(25) HashAggregate [codegen id : 3] +(23) HashAggregate [codegen id : 3] Input [7]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16, sum#19, count#20] Keys [5]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, spark_grouping_id#16] Functions [1]: [avg(inv_quantity_on_hand#2)] Aggregate Attributes [1]: [avg(inv_quantity_on_hand#2)#21] Results [5]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, avg(inv_quantity_on_hand#2)#21 AS qoh#22] -(26) TakeOrderedAndProject +(24) TakeOrderedAndProject Input [5]: [i_product_name#12, i_brand#13, i_class#14, i_category#15, qoh#22] Arguments: 100, [qoh#22 ASC NULLS FIRST, i_product_name#12 ASC NULLS FIRST, i_brand#13 ASC NULLS FIRST, i_class#14 ASC NULLS FIRST, i_category#15 ASC NULLS FIRST], [i_product_name#12, i_brand#13, i_class#14, i_category#15, qoh#22] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = inv_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (31) -+- * ColumnarToRow (30) - +- CometProject (29) - +- CometFilter (28) - +- CometScan parquet spark_catalog.default.date_dim (27) +BroadcastExchange (29) ++- * ColumnarToRow (28) + +- CometProject (27) + +- CometFilter (26) + +- CometScan parquet spark_catalog.default.date_dim (25) -(27) Scan parquet spark_catalog.default.date_dim +(25) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1200), LessThanOrEqual(d_month_seq,1211), IsNotNull(d_date_sk)] ReadSchema: struct -(28) CometFilter +(26) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1200)) AND (d_month_seq#6 <= 1211)) AND isnotnull(d_date_sk#5)) -(29) CometProject +(27) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(30) ColumnarToRow [codegen id : 1] +(28) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(31) BroadcastExchange +(29) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=3] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt index f5b281ac6..0e864ab70 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22/simplified.txt @@ -1,41 +1,39 @@ TakeOrderedAndProject [qoh,i_product_name,i_brand,i_class,i_category] WholeStageCodegen (3) HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,sum,count] [avg(inv_quantity_on_hand),qoh,sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] - Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] - BroadcastNestedLoopJoin - ColumnarToRow - InputAdapter - CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] - CometProject [inv_item_sk,inv_quantity_on_hand] - CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] - CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #2 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #3 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 - CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] - CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometScan parquet spark_catalog.default.warehouse + InputAdapter + Exchange [i_product_name,i_brand,i_class,i_category,spark_grouping_id] #1 + WholeStageCodegen (2) + HashAggregate [i_product_name,i_brand,i_class,i_category,spark_grouping_id,inv_quantity_on_hand] [sum,count,sum,count] + Expand [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + Project [inv_quantity_on_hand,i_product_name,i_brand,i_class,i_category] + BroadcastNestedLoopJoin + ColumnarToRow + InputAdapter + CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,i_item_sk,i_brand,i_class,i_category,i_product_name] + CometProject [inv_item_sk,inv_quantity_on_hand] + CometBroadcastHashJoin [inv_item_sk,inv_quantity_on_hand,inv_date_sk,d_date_sk] + CometFilter [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_quantity_on_hand,inv_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #2 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #3 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [i_item_sk,i_brand,i_class,i_category,i_product_name] #4 + CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] + CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometScan parquet spark_catalog.default.warehouse diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt index c4342ab4e..852eedea6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/explain.txt @@ -5,7 +5,7 @@ :- CometHashAggregate (23) : +- CometHashAggregate (22) : +- CometHashAggregate (21) - : +- CometColumnarExchange (20) + : +- CometExchange (20) : +- CometHashAggregate (19) : +- CometProject (18) : +- CometBroadcastHashJoin (17) @@ -26,22 +26,22 @@ : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.warehouse (14) :- CometHashAggregate (28) - : +- CometColumnarExchange (27) + : +- CometExchange (27) : +- CometHashAggregate (26) : +- CometHashAggregate (25) : +- ReusedExchange (24) :- CometHashAggregate (33) - : +- CometColumnarExchange (32) + : +- CometExchange (32) : +- CometHashAggregate (31) : +- CometHashAggregate (30) : +- ReusedExchange (29) :- CometHashAggregate (38) - : +- CometColumnarExchange (37) + : +- CometExchange (37) : +- CometHashAggregate (36) : +- CometHashAggregate (35) : +- ReusedExchange (34) +- CometHashAggregate (43) - +- CometColumnarExchange (42) + +- CometExchange (42) +- CometHashAggregate (41) +- CometHashAggregate (40) +- ReusedExchange (39) @@ -140,9 +140,9 @@ Input [5]: [inv_quantity_on_hand#3, i_brand#9, i_class#10, i_category#11, i_prod Keys [4]: [i_product_name#12, i_brand#9, i_class#10, i_category#11] Functions [1]: [partial_avg(inv_quantity_on_hand#3)] -(20) CometColumnarExchange +(20) CometExchange Input [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#14, count#15] -Arguments: hashpartitioning(i_product_name#12, i_brand#9, i_class#10, i_category#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_product_name#12, i_brand#9, i_class#10, i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [6]: [i_product_name#12, i_brand#9, i_class#10, i_category#11, sum#14, count#15] @@ -172,9 +172,9 @@ Input [4]: [i_product_name#12, i_brand#9, i_class#10, qoh#16] Keys [3]: [i_product_name#12, i_brand#9, i_class#10] Functions [1]: [partial_avg(qoh#16)] -(27) CometColumnarExchange +(27) CometExchange Input [5]: [i_product_name#12, i_brand#9, i_class#10, sum#19, count#20] -Arguments: hashpartitioning(i_product_name#12, i_brand#9, i_class#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_product_name#12, i_brand#9, i_class#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (28) CometHashAggregate Input [5]: [i_product_name#12, i_brand#9, i_class#10, sum#19, count#20] @@ -194,9 +194,9 @@ Input [3]: [i_product_name#12, i_brand#9, qoh#16] Keys [2]: [i_product_name#12, i_brand#9] Functions [1]: [partial_avg(qoh#16)] -(32) CometColumnarExchange +(32) CometExchange Input [4]: [i_product_name#12, i_brand#9, sum#21, count#22] -Arguments: hashpartitioning(i_product_name#12, i_brand#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_product_name#12, i_brand#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (33) CometHashAggregate Input [4]: [i_product_name#12, i_brand#9, sum#21, count#22] @@ -216,9 +216,9 @@ Input [2]: [i_product_name#12, qoh#16] Keys [1]: [i_product_name#12] Functions [1]: [partial_avg(qoh#16)] -(37) CometColumnarExchange +(37) CometExchange Input [3]: [i_product_name#12, sum#23, count#24] -Arguments: hashpartitioning(i_product_name#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_product_name#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (38) CometHashAggregate Input [3]: [i_product_name#12, sum#23, count#24] @@ -238,9 +238,9 @@ Input [1]: [qoh#16] Keys: [] Functions [1]: [partial_avg(qoh#16)] -(42) CometColumnarExchange +(42) CometExchange Input [2]: [sum#25, count#26] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (43) CometHashAggregate Input [2]: [sum#25, count#26] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt index d5a409c3e..63eda27a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q22a/simplified.txt @@ -6,7 +6,7 @@ WholeStageCodegen (1) CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count,qoh] CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(inv_quantity_on_hand)] - CometColumnarExchange [i_product_name,i_brand,i_class,i_category] #1 + CometExchange [i_product_name,i_brand,i_class,i_category] #1 CometHashAggregate [i_product_name,i_brand,i_class,i_category,sum,count,inv_quantity_on_hand] CometProject [inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name] CometBroadcastHashJoin [inv_warehouse_sk,inv_quantity_on_hand,i_brand,i_class,i_category,i_product_name,w_warehouse_sk] @@ -35,22 +35,22 @@ WholeStageCodegen (1) CometFilter [w_warehouse_sk] CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk] CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometColumnarExchange [i_product_name,i_brand,i_class] #6 + CometExchange [i_product_name,i_brand,i_class] #6 CometHashAggregate [i_product_name,i_brand,i_class,sum,count,qoh] CometHashAggregate [i_product_name,i_brand,i_class,qoh,i_category,sum,count,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometColumnarExchange [i_product_name,i_brand] #7 + CometExchange [i_product_name,i_brand] #7 CometHashAggregate [i_product_name,i_brand,sum,count,qoh] CometHashAggregate [i_product_name,i_brand,qoh,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometColumnarExchange [i_product_name] #8 + CometExchange [i_product_name] #8 CometHashAggregate [i_product_name,sum,count,qoh] CometHashAggregate [i_product_name,qoh,i_brand,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 CometHashAggregate [i_product_name,i_brand,i_class,i_category,qoh,sum,count,avg(qoh)] - CometColumnarExchange #9 + CometExchange #9 CometHashAggregate [sum,count,qoh] CometHashAggregate [qoh,i_product_name,i_brand,i_class,i_category,sum,count,avg(inv_quantity_on_hand)] ReusedExchange [i_product_name,i_brand,i_class,i_category,sum,count] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt index 490d5cd18..085e68ac9 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/explain.txt @@ -1,54 +1,50 @@ == Physical Plan == -* ColumnarToRow (50) -+- CometSort (49) - +- CometColumnarExchange (48) - +- RowToColumnar (47) - +- * Filter (46) - +- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * HashAggregate (40) - +- * ColumnarToRow (39) - +- CometColumnarExchange (38) - +- RowToColumnar (37) - +- * HashAggregate (36) - +- * Project (35) - +- * BroadcastHashJoin Inner BuildRight (34) - :- * ColumnarToRow (29) - : +- CometProject (28) - : +- CometBroadcastHashJoin (27) - : :- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometProject (18) - : : : +- CometBroadcastHashJoin (17) - : : : :- CometProject (12) - : : : : +- CometSortMergeJoin (11) - : : : : :- CometSort (5) - : : : : : +- CometColumnarExchange (4) - : : : : : +- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometSort (10) - : : : : +- CometColumnarExchange (9) - : : : : +- CometProject (8) - : : : : +- CometFilter (7) - : : : : +- CometScan parquet spark_catalog.default.store_returns (6) - : : : +- CometBroadcastExchange (16) - : : : +- CometProject (15) - : : : +- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store (13) - : : +- CometBroadcastExchange (21) - : : +- CometFilter (20) - : : +- CometScan parquet spark_catalog.default.item (19) - : +- CometBroadcastExchange (26) - : +- CometFilter (25) - : +- CometScan parquet spark_catalog.default.customer (24) - +- BroadcastExchange (33) - +- * ColumnarToRow (32) - +- CometFilter (31) - +- CometScan parquet spark_catalog.default.customer_address (30) +* ColumnarToRow (46) ++- CometSort (45) + +- CometColumnarExchange (44) + +- RowToColumnar (43) + +- * Filter (42) + +- * HashAggregate (41) + +- Exchange (40) + +- * HashAggregate (39) + +- * HashAggregate (38) + +- Exchange (37) + +- * HashAggregate (36) + +- * Project (35) + +- * BroadcastHashJoin Inner BuildRight (34) + :- * ColumnarToRow (29) + : +- CometProject (28) + : +- CometBroadcastHashJoin (27) + : :- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometProject (18) + : : : +- CometBroadcastHashJoin (17) + : : : :- CometProject (12) + : : : : +- CometSortMergeJoin (11) + : : : : :- CometSort (5) + : : : : : +- CometExchange (4) + : : : : : +- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometSort (10) + : : : : +- CometExchange (9) + : : : : +- CometProject (8) + : : : : +- CometFilter (7) + : : : : +- CometScan parquet spark_catalog.default.store_returns (6) + : : : +- CometBroadcastExchange (16) + : : : +- CometProject (15) + : : : +- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store (13) + : : +- CometBroadcastExchange (21) + : : +- CometFilter (20) + : : +- CometScan parquet spark_catalog.default.item (19) + : +- CometBroadcastExchange (26) + : +- CometFilter (25) + : +- CometScan parquet spark_catalog.default.customer (24) + +- BroadcastExchange (33) + +- * ColumnarToRow (32) + +- CometFilter (31) + +- CometScan parquet spark_catalog.default.customer_address (30) (1) Scan parquet spark_catalog.default.store_sales @@ -66,9 +62,9 @@ Condition : (((isnotnull(ss_ticket_number#4) AND isnotnull(ss_item_sk#1)) AND is Input [6]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, ss_sold_date_sk#6] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(4) CometColumnarExchange +(4) CometExchange Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (5) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] @@ -89,9 +85,9 @@ Condition : (isnotnull(sr_ticket_number#8) AND isnotnull(sr_item_sk#7)) Input [3]: [sr_item_sk#7, sr_ticket_number#8, sr_returned_date_sk#9] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_item_sk#7, sr_ticket_number#8] -(9) CometColumnarExchange +(9) CometExchange Input [2]: [sr_item_sk#7, sr_ticket_number#8] -Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(sr_ticket_number#8, sr_item_sk#7, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (10) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] @@ -220,230 +216,202 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#30] Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -(37) RowToColumnar +(37) Exchange Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(38) CometColumnarExchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(39) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] - -(40) HashAggregate [codegen id : 3] +(38) HashAggregate [codegen id : 3] Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#31] Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] -(41) HashAggregate [codegen id : 3] +(39) HashAggregate [codegen id : 3] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, netpaid#33] Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] Functions [1]: [partial_sum(netpaid#33)] Aggregate Attributes [2]: [sum#34, isEmpty#35] Results [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -(42) RowToColumnar -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] - -(43) CometColumnarExchange -Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 4] +(40) Exchange Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(45) HashAggregate [codegen id : 4] +(41) HashAggregate [codegen id : 4] Input [5]: [c_last_name#24, c_first_name#23, s_store_name#11, sum#36, isEmpty#37] Keys [3]: [c_last_name#24, c_first_name#23, s_store_name#11] Functions [1]: [sum(netpaid#33)] Aggregate Attributes [1]: [sum(netpaid#33)#38] Results [4]: [c_last_name#24, c_first_name#23, s_store_name#11, sum(netpaid#33)#38 AS paid#39] -(46) Filter [codegen id : 4] +(42) Filter [codegen id : 4] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] Condition : (isnotnull(paid#39) AND (cast(paid#39 as decimal(33,8)) > cast(Subquery scalar-subquery#40, [id=#41] as decimal(33,8)))) -(47) RowToColumnar +(43) RowToColumnar Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] -(48) CometColumnarExchange +(44) CometColumnarExchange Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] Arguments: rangepartitioning(c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] -(49) CometSort +(45) CometSort Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] Arguments: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39], [c_last_name#24 ASC NULLS FIRST, c_first_name#23 ASC NULLS FIRST, s_store_name#11 ASC NULLS FIRST] -(50) ColumnarToRow [codegen id : 5] +(46) ColumnarToRow [codegen id : 5] Input [4]: [c_last_name#24, c_first_name#23, s_store_name#11, paid#39] ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery scalar-subquery#40, [id=#41] -* HashAggregate (81) -+- * ColumnarToRow (80) - +- CometColumnarExchange (79) - +- RowToColumnar (78) - +- * HashAggregate (77) - +- * HashAggregate (76) - +- * ColumnarToRow (75) - +- CometColumnarExchange (74) - +- RowToColumnar (73) - +- * HashAggregate (72) - +- * Project (71) - +- * BroadcastHashJoin Inner BuildRight (70) - :- * ColumnarToRow (68) - : +- CometProject (67) - : +- CometBroadcastHashJoin (66) - : :- CometProject (64) - : : +- CometBroadcastHashJoin (63) - : : :- CometProject (59) - : : : +- CometBroadcastHashJoin (58) - : : : :- CometProject (56) - : : : : +- CometSortMergeJoin (55) - : : : : :- CometSort (52) - : : : : : +- ReusedExchange (51) - : : : : +- CometSort (54) - : : : : +- ReusedExchange (53) - : : : +- ReusedExchange (57) - : : +- CometBroadcastExchange (62) - : : +- CometFilter (61) - : : +- CometScan parquet spark_catalog.default.item (60) - : +- ReusedExchange (65) - +- ReusedExchange (69) - - -(51) ReusedExchange [Reuses operator id: 4] +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery scalar-subquery#40, [id=#41] +* HashAggregate (73) ++- Exchange (72) + +- * HashAggregate (71) + +- * HashAggregate (70) + +- Exchange (69) + +- * HashAggregate (68) + +- * Project (67) + +- * BroadcastHashJoin Inner BuildRight (66) + :- * ColumnarToRow (64) + : +- CometProject (63) + : +- CometBroadcastHashJoin (62) + : :- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (55) + : : : +- CometBroadcastHashJoin (54) + : : : :- CometProject (52) + : : : : +- CometSortMergeJoin (51) + : : : : :- CometSort (48) + : : : : : +- ReusedExchange (47) + : : : : +- CometSort (50) + : : : : +- ReusedExchange (49) + : : : +- ReusedExchange (53) + : : +- CometBroadcastExchange (58) + : : +- CometFilter (57) + : : +- CometScan parquet spark_catalog.default.item (56) + : +- ReusedExchange (61) + +- ReusedExchange (65) + + +(47) ReusedExchange [Reuses operator id: 4] Output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] -(52) CometSort +(48) CometSort Input [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5], [ss_ticket_number#4 ASC NULLS FIRST, ss_item_sk#1 ASC NULLS FIRST] -(53) ReusedExchange [Reuses operator id: 9] +(49) ReusedExchange [Reuses operator id: 9] Output [2]: [sr_item_sk#7, sr_ticket_number#8] -(54) CometSort +(50) CometSort Input [2]: [sr_item_sk#7, sr_ticket_number#8] Arguments: [sr_item_sk#7, sr_ticket_number#8], [sr_ticket_number#8 ASC NULLS FIRST, sr_item_sk#7 ASC NULLS FIRST] -(55) CometSortMergeJoin +(51) CometSortMergeJoin Left output [5]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5] Right output [2]: [sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_ticket_number#4, ss_item_sk#1], [sr_ticket_number#8, sr_item_sk#7], Inner -(56) CometProject +(52) CometProject Input [7]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_ticket_number#4, ss_net_paid#5, sr_item_sk#7, sr_ticket_number#8] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5], [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] -(57) ReusedExchange [Reuses operator id: 16] +(53) ReusedExchange [Reuses operator id: 16] Output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] -(58) CometBroadcastHashJoin +(54) CometBroadcastHashJoin Left output [4]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5] Right output [4]: [s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] Arguments: [ss_store_sk#3], [s_store_sk#10], Inner, BuildRight -(59) CometProject +(55) CometProject Input [8]: [ss_item_sk#1, ss_customer_sk#2, ss_store_sk#3, ss_net_paid#5, s_store_sk#10, s_store_name#11, s_state#13, s_zip#14] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14], [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] -(60) Scan parquet spark_catalog.default.item +(56) Scan parquet spark_catalog.default.item Output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Batched: true Location [not included in comparison]/{warehouse_dir}/item] PushedFilters: [IsNotNull(i_item_sk)] ReadSchema: struct -(61) CometFilter +(57) CometFilter Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Condition : isnotnull(i_item_sk#15) -(62) CometBroadcastExchange +(58) CometBroadcastExchange Input [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Arguments: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -(63) CometBroadcastHashJoin +(59) CometBroadcastHashJoin Left output [6]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14] Right output [6]: [i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Arguments: [ss_item_sk#1], [i_item_sk#15], Inner, BuildRight -(64) CometProject +(60) CometProject Input [12]: [ss_item_sk#1, ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_item_sk#15, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Arguments: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20], [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] -(65) ReusedExchange [Reuses operator id: 26] +(61) ReusedExchange [Reuses operator id: 26] Output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -(66) CometBroadcastHashJoin +(62) CometBroadcastHashJoin Left output [10]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20] Right output [5]: [c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] Arguments: [ss_customer_sk#2], [c_customer_sk#21], Inner, BuildRight -(67) CometProject +(63) CometProject Input [15]: [ss_customer_sk#2, ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_customer_sk#21, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] Arguments: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25], [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -(68) ColumnarToRow [codegen id : 2] +(64) ColumnarToRow [codegen id : 2] Input [13]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25] -(69) ReusedExchange [Reuses operator id: 33] +(65) ReusedExchange [Reuses operator id: 33] Output [4]: [ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -(70) BroadcastHashJoin [codegen id : 2] +(66) BroadcastHashJoin [codegen id : 2] Left keys [3]: [c_current_addr_sk#22, c_birth_country#25, s_zip#14] Right keys [3]: [ca_address_sk#26, upper(ca_country#29), ca_zip#28] Join type: Inner Join condition: None -(71) Project [codegen id : 2] +(67) Project [codegen id : 2] Output [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] Input [17]: [ss_net_paid#5, s_store_name#11, s_state#13, s_zip#14, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_current_addr_sk#22, c_first_name#23, c_last_name#24, c_birth_country#25, ca_address_sk#26, ca_state#27, ca_zip#28, ca_country#29] -(72) HashAggregate [codegen id : 2] +(68) HashAggregate [codegen id : 2] Input [11]: [ss_net_paid#5, s_store_name#11, s_state#13, i_current_price#16, i_size#17, i_color#18, i_units#19, i_manager_id#20, c_first_name#23, c_last_name#24, ca_state#27] Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum#42] Results [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] -(73) RowToColumnar +(69) Exchange Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] +Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(74) CometColumnarExchange -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] -Arguments: hashpartitioning(c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(75) ColumnarToRow [codegen id : 3] -Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] - -(76) HashAggregate [codegen id : 3] +(70) HashAggregate [codegen id : 3] Input [11]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17, sum#43] Keys [10]: [c_last_name#24, c_first_name#23, s_store_name#11, ca_state#27, s_state#13, i_color#18, i_current_price#16, i_manager_id#20, i_units#19, i_size#17] Functions [1]: [sum(UnscaledValue(ss_net_paid#5))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_paid#5))#32] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_paid#5))#32,17,2) AS netpaid#33] -(77) HashAggregate [codegen id : 3] +(71) HashAggregate [codegen id : 3] Input [1]: [netpaid#33] Keys: [] Functions [1]: [partial_avg(netpaid#33)] Aggregate Attributes [2]: [sum#44, count#45] Results [2]: [sum#46, count#47] -(78) RowToColumnar -Input [2]: [sum#46, count#47] - -(79) CometColumnarExchange -Input [2]: [sum#46, count#47] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(80) ColumnarToRow [codegen id : 4] +(72) Exchange Input [2]: [sum#46, count#47] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=8] -(81) HashAggregate [codegen id : 4] +(73) HashAggregate [codegen id : 4] Input [2]: [sum#46, count#47] Keys: [] Functions [1]: [avg(netpaid#33)] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/simplified.txt index 42c283018..1e6a1a1c1 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q24/simplified.txt @@ -9,92 +9,84 @@ WholeStageCodegen (5) Subquery #1 WholeStageCodegen (4) HashAggregate [sum,count] [avg(netpaid),(0.05 * avg(netpaid)),sum,count] - ColumnarToRow - InputAdapter - CometColumnarExchange #10 - RowToColumnar - WholeStageCodegen (3) - HashAggregate [netpaid] [sum,count,sum,count] - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 - CometSort [sr_item_sk,sr_ticket_number] - ReusedExchange [sr_item_sk,sr_ticket_number] #5 - ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - InputAdapter - ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 - HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name] #2 - RowToColumnar + InputAdapter + Exchange #10 WholeStageCodegen (3) - HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [netpaid] [sum,count,sum,count] HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 - RowToColumnar - WholeStageCodegen (2) - HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] - Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] - BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] - ColumnarToRow - InputAdapter - CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] - CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] - CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] - CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #4 - CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] - CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] - CometSort [sr_item_sk,sr_ticket_number] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #5 - CometProject [sr_item_sk,sr_ticket_number] - CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] - CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 - CometProject [s_store_sk,s_store_name,s_state,s_zip] - CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] - CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 - CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] - CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 - CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #11 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + ReusedExchange [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] #4 + CometSort [sr_item_sk,sr_ticket_number] + ReusedExchange [sr_item_sk,sr_ticket_number] #5 + ReusedExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #12 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + ReusedExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + InputAdapter + ReusedExchange [ca_address_sk,ca_state,ca_zip,ca_country] #9 + HashAggregate [c_last_name,c_first_name,s_store_name,sum,isEmpty] [sum(netpaid),paid,sum,isEmpty] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name] #2 + WholeStageCodegen (3) + HashAggregate [c_last_name,c_first_name,s_store_name,netpaid] [sum,isEmpty,sum,isEmpty] + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,sum] [sum(UnscaledValue(ss_net_paid)),netpaid,sum] + InputAdapter + Exchange [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size] #3 + WholeStageCodegen (2) + HashAggregate [c_last_name,c_first_name,s_store_name,ca_state,s_state,i_color,i_current_price,i_manager_id,i_units,i_size,ss_net_paid] [sum,sum] + Project [ss_net_paid,s_store_name,s_state,i_current_price,i_size,i_color,i_units,i_manager_id,c_first_name,c_last_name,ca_state] + BroadcastHashJoin [c_current_addr_sk,c_birth_country,s_zip,ca_address_sk,ca_country,ca_zip] + ColumnarToRow + InputAdapter + CometProject [ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometBroadcastHashJoin [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id,c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometProject [ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip,i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometProject [ss_item_sk,ss_customer_sk,ss_net_paid,s_store_name,s_state,s_zip] + CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid,s_store_sk,s_store_name,s_state,s_zip] + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_net_paid] + CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,sr_item_sk,sr_ticket_number] + CometSort [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometExchange [ss_ticket_number,ss_item_sk] #4 + CometProject [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid] + CometFilter [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_customer_sk,ss_store_sk,ss_ticket_number,ss_net_paid,ss_sold_date_sk] + CometSort [sr_item_sk,sr_ticket_number] + CometExchange [sr_ticket_number,sr_item_sk] #5 + CometProject [sr_item_sk,sr_ticket_number] + CometFilter [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_returned_date_sk] + CometBroadcastExchange [s_store_sk,s_store_name,s_state,s_zip] #6 + CometProject [s_store_sk,s_store_name,s_state,s_zip] + CometFilter [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_name,s_market_id,s_state,s_zip] + CometBroadcastExchange [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] #7 + CometFilter [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price,i_size,i_color,i_units,i_manager_id] + CometBroadcastExchange [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] #8 + CometFilter [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_addr_sk,c_first_name,c_last_name,c_birth_country] + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (1) + ColumnarToRow InputAdapter - BroadcastExchange #9 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] + CometFilter [ca_address_sk,ca_state,ca_zip,ca_country] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state,ca_zip,ca_country] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt index 8d5df655c..fa6fbee0f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (68) +- CometUnion (67) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometProject (24) : +- CometBroadcastHashJoin (23) @@ -30,7 +30,7 @@ : +- CometFilter (21) : +- CometScan parquet spark_catalog.default.item (20) :- CometHashAggregate (47) - : +- CometColumnarExchange (46) + : +- CometExchange (46) : +- CometHashAggregate (45) : +- CometProject (44) : +- CometBroadcastHashJoin (43) @@ -50,7 +50,7 @@ : : +- CometScan parquet spark_catalog.default.store (36) : +- ReusedExchange (42) +- CometHashAggregate (66) - +- CometColumnarExchange (65) + +- CometExchange (65) +- CometHashAggregate (64) +- CometProject (63) +- CometBroadcastHashJoin (62) @@ -191,9 +191,9 @@ Input [6]: [i_item_id#19, s_state#17, agg1#20, agg2#21, agg3#22, agg4#23] Keys [2]: [i_item_id#19, s_state#17] Functions [4]: [partial_avg(agg1#20), partial_avg(UnscaledValue(agg2#21)), partial_avg(UnscaledValue(agg3#22)), partial_avg(UnscaledValue(agg4#23))] -(26) CometColumnarExchange +(26) CometExchange Input [10]: [i_item_id#19, s_state#17, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] -Arguments: hashpartitioning(i_item_id#19, s_state#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#19, s_state#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (27) CometHashAggregate Input [10]: [i_item_id#19, s_state#17, sum#24, count#25, sum#26, count#27, sum#28, count#29, sum#30, count#31] @@ -281,9 +281,9 @@ Input [5]: [i_item_id#19, agg1#20, agg2#21, agg3#22, agg4#23] Keys [1]: [i_item_id#19] Functions [4]: [partial_avg(agg1#20), partial_avg(UnscaledValue(agg2#21)), partial_avg(UnscaledValue(agg3#22)), partial_avg(UnscaledValue(agg4#23))] -(46) CometColumnarExchange +(46) CometExchange Input [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40] -Arguments: hashpartitioning(i_item_id#19, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_item_id#19, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (47) CometHashAggregate Input [9]: [i_item_id#19, sum#33, count#34, sum#35, count#36, sum#37, count#38, sum#39, count#40] @@ -367,9 +367,9 @@ Input [4]: [agg1#20, agg2#21, agg3#22, agg4#23] Keys: [] Functions [4]: [partial_avg(agg1#20), partial_avg(UnscaledValue(agg2#21)), partial_avg(UnscaledValue(agg3#22)), partial_avg(UnscaledValue(agg4#23))] -(65) CometColumnarExchange +(65) CometExchange Input [8]: [sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (66) CometHashAggregate Input [8]: [sum#42, count#43, sum#44, count#45, sum#46, count#47, sum#48, count#49] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt index 43f9cbc6e..ed169acda 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q27a/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] CometUnion [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4] CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometColumnarExchange [i_item_id,s_state] #1 + CometExchange [i_item_id,s_state] #1 CometHashAggregate [i_item_id,s_state,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,s_state,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,s_state,i_item_sk,i_item_id] @@ -39,7 +39,7 @@ WholeStageCodegen (1) CometFilter [i_item_sk,i_item_id] CometScan parquet spark_catalog.default.item [i_item_sk,i_item_id] CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometColumnarExchange [i_item_id] #7 + CometExchange [i_item_id] #7 CometHashAggregate [i_item_id,sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [i_item_id,agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk,i_item_id] @@ -60,7 +60,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.store [s_store_sk,s_state] ReusedExchange [i_item_sk,i_item_id] #6 CometHashAggregate [i_item_id,s_state,g_state,agg1,agg2,agg3,agg4,sum,count,sum,count,sum,count,sum,count,avg(agg1),avg(UnscaledValue(agg2)),avg(UnscaledValue(agg3)),avg(UnscaledValue(agg4))] - CometColumnarExchange #9 + CometExchange #9 CometHashAggregate [sum,count,sum,count,sum,count,sum,count,agg1,agg2,agg3,agg4] CometProject [ss_quantity,ss_list_price,ss_coupon_amt,ss_sales_price] [agg1,agg2,agg3,agg4] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_list_price,ss_sales_price,ss_coupon_amt,i_item_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt index d978de885..a8800a70f 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/explain.txt @@ -6,7 +6,7 @@ +- CometBroadcastHashJoin (28) :- CometFilter (24) : +- CometHashAggregate (23) - : +- CometColumnarExchange (22) + : +- CometExchange (22) : +- CometHashAggregate (21) : +- CometProject (20) : +- CometBroadcastHashJoin (19) @@ -134,9 +134,9 @@ Input [2]: [ss_customer_sk#1, ss_ticket_number#4] Keys [2]: [ss_ticket_number#4, ss_customer_sk#1] Functions [1]: [partial_count(1)] -(22) CometColumnarExchange +(22) CometExchange Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] -Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_ticket_number#4, ss_customer_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (23) CometHashAggregate Input [3]: [ss_ticket_number#4, ss_customer_sk#1, count#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt index fafc612d6..7da6076f5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q34/simplified.txt @@ -7,7 +7,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [ss_ticket_number,ss_customer_sk,cnt,c_customer_sk,c_salutation,c_first_name,c_last_name,c_preferred_cust_flag] CometFilter [ss_ticket_number,ss_customer_sk,cnt] CometHashAggregate [ss_ticket_number,ss_customer_sk,cnt,count,count(1)] - CometColumnarExchange [ss_ticket_number,ss_customer_sk] #2 + CometExchange [ss_ticket_number,ss_customer_sk] #2 CometHashAggregate [ss_ticket_number,ss_customer_sk,count] CometProject [ss_customer_sk,ss_ticket_number] CometBroadcastHashJoin [ss_customer_sk,ss_hdemo_sk,ss_ticket_number,hd_demo_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt index 721036bfb..790000085 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/explain.txt @@ -1,50 +1,48 @@ == Physical Plan == -TakeOrderedAndProject (46) -+- * HashAggregate (45) - +- * ColumnarToRow (44) - +- CometColumnarExchange (43) - +- RowToColumnar (42) - +- * HashAggregate (41) - +- * Project (40) - +- * BroadcastHashJoin Inner BuildRight (39) - :- * Project (34) - : +- * BroadcastHashJoin Inner BuildRight (33) - : :- * Project (28) - : : +- * Filter (27) - : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) - : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) - : : : :- * ColumnarToRow (12) - : : : : +- CometBroadcastHashJoin (11) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.customer (1) - : : : : +- CometBroadcastExchange (10) - : : : : +- CometProject (9) - : : : : +- CometBroadcastHashJoin (8) - : : : : :- CometScan parquet spark_catalog.default.store_sales (3) - : : : : +- CometBroadcastExchange (7) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.date_dim (4) - : : : +- BroadcastExchange (18) - : : : +- * ColumnarToRow (17) - : : : +- CometProject (16) - : : : +- CometBroadcastHashJoin (15) - : : : :- CometScan parquet spark_catalog.default.web_sales (13) - : : : +- ReusedExchange (14) - : : +- BroadcastExchange (25) - : : +- * ColumnarToRow (24) - : : +- CometProject (23) - : : +- CometBroadcastHashJoin (22) - : : :- CometScan parquet spark_catalog.default.catalog_sales (20) - : : +- ReusedExchange (21) - : +- BroadcastExchange (32) - : +- * ColumnarToRow (31) - : +- CometFilter (30) - : +- CometScan parquet spark_catalog.default.customer_address (29) - +- BroadcastExchange (38) - +- * ColumnarToRow (37) - +- CometFilter (36) - +- CometScan parquet spark_catalog.default.customer_demographics (35) +TakeOrderedAndProject (44) ++- * HashAggregate (43) + +- Exchange (42) + +- * HashAggregate (41) + +- * Project (40) + +- * BroadcastHashJoin Inner BuildRight (39) + :- * Project (34) + : +- * BroadcastHashJoin Inner BuildRight (33) + : :- * Project (28) + : : +- * Filter (27) + : : +- * BroadcastHashJoin ExistenceJoin(exists#1) BuildRight (26) + : : :- * BroadcastHashJoin ExistenceJoin(exists#2) BuildRight (19) + : : : :- * ColumnarToRow (12) + : : : : +- CometBroadcastHashJoin (11) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.customer (1) + : : : : +- CometBroadcastExchange (10) + : : : : +- CometProject (9) + : : : : +- CometBroadcastHashJoin (8) + : : : : :- CometScan parquet spark_catalog.default.store_sales (3) + : : : : +- CometBroadcastExchange (7) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.date_dim (4) + : : : +- BroadcastExchange (18) + : : : +- * ColumnarToRow (17) + : : : +- CometProject (16) + : : : +- CometBroadcastHashJoin (15) + : : : :- CometScan parquet spark_catalog.default.web_sales (13) + : : : +- ReusedExchange (14) + : : +- BroadcastExchange (25) + : : +- * ColumnarToRow (24) + : : +- CometProject (23) + : : +- CometBroadcastHashJoin (22) + : : :- CometScan parquet spark_catalog.default.catalog_sales (20) + : : +- ReusedExchange (21) + : +- BroadcastExchange (32) + : +- * ColumnarToRow (31) + : +- CometFilter (30) + : +- CometScan parquet spark_catalog.default.customer_address (29) + +- BroadcastExchange (38) + +- * ColumnarToRow (37) + +- CometFilter (36) + +- CometScan parquet spark_catalog.default.customer_demographics (35) (1) Scan parquet spark_catalog.default.customer @@ -240,56 +238,50 @@ Functions [10]: [partial_count(1), partial_avg(cd_dep_count#25), partial_max(cd_ Aggregate Attributes [13]: [count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] Results [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] -(42) RowToColumnar +(42) Exchange Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] +Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(43) CometColumnarExchange -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] -Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(44) ColumnarToRow [codegen id : 6] -Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] - -(45) HashAggregate [codegen id : 6] +(43) HashAggregate [codegen id : 6] Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#41, sum#42, count#43, max#44, sum#45, sum#46, count#47, max#48, sum#49, sum#50, count#51, max#52, sum#53] Keys [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [10]: [count(1), avg(cd_dep_count#25), max(cd_dep_count#25), sum(cd_dep_count#25), avg(cd_dep_employed_count#26), max(cd_dep_employed_count#26), sum(cd_dep_employed_count#26), avg(cd_dep_college_count#27), max(cd_dep_college_count#27), sum(cd_dep_college_count#27)] Aggregate Attributes [10]: [count(1)#54, avg(cd_dep_count#25)#55, max(cd_dep_count#25)#56, sum(cd_dep_count#25)#57, avg(cd_dep_employed_count#26)#58, max(cd_dep_employed_count#26)#59, sum(cd_dep_employed_count#26)#60, avg(cd_dep_college_count#27)#61, max(cd_dep_college_count#27)#62, sum(cd_dep_college_count#27)#63] Results [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, count(1)#54 AS cnt1#64, avg(cd_dep_count#25)#55 AS avg(cd_dep_count)#65, max(cd_dep_count#25)#56 AS max(cd_dep_count)#66, sum(cd_dep_count#25)#57 AS sum(cd_dep_count)#67, cd_dep_employed_count#26, count(1)#54 AS cnt2#68, avg(cd_dep_employed_count#26)#58 AS avg(cd_dep_employed_count)#69, max(cd_dep_employed_count#26)#59 AS max(cd_dep_employed_count)#70, sum(cd_dep_employed_count#26)#60 AS sum(cd_dep_employed_count)#71, cd_dep_college_count#27, count(1)#54 AS cnt3#72, avg(cd_dep_college_count#27)#61 AS avg(cd_dep_college_count)#73, max(cd_dep_college_count#27)#62 AS max(cd_dep_college_count)#74, sum(cd_dep_college_count#27)#63 AS sum(cd_dep_college_count)#75] -(46) TakeOrderedAndProject +(44) TakeOrderedAndProject Input [18]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] Arguments: 100, [ca_state#21 ASC NULLS FIRST, cd_gender#23 ASC NULLS FIRST, cd_marital_status#24 ASC NULLS FIRST, cd_dep_count#25 ASC NULLS FIRST, cd_dep_employed_count#26 ASC NULLS FIRST, cd_dep_college_count#27 ASC NULLS FIRST], [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cnt1#64, avg(cd_dep_count)#65, max(cd_dep_count)#66, sum(cd_dep_count)#67, cd_dep_employed_count#26, cnt2#68, avg(cd_dep_employed_count)#69, max(cd_dep_employed_count)#70, sum(cd_dep_employed_count)#71, cd_dep_college_count#27, cnt3#72, avg(cd_dep_college_count)#73, max(cd_dep_college_count)#74, sum(cd_dep_college_count)#75] ===== Subqueries ===== Subquery:1 Hosting operator id = 3 Hosting Expression = ss_sold_date_sk#7 IN dynamicpruning#8 -BroadcastExchange (51) -+- * ColumnarToRow (50) - +- CometProject (49) - +- CometFilter (48) - +- CometScan parquet spark_catalog.default.date_dim (47) +BroadcastExchange (49) ++- * ColumnarToRow (48) + +- CometProject (47) + +- CometFilter (46) + +- CometScan parquet spark_catalog.default.date_dim (45) -(47) Scan parquet spark_catalog.default.date_dim +(45) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#9, d_year#10, d_qoy#11] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_qoy), EqualTo(d_year,2002), LessThan(d_qoy,4), IsNotNull(d_date_sk)] ReadSchema: struct -(48) CometFilter +(46) CometFilter Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Condition : ((((isnotnull(d_year#10) AND isnotnull(d_qoy#11)) AND (d_year#10 = 2002)) AND (d_qoy#11 < 4)) AND isnotnull(d_date_sk#9)) -(49) CometProject +(47) CometProject Input [3]: [d_date_sk#9, d_year#10, d_qoy#11] Arguments: [d_date_sk#9], [d_date_sk#9] -(50) ColumnarToRow [codegen id : 1] +(48) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#9] -(51) BroadcastExchange +(49) BroadcastExchange Input [1]: [d_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt index 221648657..0b55e23ab 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35/simplified.txt @@ -1,72 +1,70 @@ TakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] WholeStageCodegen (6) HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] [count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] - Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] - Project [c_current_cdemo_sk,ca_state] - BroadcastHashJoin [c_current_addr_sk,ca_address_sk] - Project [c_current_cdemo_sk,c_current_addr_sk] - Filter [exists,exists] - BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] - BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + InputAdapter + Exchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + WholeStageCodegen (5) + HashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] [count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] + Project [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + BroadcastHashJoin [c_current_cdemo_sk,cd_demo_sk] + Project [c_current_cdemo_sk,ca_state] + BroadcastHashJoin [c_current_addr_sk,ca_address_sk] + Project [c_current_cdemo_sk,c_current_addr_sk] + Filter [exists,exists] + BroadcastHashJoin [c_customer_sk,cs_ship_customer_sk] + BroadcastHashJoin [c_customer_sk,ws_bill_customer_sk] + ColumnarToRow + InputAdapter + CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] + CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] + CometBroadcastExchange [ss_customer_sk] #2 + CometProject [ss_customer_sk] + CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + CometBroadcastExchange [d_date_sk] #4 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_qoy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] + InputAdapter + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [ws_bill_customer_sk] + CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometBroadcastHashJoin [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk,ss_customer_sk] - CometFilter [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometScan parquet spark_catalog.default.customer [c_customer_sk,c_current_cdemo_sk,c_current_addr_sk] - CometBroadcastExchange [ss_customer_sk] #2 - CometProject [ss_customer_sk] - CometBroadcastHashJoin [ss_customer_sk,ss_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_customer_sk,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - CometBroadcastExchange [d_date_sk] #4 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_qoy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_qoy] - InputAdapter - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [ws_bill_customer_sk] - CometBroadcastHashJoin [ws_bill_customer_sk,ws_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_bill_customer_sk,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometProject [cs_ship_customer_sk] - CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #4 - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (3) - ColumnarToRow - InputAdapter - CometFilter [ca_address_sk,ca_state] - CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + CometProject [cs_ship_customer_sk] + CometBroadcastHashJoin [cs_ship_customer_sk,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_customer_sk,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #4 InputAdapter - BroadcastExchange #8 - WholeStageCodegen (4) + BroadcastExchange #7 + WholeStageCodegen (3) ColumnarToRow InputAdapter - CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometFilter [ca_address_sk,ca_state] + CometScan parquet spark_catalog.default.customer_address [ca_address_sk,ca_state] + InputAdapter + BroadcastExchange #8 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometFilter [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt index 476dab310..9f321ebda 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/explain.txt @@ -2,7 +2,7 @@ * ColumnarToRow (38) +- CometTakeOrderedAndProject (37) +- CometHashAggregate (36) - +- CometColumnarExchange (35) + +- CometExchange (35) +- CometHashAggregate (34) +- CometProject (33) +- CometBroadcastHashJoin (32) @@ -202,9 +202,9 @@ Input [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd Keys [6]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27] Functions [10]: [partial_count(1), partial_avg(cd_dep_count#25), partial_max(cd_dep_count#25), partial_sum(cd_dep_count#25), partial_avg(cd_dep_employed_count#26), partial_max(cd_dep_employed_count#26), partial_sum(cd_dep_employed_count#26), partial_avg(cd_dep_college_count#27), partial_max(cd_dep_college_count#27), partial_sum(cd_dep_college_count#27)] -(35) CometColumnarExchange +(35) CometExchange Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] -Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (36) CometHashAggregate Input [19]: [ca_state#21, cd_gender#23, cd_marital_status#24, cd_dep_count#25, cd_dep_employed_count#26, cd_dep_college_count#27, count#28, sum#29, count#30, max#31, sum#32, sum#33, count#34, max#35, sum#36, sum#37, count#38, max#39, sum#40] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt index 25f72e37b..5ebac3026 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q35a/simplified.txt @@ -3,7 +3,7 @@ WholeStageCodegen (1) InputAdapter CometTakeOrderedAndProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cnt1,avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),cd_dep_employed_count,cnt2,avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),cd_dep_college_count,cnt3,avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count),count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum,count(1),avg(cd_dep_count),max(cd_dep_count),sum(cd_dep_count),avg(cd_dep_employed_count),max(cd_dep_employed_count),sum(cd_dep_employed_count),avg(cd_dep_college_count),max(cd_dep_college_count),sum(cd_dep_college_count)] - CometColumnarExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 + CometExchange [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] #1 CometHashAggregate [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count,count,sum,count,max,sum,sum,count,max,sum,sum,count,max,sum] CometProject [ca_state,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] CometBroadcastHashJoin [c_current_cdemo_sk,ca_state,cd_demo_sk,cd_gender,cd_marital_status,cd_dep_count,cd_dep_employed_count,cd_dep_college_count] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt index 149347c76..92441bde0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/explain.txt @@ -4,13 +4,13 @@ TakeOrderedAndProject (42) +- Window (40) +- * ColumnarToRow (39) +- CometSort (38) - +- CometColumnarExchange (37) + +- CometExchange (37) +- CometHashAggregate (36) - +- CometColumnarExchange (35) + +- CometExchange (35) +- CometHashAggregate (34) +- CometUnion (33) :- CometHashAggregate (22) - : +- CometColumnarExchange (21) + : +- CometExchange (21) : +- CometHashAggregate (20) : +- CometProject (19) : +- CometBroadcastHashJoin (18) @@ -32,12 +32,12 @@ TakeOrderedAndProject (42) : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.store (14) :- CometHashAggregate (27) - : +- CometColumnarExchange (26) + : +- CometExchange (26) : +- CometHashAggregate (25) : +- CometHashAggregate (24) : +- ReusedExchange (23) +- CometHashAggregate (32) - +- CometColumnarExchange (31) + +- CometExchange (31) +- CometHashAggregate (30) +- CometHashAggregate (29) +- ReusedExchange (28) @@ -140,9 +140,9 @@ Input [4]: [ss_ext_sales_price#3, ss_net_profit#4, i_class#10, i_category#11] Keys [2]: [i_category#11, i_class#10] Functions [2]: [partial_sum(UnscaledValue(ss_net_profit#4)), partial_sum(UnscaledValue(ss_ext_sales_price#3))] -(21) CometColumnarExchange +(21) CometExchange Input [4]: [i_category#11, i_class#10, sum#14, sum#15] -Arguments: hashpartitioning(i_category#11, i_class#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#11, i_class#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (22) CometHashAggregate Input [4]: [i_category#11, i_class#10, sum#14, sum#15] @@ -162,9 +162,9 @@ Input [3]: [ss_net_profit#18, ss_ext_sales_price#19, i_category#11] Keys [1]: [i_category#11] Functions [2]: [partial_sum(ss_net_profit#18), partial_sum(ss_ext_sales_price#19)] -(26) CometColumnarExchange +(26) CometExchange Input [5]: [i_category#11, sum#20, isEmpty#21, sum#22, isEmpty#23] -Arguments: hashpartitioning(i_category#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate Input [5]: [i_category#11, sum#20, isEmpty#21, sum#22, isEmpty#23] @@ -184,9 +184,9 @@ Input [2]: [ss_net_profit#18, ss_ext_sales_price#19] Keys: [] Functions [2]: [partial_sum(ss_net_profit#18), partial_sum(ss_ext_sales_price#19)] -(31) CometColumnarExchange +(31) CometExchange Input [4]: [sum#26, isEmpty#27, sum#28, isEmpty#29] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometHashAggregate Input [4]: [sum#26, isEmpty#27, sum#28, isEmpty#29] @@ -203,18 +203,18 @@ Input [6]: [gross_margin#30, i_category#11, i_class#10, t_category#31, t_class#3 Keys [6]: [gross_margin#30, i_category#11, i_class#10, t_category#31, t_class#32, lochierarchy#33] Functions: [] -(35) CometColumnarExchange +(35) CometExchange Input [6]: [gross_margin#30, i_category#11, i_class#10, t_category#31, t_class#32, lochierarchy#33] -Arguments: hashpartitioning(gross_margin#30, i_category#11, i_class#10, t_category#31, t_class#32, lochierarchy#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(gross_margin#30, i_category#11, i_class#10, t_category#31, t_class#32, lochierarchy#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (36) CometHashAggregate Input [6]: [gross_margin#30, i_category#11, i_class#10, t_category#31, t_class#32, lochierarchy#33] Keys [6]: [gross_margin#30, i_category#11, i_class#10, t_category#31, t_class#32, lochierarchy#33] Functions: [] -(37) CometColumnarExchange +(37) CometExchange Input [5]: [gross_margin#30, i_category#11, i_class#10, lochierarchy#33, _w0#45] -Arguments: hashpartitioning(lochierarchy#33, _w0#45, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(lochierarchy#33, _w0#45, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (38) CometSort Input [5]: [gross_margin#30, i_category#11, i_class#10, lochierarchy#33, _w0#45] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt index 9fc6f31f9..6eb469dae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q36a/simplified.txt @@ -7,13 +7,13 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i ColumnarToRow InputAdapter CometSort [gross_margin,i_category,i_class,lochierarchy,_w0] - CometColumnarExchange [lochierarchy,_w0] #1 + CometExchange [lochierarchy,_w0] #1 CometHashAggregate [gross_margin,i_category,i_class,lochierarchy,_w0,t_category,t_class] - CometColumnarExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 + CometExchange [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] #2 CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] CometUnion [gross_margin,i_category,i_class,t_category,t_class,lochierarchy] CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_category,i_class] #3 + CometExchange [i_category,i_class] #3 CometHashAggregate [i_category,i_class,sum,sum,ss_net_profit,ss_ext_sales_price] CometProject [ss_ext_sales_price,ss_net_profit,i_class,i_category] CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,i_class,i_category,s_store_sk] @@ -43,12 +43,12 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,gross_margin,i CometFilter [s_store_sk,s_state] CometScan parquet spark_catalog.default.store [s_store_sk,s_state] CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty,sum(ss_net_profit),sum(ss_ext_sales_price)] - CometColumnarExchange [i_category] #8 + CometExchange [i_category] #8 CometHashAggregate [i_category,sum,isEmpty,sum,isEmpty,ss_net_profit,ss_ext_sales_price] CometHashAggregate [ss_net_profit,ss_ext_sales_price,i_category,i_class,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 CometHashAggregate [gross_margin,i_category,i_class,t_category,t_class,lochierarchy,sum,isEmpty,sum,isEmpty,sum(ss_net_profit),sum(ss_ext_sales_price)] - CometColumnarExchange #9 + CometExchange #9 CometHashAggregate [sum,isEmpty,sum,isEmpty,ss_net_profit,ss_ext_sales_price] CometHashAggregate [ss_net_profit,ss_ext_sales_price,i_category,i_class,sum,sum,sum(UnscaledValue(ss_net_profit)),sum(UnscaledValue(ss_ext_sales_price))] ReusedExchange [i_category,i_class,sum,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt index b6ab28625..3bc9bb6e0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/explain.txt @@ -11,9 +11,9 @@ TakeOrderedAndProject (45) : : +- Window (24) : : +- * ColumnarToRow (23) : : +- CometSort (22) - : : +- CometColumnarExchange (21) + : : +- CometExchange (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -135,18 +135,18 @@ Input [7]: [i_brand#2, i_category#3, ss_sales_price#6, d_year#10, d_moy#11, s_st Keys [6]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#6))] -(19) CometColumnarExchange +(19) CometExchange Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#15] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (20) CometHashAggregate Input [7]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum#15] Keys [6]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(ss_sales_price#6))] -(21) CometColumnarExchange +(21) CometExchange Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#16, _w0#17] -Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, s_store_name#13, s_company_name#14, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (22) CometSort Input [8]: [i_category#3, i_brand#2, s_store_name#13, s_company_name#14, d_year#10, d_moy#11, sum_sales#16, _w0#17] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt index 3c44ef74c..c16c278d5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q47/simplified.txt @@ -16,9 +16,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_moy,i_category,d_year,psum, ColumnarToRow InputAdapter CometSort [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name] #1 + CometExchange [i_category,i_brand,s_store_name,s_company_name] #1 CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 + CometExchange [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy] #2 CometHashAggregate [i_category,i_brand,s_store_name,s_company_name,d_year,d_moy,sum,ss_sales_price] CometProject [i_brand,i_category,ss_sales_price,d_year,d_moy,s_store_name,s_company_name] CometBroadcastHashJoin [i_brand,i_category,ss_store_sk,ss_sales_price,d_year,d_moy,s_store_sk,s_store_name,s_company_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt index ed68082f6..7d9198ed5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/explain.txt @@ -1,83 +1,81 @@ == Physical Plan == -* ColumnarToRow (79) -+- CometTakeOrderedAndProject (78) - +- CometHashAggregate (77) - +- CometColumnarExchange (76) - +- RowToColumnar (75) - +- * HashAggregate (74) - +- Union (73) - :- * Project (26) - : +- * Filter (25) - : +- Window (24) - : +- * Sort (23) - : +- Window (22) - : +- * ColumnarToRow (21) - : +- CometSort (20) - : +- CometColumnarExchange (19) - : +- CometHashAggregate (18) - : +- CometColumnarExchange (17) - : +- CometHashAggregate (16) - : +- CometProject (15) - : +- CometBroadcastHashJoin (14) - : :- CometProject (9) - : : +- CometBroadcastHashJoin (8) - : : :- CometBroadcastExchange (4) - : : : +- CometProject (3) - : : : +- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : +- CometProject (7) - : : +- CometFilter (6) - : : +- CometScan parquet spark_catalog.default.web_returns (5) - : +- CometBroadcastExchange (13) - : +- CometProject (12) - : +- CometFilter (11) - : +- CometScan parquet spark_catalog.default.date_dim (10) - :- * Project (49) - : +- * Filter (48) - : +- Window (47) - : +- * Sort (46) - : +- Window (45) - : +- * ColumnarToRow (44) - : +- CometSort (43) - : +- CometColumnarExchange (42) - : +- CometHashAggregate (41) - : +- CometColumnarExchange (40) - : +- CometHashAggregate (39) - : +- CometProject (38) - : +- CometBroadcastHashJoin (37) - : :- CometProject (35) - : : +- CometBroadcastHashJoin (34) - : : :- CometBroadcastExchange (30) - : : : +- CometProject (29) - : : : +- CometFilter (28) - : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) - : : +- CometProject (33) - : : +- CometFilter (32) - : : +- CometScan parquet spark_catalog.default.catalog_returns (31) - : +- ReusedExchange (36) - +- * Project (72) - +- * Filter (71) - +- Window (70) - +- * Sort (69) - +- Window (68) - +- * ColumnarToRow (67) - +- CometSort (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometColumnarExchange (63) - +- CometHashAggregate (62) - +- CometProject (61) - +- CometBroadcastHashJoin (60) - :- CometProject (58) - : +- CometBroadcastHashJoin (57) - : :- CometBroadcastExchange (53) - : : +- CometProject (52) - : : +- CometFilter (51) - : : +- CometScan parquet spark_catalog.default.store_sales (50) - : +- CometProject (56) - : +- CometFilter (55) - : +- CometScan parquet spark_catalog.default.store_returns (54) - +- ReusedExchange (59) +TakeOrderedAndProject (77) ++- * HashAggregate (76) + +- Exchange (75) + +- * HashAggregate (74) + +- Union (73) + :- * Project (26) + : +- * Filter (25) + : +- Window (24) + : +- * Sort (23) + : +- Window (22) + : +- * ColumnarToRow (21) + : +- CometSort (20) + : +- CometExchange (19) + : +- CometHashAggregate (18) + : +- CometExchange (17) + : +- CometHashAggregate (16) + : +- CometProject (15) + : +- CometBroadcastHashJoin (14) + : :- CometProject (9) + : : +- CometBroadcastHashJoin (8) + : : :- CometBroadcastExchange (4) + : : : +- CometProject (3) + : : : +- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : +- CometProject (7) + : : +- CometFilter (6) + : : +- CometScan parquet spark_catalog.default.web_returns (5) + : +- CometBroadcastExchange (13) + : +- CometProject (12) + : +- CometFilter (11) + : +- CometScan parquet spark_catalog.default.date_dim (10) + :- * Project (49) + : +- * Filter (48) + : +- Window (47) + : +- * Sort (46) + : +- Window (45) + : +- * ColumnarToRow (44) + : +- CometSort (43) + : +- CometExchange (42) + : +- CometHashAggregate (41) + : +- CometExchange (40) + : +- CometHashAggregate (39) + : +- CometProject (38) + : +- CometBroadcastHashJoin (37) + : :- CometProject (35) + : : +- CometBroadcastHashJoin (34) + : : :- CometBroadcastExchange (30) + : : : +- CometProject (29) + : : : +- CometFilter (28) + : : : +- CometScan parquet spark_catalog.default.catalog_sales (27) + : : +- CometProject (33) + : : +- CometFilter (32) + : : +- CometScan parquet spark_catalog.default.catalog_returns (31) + : +- ReusedExchange (36) + +- * Project (72) + +- * Filter (71) + +- Window (70) + +- * Sort (69) + +- Window (68) + +- * ColumnarToRow (67) + +- CometSort (66) + +- CometExchange (65) + +- CometHashAggregate (64) + +- CometExchange (63) + +- CometHashAggregate (62) + +- CometProject (61) + +- CometBroadcastHashJoin (60) + :- CometProject (58) + : +- CometBroadcastHashJoin (57) + : :- CometBroadcastExchange (53) + : : +- CometProject (52) + : : +- CometFilter (51) + : : +- CometScan parquet spark_catalog.default.store_sales (50) + : +- CometProject (56) + : +- CometFilter (55) + : +- CometScan parquet spark_catalog.default.store_returns (54) + +- ReusedExchange (59) (1) Scan parquet spark_catalog.default.web_sales @@ -157,18 +155,18 @@ Input [5]: [ws_item_sk#1, ws_quantity#3, ws_net_paid#4, wr_return_quantity#10, w Keys [1]: [ws_item_sk#1] Functions [4]: [partial_sum(coalesce(wr_return_quantity#10, 0)), partial_sum(coalesce(ws_quantity#3, 0)), partial_sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (18) CometHashAggregate Input [7]: [ws_item_sk#1, sum#16, sum#17, sum#18, isEmpty#19, sum#20, isEmpty#21] Keys [1]: [ws_item_sk#1] Functions [4]: [sum(coalesce(wr_return_quantity#10, 0)), sum(coalesce(ws_quantity#3, 0)), sum(coalesce(cast(wr_return_amt#11 as decimal(12,2)), 0.00)), sum(coalesce(cast(ws_net_paid#4 as decimal(12,2)), 0.00))] -(19) CometColumnarExchange +(19) CometExchange Input [3]: [item#22, return_ratio#23, currency_ratio#24] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort Input [3]: [item#22, return_ratio#23, currency_ratio#24] @@ -258,18 +256,18 @@ Input [5]: [cs_item_sk#28, cs_quantity#30, cs_net_paid#31, cr_return_quantity#37 Keys [1]: [cs_item_sk#28] Functions [4]: [partial_sum(coalesce(cr_return_quantity#37, 0)), partial_sum(coalesce(cs_quantity#30, 0)), partial_sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] -(40) CometColumnarExchange +(40) CometExchange Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] -Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cs_item_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (41) CometHashAggregate Input [7]: [cs_item_sk#28, sum#41, sum#42, sum#43, isEmpty#44, sum#45, isEmpty#46] Keys [1]: [cs_item_sk#28] Functions [4]: [sum(coalesce(cr_return_quantity#37, 0)), sum(coalesce(cs_quantity#30, 0)), sum(coalesce(cast(cr_return_amount#38 as decimal(12,2)), 0.00)), sum(coalesce(cast(cs_net_paid#31 as decimal(12,2)), 0.00))] -(42) CometColumnarExchange +(42) CometExchange Input [3]: [item#47, return_ratio#48, currency_ratio#49] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (43) CometSort Input [3]: [item#47, return_ratio#48, currency_ratio#49] @@ -359,18 +357,18 @@ Input [5]: [ss_item_sk#53, ss_quantity#55, ss_net_paid#56, sr_return_quantity#62 Keys [1]: [ss_item_sk#53] Functions [4]: [partial_sum(coalesce(sr_return_quantity#62, 0)), partial_sum(coalesce(ss_quantity#55, 0)), partial_sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), partial_sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] -(63) CometColumnarExchange +(63) CometExchange Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] -Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ss_item_sk#53, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (64) CometHashAggregate Input [7]: [ss_item_sk#53, sum#66, sum#67, sum#68, isEmpty#69, sum#70, isEmpty#71] Keys [1]: [ss_item_sk#53] Functions [4]: [sum(coalesce(sr_return_quantity#62, 0)), sum(coalesce(ss_quantity#55, 0)), sum(coalesce(cast(sr_return_amt#63 as decimal(12,2)), 0.00)), sum(coalesce(cast(ss_net_paid#56 as decimal(12,2)), 0.00))] -(65) CometColumnarExchange +(65) CometExchange Input [3]: [item#72, return_ratio#73, currency_ratio#74] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (66) CometSort Input [3]: [item#72, return_ratio#73, currency_ratio#74] @@ -408,54 +406,50 @@ Functions: [] Aggregate Attributes: [] Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -(75) RowToColumnar +(75) Exchange Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(76) CometColumnarExchange -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: hashpartitioning(channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(77) CometHashAggregate +(76) HashAggregate [codegen id : 11] Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] Keys [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -(78) CometTakeOrderedAndProject -Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#27 ASC NULLS FIRST,return_rank#25 ASC NULLS FIRST,currency_rank#26 ASC NULLS FIRST,item#22 ASC NULLS FIRST], output=[channel#27,item#22,return_ratio#23,return_rank#25,currency_rank#26]), [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26], 100, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST, item#22 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] - -(79) ColumnarToRow [codegen id : 11] +(77) TakeOrderedAndProject Input [5]: [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] +Arguments: 100, [channel#27 ASC NULLS FIRST, return_rank#25 ASC NULLS FIRST, currency_rank#26 ASC NULLS FIRST, item#22 ASC NULLS FIRST], [channel#27, item#22, return_ratio#23, return_rank#25, currency_rank#26] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#6 IN dynamicpruning#7 -BroadcastExchange (84) -+- * ColumnarToRow (83) - +- CometProject (82) - +- CometFilter (81) - +- CometScan parquet spark_catalog.default.date_dim (80) +BroadcastExchange (82) ++- * ColumnarToRow (81) + +- CometProject (80) + +- CometFilter (79) + +- CometScan parquet spark_catalog.default.date_dim (78) -(80) Scan parquet spark_catalog.default.date_dim +(78) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#13, d_year#14, d_moy#15] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), IsNotNull(d_moy), EqualTo(d_year,2001), EqualTo(d_moy,12), IsNotNull(d_date_sk)] ReadSchema: struct -(81) CometFilter +(79) CometFilter Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Condition : ((((isnotnull(d_year#14) AND isnotnull(d_moy#15)) AND (d_year#14 = 2001)) AND (d_moy#15 = 12)) AND isnotnull(d_date_sk#13)) -(82) CometProject +(80) CometProject Input [3]: [d_date_sk#13, d_year#14, d_moy#15] Arguments: [d_date_sk#13], [d_date_sk#13] -(83) ColumnarToRow [codegen id : 1] +(81) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#13] -(84) BroadcastExchange +(82) BroadcastExchange Input [1]: [d_date_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt index 0d02a9c39..d98b2b0a6 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q49/simplified.txt @@ -1,111 +1,109 @@ -WholeStageCodegen (11) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,item,return_ratio,return_rank,currency_rank] - CometHashAggregate [channel,item,return_ratio,return_rank,currency_rank] - CometColumnarExchange [channel,item,return_ratio,return_rank,currency_rank] #1 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [channel,item,return_ratio,return_rank,currency_rank] - InputAdapter - Union - WholeStageCodegen (3) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (2) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #2 - CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [ws_item_sk] #3 - CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] - CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] - CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 - CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] - CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #6 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_year,d_moy] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] - WholeStageCodegen (6) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (5) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #7 - CometHashAggregate [item,return_ratio,currency_ratio,cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [cs_item_sk] #8 - CometHashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] - CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] - CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 - CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] - CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] - ReusedExchange [d_date_sk] #6 - WholeStageCodegen (9) - Project [item,return_ratio,return_rank,currency_rank] - Filter [return_rank,currency_rank] - InputAdapter - Window [currency_ratio] - WholeStageCodegen (8) - Sort [currency_ratio] - InputAdapter - Window [return_ratio] - WholeStageCodegen (7) - ColumnarToRow - InputAdapter - CometSort [item,return_ratio,currency_ratio] - CometColumnarExchange #10 - CometHashAggregate [item,return_ratio,currency_ratio,ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] - CometColumnarExchange [ss_item_sk] #11 - CometHashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] - CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] - CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 - CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] - CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] - ReusedExchange [d_date_sk] #6 +TakeOrderedAndProject [channel,return_rank,currency_rank,item,return_ratio] + WholeStageCodegen (11) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Exchange [channel,item,return_ratio,return_rank,currency_rank] #1 + WholeStageCodegen (10) + HashAggregate [channel,item,return_ratio,return_rank,currency_rank] + InputAdapter + Union + WholeStageCodegen (3) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (2) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #2 + CometHashAggregate [item,return_ratio,currency_ratio,ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(wr_return_quantity, 0)),sum(coalesce(ws_quantity, 0)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ws_net_paid as decimal(12,2)), 0.00))] + CometExchange [ws_item_sk] #3 + CometHashAggregate [ws_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,wr_return_quantity,ws_quantity,wr_return_amt,ws_net_paid] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt,d_date_sk] + CometProject [ws_item_sk,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_return_quantity,wr_return_amt] + CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometBroadcastExchange [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] #4 + CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_sold_date_sk] + CometFilter [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_order_number,ws_quantity,ws_net_paid,ws_net_profit,ws_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] + CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #6 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_year,d_moy] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year,d_moy] + WholeStageCodegen (6) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (5) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #7 + CometHashAggregate [item,return_ratio,currency_ratio,cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(cr_return_quantity, 0)),sum(coalesce(cs_quantity, 0)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum(coalesce(cast(cs_net_paid as decimal(12,2)), 0.00))] + CometExchange [cs_item_sk] #8 + CometHashAggregate [cs_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,cr_return_quantity,cs_quantity,cr_return_amount,cs_net_paid] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount,d_date_sk] + CometProject [cs_item_sk,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_return_quantity,cr_return_amount] + CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometBroadcastExchange [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] #9 + CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_sold_date_sk] + CometFilter [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_quantity,cs_net_paid,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] + CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] + ReusedExchange [d_date_sk] #6 + WholeStageCodegen (9) + Project [item,return_ratio,return_rank,currency_rank] + Filter [return_rank,currency_rank] + InputAdapter + Window [currency_ratio] + WholeStageCodegen (8) + Sort [currency_ratio] + InputAdapter + Window [return_ratio] + WholeStageCodegen (7) + ColumnarToRow + InputAdapter + CometSort [item,return_ratio,currency_ratio] + CometExchange #10 + CometHashAggregate [item,return_ratio,currency_ratio,ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sum(coalesce(sr_return_quantity, 0)),sum(coalesce(ss_quantity, 0)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum(coalesce(cast(ss_net_paid as decimal(12,2)), 0.00))] + CometExchange [ss_item_sk] #11 + CometHashAggregate [ss_item_sk,sum,sum,sum,isEmpty,sum,isEmpty,sr_return_quantity,ss_quantity,sr_return_amt,ss_net_paid] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt,d_date_sk] + CometProject [ss_item_sk,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_return_quantity,sr_return_amt] + CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometBroadcastExchange [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] #12 + CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_sold_date_sk] + CometFilter [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_ticket_number,ss_quantity,ss_net_paid,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] + ReusedExchange [d_date_sk] #6 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt index 20a3f1c52..5d728a1c5 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/explain.txt @@ -1,84 +1,76 @@ == Physical Plan == -TakeOrderedAndProject (80) -+- * Filter (79) - +- * HashAggregate (78) - +- * HashAggregate (77) - +- * Project (76) - +- * BroadcastHashJoin Inner BuildRight (75) - :- Window (68) - : +- * ColumnarToRow (67) - : +- CometSort (66) - : +- CometColumnarExchange (65) - : +- CometProject (64) - : +- CometFilter (63) - : +- CometSortMergeJoin (62) - : :- CometSort (32) - : : +- CometColumnarExchange (31) - : : +- RowToColumnar (30) - : : +- * HashAggregate (29) - : : +- * ColumnarToRow (28) - : : +- CometColumnarExchange (27) - : : +- RowToColumnar (26) - : : +- * HashAggregate (25) - : : +- * Project (24) - : : +- * BroadcastHashJoin Inner BuildRight (23) - : : :- * Project (16) - : : : +- Window (15) - : : : +- * ColumnarToRow (14) - : : : +- CometSort (13) - : : : +- CometColumnarExchange (12) - : : : +- CometHashAggregate (11) - : : : +- CometColumnarExchange (10) - : : : +- CometHashAggregate (9) - : : : +- CometProject (8) - : : : +- CometBroadcastHashJoin (7) - : : : :- CometFilter (2) - : : : : +- CometScan parquet spark_catalog.default.web_sales (1) - : : : +- CometBroadcastExchange (6) - : : : +- CometProject (5) - : : : +- CometFilter (4) - : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : +- BroadcastExchange (22) - : : +- * Project (21) - : : +- Window (20) - : : +- * ColumnarToRow (19) - : : +- CometSort (18) - : : +- ReusedExchange (17) - : +- CometSort (61) - : +- CometColumnarExchange (60) - : +- RowToColumnar (59) - : +- * HashAggregate (58) - : +- * ColumnarToRow (57) - : +- CometColumnarExchange (56) - : +- RowToColumnar (55) - : +- * HashAggregate (54) - : +- * Project (53) - : +- * BroadcastHashJoin Inner BuildRight (52) - : :- * Project (45) - : : +- Window (44) - : : +- * ColumnarToRow (43) - : : +- CometSort (42) - : : +- CometColumnarExchange (41) - : : +- CometHashAggregate (40) - : : +- CometColumnarExchange (39) - : : +- CometHashAggregate (38) - : : +- CometProject (37) - : : +- CometBroadcastHashJoin (36) - : : :- CometFilter (34) - : : : +- CometScan parquet spark_catalog.default.store_sales (33) - : : +- ReusedExchange (35) - : +- BroadcastExchange (51) - : +- * Project (50) - : +- Window (49) - : +- * ColumnarToRow (48) - : +- CometSort (47) - : +- ReusedExchange (46) - +- BroadcastExchange (74) - +- * Project (73) - +- Window (72) - +- * ColumnarToRow (71) - +- CometSort (70) - +- ReusedExchange (69) +TakeOrderedAndProject (72) ++- * Filter (71) + +- * HashAggregate (70) + +- * HashAggregate (69) + +- * Project (68) + +- * BroadcastHashJoin Inner BuildRight (67) + :- Window (61) + : +- * Sort (60) + : +- Exchange (59) + : +- * Project (58) + : +- * Filter (57) + : +- * SortMergeJoin FullOuter (56) + : :- * Sort (29) + : : +- Exchange (28) + : : +- * HashAggregate (27) + : : +- Exchange (26) + : : +- * HashAggregate (25) + : : +- * Project (24) + : : +- * BroadcastHashJoin Inner BuildRight (23) + : : :- * Project (16) + : : : +- Window (15) + : : : +- * ColumnarToRow (14) + : : : +- CometSort (13) + : : : +- CometExchange (12) + : : : +- CometHashAggregate (11) + : : : +- CometExchange (10) + : : : +- CometHashAggregate (9) + : : : +- CometProject (8) + : : : +- CometBroadcastHashJoin (7) + : : : :- CometFilter (2) + : : : : +- CometScan parquet spark_catalog.default.web_sales (1) + : : : +- CometBroadcastExchange (6) + : : : +- CometProject (5) + : : : +- CometFilter (4) + : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : +- BroadcastExchange (22) + : : +- * Project (21) + : : +- Window (20) + : : +- * ColumnarToRow (19) + : : +- CometSort (18) + : : +- ReusedExchange (17) + : +- * Sort (55) + : +- Exchange (54) + : +- * HashAggregate (53) + : +- Exchange (52) + : +- * HashAggregate (51) + : +- * Project (50) + : +- * BroadcastHashJoin Inner BuildRight (49) + : :- * Project (42) + : : +- Window (41) + : : +- * ColumnarToRow (40) + : : +- CometSort (39) + : : +- CometExchange (38) + : : +- CometHashAggregate (37) + : : +- CometExchange (36) + : : +- CometHashAggregate (35) + : : +- CometProject (34) + : : +- CometBroadcastHashJoin (33) + : : :- CometFilter (31) + : : : +- CometScan parquet spark_catalog.default.store_sales (30) + : : +- ReusedExchange (32) + : +- BroadcastExchange (48) + : +- * Project (47) + : +- Window (46) + : +- * ColumnarToRow (45) + : +- CometSort (44) + : +- ReusedExchange (43) + +- BroadcastExchange (66) + +- * Project (65) + +- Window (64) + +- * Sort (63) + +- ReusedExchange (62) (1) Scan parquet spark_catalog.default.web_sales @@ -126,18 +118,18 @@ Input [3]: [ws_item_sk#1, ws_sales_price#2, d_date#6] Keys [2]: [ws_item_sk#1, d_date#6] Functions [1]: [partial_sum(UnscaledValue(ws_sales_price#2))] -(10) CometColumnarExchange +(10) CometExchange Input [3]: [ws_item_sk#1, d_date#6, sum#8] -Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ws_item_sk#1, d_date#6, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (11) CometHashAggregate Input [3]: [ws_item_sk#1, d_date#6, sum#8] Keys [2]: [ws_item_sk#1, d_date#6] Functions [1]: [sum(UnscaledValue(ws_sales_price#2))] -(12) CometColumnarExchange +(12) CometExchange Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] -Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ws_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (13) CometSort Input [4]: [item_sk#9, d_date#6, sumws#10, ws_item_sk#1] @@ -193,35 +185,26 @@ Functions [1]: [partial_sum(sumws#16)] Aggregate Attributes [2]: [sum#17, isEmpty#18] Results [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] -(26) RowToColumnar +(26) Exchange Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] +Arguments: hashpartitioning(item_sk#9, d_date#6, sumws#10, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(27) CometColumnarExchange -Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] -Arguments: hashpartitioning(item_sk#9, d_date#6, sumws#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(28) ColumnarToRow [codegen id : 5] -Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] - -(29) HashAggregate [codegen id : 5] +(27) HashAggregate [codegen id : 5] Input [5]: [item_sk#9, d_date#6, sumws#10, sum#19, isEmpty#20] Keys [3]: [item_sk#9, d_date#6, sumws#10] Functions [1]: [sum(sumws#16)] Aggregate Attributes [1]: [sum(sumws#16)#21] Results [3]: [item_sk#9, d_date#6, sum(sumws#16)#21 AS cume_sales#22] -(30) RowToColumnar +(28) Exchange Input [3]: [item_sk#9, d_date#6, cume_sales#22] +Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(31) CometColumnarExchange +(29) Sort [codegen id : 6] Input [3]: [item_sk#9, d_date#6, cume_sales#22] -Arguments: hashpartitioning(item_sk#9, d_date#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST], false, 0 -(32) CometSort -Input [3]: [item_sk#9, d_date#6, cume_sales#22] -Arguments: [item_sk#9, d_date#6, cume_sales#22], [item_sk#9 ASC NULLS FIRST, d_date#6 ASC NULLS FIRST] - -(33) Scan parquet spark_catalog.default.store_sales +(30) Scan parquet spark_catalog.default.store_sales Output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] Batched: true Location: InMemoryFileIndex [] @@ -229,236 +212,222 @@ PartitionFilters: [isnotnull(ss_sold_date_sk#25), dynamicpruningexpression(ss_so PushedFilters: [IsNotNull(ss_item_sk)] ReadSchema: struct -(34) CometFilter +(31) CometFilter Input [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] Condition : isnotnull(ss_item_sk#23) -(35) ReusedExchange [Reuses operator id: 6] +(32) ReusedExchange [Reuses operator id: 6] Output [2]: [d_date_sk#27, d_date#28] -(36) CometBroadcastHashJoin +(33) CometBroadcastHashJoin Left output [3]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25] Right output [2]: [d_date_sk#27, d_date#28] Arguments: [ss_sold_date_sk#25], [d_date_sk#27], Inner, BuildRight -(37) CometProject +(34) CometProject Input [5]: [ss_item_sk#23, ss_sales_price#24, ss_sold_date_sk#25, d_date_sk#27, d_date#28] Arguments: [ss_item_sk#23, ss_sales_price#24, d_date#28], [ss_item_sk#23, ss_sales_price#24, d_date#28] -(38) CometHashAggregate +(35) CometHashAggregate Input [3]: [ss_item_sk#23, ss_sales_price#24, d_date#28] Keys [2]: [ss_item_sk#23, d_date#28] Functions [1]: [partial_sum(UnscaledValue(ss_sales_price#24))] -(39) CometColumnarExchange +(36) CometExchange Input [3]: [ss_item_sk#23, d_date#28, sum#29] -Arguments: hashpartitioning(ss_item_sk#23, d_date#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ss_item_sk#23, d_date#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] -(40) CometHashAggregate +(37) CometHashAggregate Input [3]: [ss_item_sk#23, d_date#28, sum#29] Keys [2]: [ss_item_sk#23, d_date#28] Functions [1]: [sum(UnscaledValue(ss_sales_price#24))] -(41) CometColumnarExchange +(38) CometExchange Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] -Arguments: hashpartitioning(ss_item_sk#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(ss_item_sk#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] -(42) CometSort +(39) CometSort Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] Arguments: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23], [ss_item_sk#23 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST] -(43) ColumnarToRow [codegen id : 6] +(40) ColumnarToRow [codegen id : 7] Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] -(44) Window +(41) Window Input [4]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23] Arguments: [row_number() windowspecdefinition(ss_item_sk#23, d_date#28 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#32], [ss_item_sk#23], [d_date#28 ASC NULLS FIRST] -(45) Project [codegen id : 9] +(42) Project [codegen id : 10] Output [4]: [item_sk#30, d_date#28, sumss#31, rk#32] Input [5]: [item_sk#30, d_date#28, sumss#31, ss_item_sk#23, rk#32] -(46) ReusedExchange [Reuses operator id: 41] +(43) ReusedExchange [Reuses operator id: 38] Output [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] -(47) CometSort +(44) CometSort Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] Arguments: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34], [ss_item_sk#34 ASC NULLS FIRST, d_date#33 ASC NULLS FIRST] -(48) ColumnarToRow [codegen id : 7] +(45) ColumnarToRow [codegen id : 8] Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] -(49) Window +(46) Window Input [4]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34] Arguments: [row_number() windowspecdefinition(ss_item_sk#34, d_date#33 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#35], [ss_item_sk#34], [d_date#33 ASC NULLS FIRST] -(50) Project [codegen id : 8] +(47) Project [codegen id : 9] Output [3]: [item_sk#30 AS item_sk#36, sumss#31 AS sumss#37, rk#35] Input [5]: [item_sk#30, d_date#33, sumss#31, ss_item_sk#34, rk#35] -(51) BroadcastExchange +(48) BroadcastExchange Input [3]: [item_sk#36, sumss#37, rk#35] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] -(52) BroadcastHashJoin [codegen id : 9] +(49) BroadcastHashJoin [codegen id : 10] Left keys [1]: [item_sk#30] Right keys [1]: [item_sk#36] Join type: Inner Join condition: (rk#32 >= rk#35) -(53) Project [codegen id : 9] +(50) Project [codegen id : 10] Output [4]: [item_sk#30, d_date#28, sumss#31, sumss#37] Input [7]: [item_sk#30, d_date#28, sumss#31, rk#32, item_sk#36, sumss#37, rk#35] -(54) HashAggregate [codegen id : 9] +(51) HashAggregate [codegen id : 10] Input [4]: [item_sk#30, d_date#28, sumss#31, sumss#37] Keys [3]: [item_sk#30, d_date#28, sumss#31] Functions [1]: [partial_sum(sumss#37)] Aggregate Attributes [2]: [sum#38, isEmpty#39] Results [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] -(55) RowToColumnar -Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] - -(56) CometColumnarExchange +(52) Exchange Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] -Arguments: hashpartitioning(item_sk#30, d_date#28, sumss#31, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: hashpartitioning(item_sk#30, d_date#28, sumss#31, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(57) ColumnarToRow [codegen id : 10] -Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] - -(58) HashAggregate [codegen id : 10] +(53) HashAggregate [codegen id : 11] Input [5]: [item_sk#30, d_date#28, sumss#31, sum#40, isEmpty#41] Keys [3]: [item_sk#30, d_date#28, sumss#31] Functions [1]: [sum(sumss#37)] Aggregate Attributes [1]: [sum(sumss#37)#42] Results [3]: [item_sk#30, d_date#28, sum(sumss#37)#42 AS cume_sales#43] -(59) RowToColumnar -Input [3]: [item_sk#30, d_date#28, cume_sales#43] - -(60) CometColumnarExchange +(54) Exchange Input [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: hashpartitioning(item_sk#30, d_date#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(item_sk#30, d_date#28, 5), ENSURE_REQUIREMENTS, [plan_id=10] -(61) CometSort +(55) Sort [codegen id : 12] Input [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#30, d_date#28, cume_sales#43], [item_sk#30 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST] +Arguments: [item_sk#30 ASC NULLS FIRST, d_date#28 ASC NULLS FIRST], false, 0 -(62) CometSortMergeJoin -Left output [3]: [item_sk#9, d_date#6, cume_sales#22] -Right output [3]: [item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#9, d_date#6], [item_sk#30, d_date#28], FullOuter +(56) SortMergeJoin [codegen id : 13] +Left keys [2]: [item_sk#9, d_date#6] +Right keys [2]: [item_sk#30, d_date#28] +Join type: FullOuter +Join condition: None -(63) CometFilter +(57) Filter [codegen id : 13] Input [6]: [item_sk#9, d_date#6, cume_sales#22, item_sk#30, d_date#28, cume_sales#43] Condition : isnotnull(CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END) -(64) CometProject +(58) Project [codegen id : 13] +Output [4]: [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END AS item_sk#44, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#28 END AS d_date#45, cume_sales#22 AS web_sales#46, cume_sales#43 AS store_sales#47] Input [6]: [item_sk#9, d_date#6, cume_sales#22, item_sk#30, d_date#28, cume_sales#43] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [CASE WHEN isnotnull(item_sk#9) THEN item_sk#9 ELSE item_sk#30 END AS item_sk#44, CASE WHEN isnotnull(d_date#6) THEN d_date#6 ELSE d_date#28 END AS d_date#45, cume_sales#22 AS web_sales#46, cume_sales#43 AS store_sales#47] -(65) CometColumnarExchange +(59) Exchange Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: hashpartitioning(item_sk#44, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(item_sk#44, 5), ENSURE_REQUIREMENTS, [plan_id=11] -(66) CometSort +(60) Sort [codegen id : 14] Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] +Arguments: [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], false, 0 -(67) ColumnarToRow [codegen id : 11] -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] - -(68) Window +(61) Window Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#48], [item_sk#44], [d_date#45 ASC NULLS FIRST] -(69) ReusedExchange [Reuses operator id: 65] +(62) ReusedExchange [Reuses operator id: 59] Output [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -(70) CometSort -Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] -Arguments: [item_sk#44, d_date#45, web_sales#46, store_sales#47], [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST] - -(71) ColumnarToRow [codegen id : 22] +(63) Sort [codegen id : 28] Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] +Arguments: [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], false, 0 -(72) Window +(64) Window Input [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] Arguments: [row_number() windowspecdefinition(item_sk#44, d_date#45 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rk#49], [item_sk#44], [d_date#45 ASC NULLS FIRST] -(73) Project [codegen id : 23] +(65) Project [codegen id : 29] Output [4]: [item_sk#44 AS item_sk#50, web_sales#46 AS web_sales#51, store_sales#47 AS store_sales#52, rk#49] Input [5]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#49] -(74) BroadcastExchange +(66) BroadcastExchange Input [4]: [item_sk#50, web_sales#51, store_sales#52, rk#49] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] -(75) BroadcastHashJoin [codegen id : 24] +(67) BroadcastHashJoin [codegen id : 30] Left keys [1]: [item_sk#44] Right keys [1]: [item_sk#50] Join type: Inner Join condition: (rk#48 >= rk#49) -(76) Project [codegen id : 24] +(68) Project [codegen id : 30] Output [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] Input [9]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, rk#48, item_sk#50, web_sales#51, store_sales#52, rk#49] -(77) HashAggregate [codegen id : 24] +(69) HashAggregate [codegen id : 30] Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_sales#51, store_sales#52] Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] Functions [2]: [partial_max(web_sales#51), partial_max(store_sales#52)] Aggregate Attributes [2]: [max#53, max#54] Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#55, max#56] -(78) HashAggregate [codegen id : 24] +(70) HashAggregate [codegen id : 30] Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max#55, max#56] Keys [4]: [item_sk#44, d_date#45, web_sales#46, store_sales#47] Functions [2]: [max(web_sales#51), max(store_sales#52)] Aggregate Attributes [2]: [max(web_sales#51)#57, max(store_sales#52)#58] Results [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, max(web_sales#51)#57 AS web_cumulative#59, max(store_sales#52)#58 AS store_cumulative#60] -(79) Filter [codegen id : 24] +(71) Filter [codegen id : 30] Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] Condition : ((isnotnull(web_cumulative#59) AND isnotnull(store_cumulative#60)) AND (web_cumulative#59 > store_cumulative#60)) -(80) TakeOrderedAndProject +(72) TakeOrderedAndProject Input [6]: [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] Arguments: 100, [item_sk#44 ASC NULLS FIRST, d_date#45 ASC NULLS FIRST], [item_sk#44, d_date#45, web_sales#46, store_sales#47, web_cumulative#59, store_cumulative#60] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ws_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (85) -+- * ColumnarToRow (84) - +- CometProject (83) - +- CometFilter (82) - +- CometScan parquet spark_catalog.default.date_dim (81) +BroadcastExchange (77) ++- * ColumnarToRow (76) + +- CometProject (75) + +- CometFilter (74) + +- CometScan parquet spark_catalog.default.date_dim (73) -(81) Scan parquet spark_catalog.default.date_dim +(73) Scan parquet spark_catalog.default.date_dim Output [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(82) CometFilter +(74) CometFilter Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Condition : (((isnotnull(d_month_seq#7) AND (d_month_seq#7 >= 1212)) AND (d_month_seq#7 <= 1223)) AND isnotnull(d_date_sk#5)) -(83) CometProject +(75) CometProject Input [3]: [d_date_sk#5, d_date#6, d_month_seq#7] Arguments: [d_date_sk#5, d_date#6], [d_date_sk#5, d_date#6] -(84) ColumnarToRow [codegen id : 1] +(76) ColumnarToRow [codegen id : 1] Input [2]: [d_date_sk#5, d_date#6] -(85) BroadcastExchange +(77) BroadcastExchange Input [2]: [d_date_sk#5, d_date#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=13] -Subquery:2 Hosting operator id = 33 Hosting Expression = ss_sold_date_sk#25 IN dynamicpruning#4 +Subquery:2 Hosting operator id = 30 Hosting Expression = ss_sold_date_sk#25 IN dynamicpruning#4 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt index 7fe01fcc8..f8e1ba235 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q51a/simplified.txt @@ -1,5 +1,5 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store_cumulative] - WholeStageCodegen (24) + WholeStageCodegen (30) Filter [web_cumulative,store_cumulative] HashAggregate [item_sk,d_date,web_sales,store_sales,max,max] [max(web_sales),max(store_sales),web_cumulative,store_cumulative,max,max] HashAggregate [item_sk,d_date,web_sales,store_sales,web_sales,store_sales] [max,max,max,max] @@ -7,23 +7,23 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store BroadcastHashJoin [item_sk,item_sk,rk,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (11) - ColumnarToRow + WholeStageCodegen (14) + Sort [item_sk,d_date] InputAdapter - CometSort [item_sk,d_date,web_sales,store_sales] - CometColumnarExchange [item_sk] #1 - CometProject [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] [item_sk,d_date,web_sales,store_sales] - CometFilter [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] - CometSortMergeJoin [item_sk,d_date,cume_sales,item_sk,d_date,cume_sales] - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #2 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [item_sk,d_date,sumws] #3 - RowToColumnar + Exchange [item_sk] #1 + WholeStageCodegen (13) + Project [item_sk,item_sk,d_date,d_date,cume_sales,cume_sales] + Filter [item_sk,item_sk] + SortMergeJoin [item_sk,d_date,item_sk,d_date] + InputAdapter + WholeStageCodegen (6) + Sort [item_sk,d_date] + InputAdapter + Exchange [item_sk,d_date] #2 + WholeStageCodegen (5) + HashAggregate [item_sk,d_date,sumws,sum,isEmpty] [sum(sumws),cume_sales,sum,isEmpty] + InputAdapter + Exchange [item_sk,d_date,sumws] #3 WholeStageCodegen (4) HashAggregate [item_sk,d_date,sumws,sumws] [sum,isEmpty,sum,isEmpty] Project [item_sk,d_date,sumws,sumws] @@ -35,9 +35,9 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store ColumnarToRow InputAdapter CometSort [item_sk,d_date,sumws,ws_item_sk] - CometColumnarExchange [ws_item_sk] #4 + CometExchange [ws_item_sk] #4 CometHashAggregate [item_sk,d_date,sumws,ws_item_sk,sum,sum(UnscaledValue(ws_sales_price))] - CometColumnarExchange [ws_item_sk,d_date] #5 + CometExchange [ws_item_sk,d_date] #5 CometHashAggregate [ws_item_sk,d_date,sum,ws_sales_price] CometProject [ws_item_sk,ws_sales_price,d_date] CometBroadcastHashJoin [ws_item_sk,ws_sales_price,ws_sold_date_sk,d_date_sk,d_date] @@ -66,29 +66,29 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store InputAdapter CometSort [item_sk,d_date,sumws,ws_item_sk] ReusedExchange [item_sk,d_date,sumws,ws_item_sk] #4 - CometSort [item_sk,d_date,cume_sales] - CometColumnarExchange [item_sk,d_date] #9 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [item_sk,d_date,sumss] #10 - RowToColumnar - WholeStageCodegen (9) + InputAdapter + WholeStageCodegen (12) + Sort [item_sk,d_date] + InputAdapter + Exchange [item_sk,d_date] #9 + WholeStageCodegen (11) + HashAggregate [item_sk,d_date,sumss,sum,isEmpty] [sum(sumss),cume_sales,sum,isEmpty] + InputAdapter + Exchange [item_sk,d_date,sumss] #10 + WholeStageCodegen (10) HashAggregate [item_sk,d_date,sumss,sumss] [sum,isEmpty,sum,isEmpty] Project [item_sk,d_date,sumss,sumss] BroadcastHashJoin [item_sk,item_sk,rk,rk] Project [item_sk,d_date,sumss,rk] InputAdapter Window [ss_item_sk,d_date] - WholeStageCodegen (6) + WholeStageCodegen (7) ColumnarToRow InputAdapter CometSort [item_sk,d_date,sumss,ss_item_sk] - CometColumnarExchange [ss_item_sk] #11 + CometExchange [ss_item_sk] #11 CometHashAggregate [item_sk,d_date,sumss,ss_item_sk,sum,sum(UnscaledValue(ss_sales_price))] - CometColumnarExchange [ss_item_sk,d_date] #12 + CometExchange [ss_item_sk,d_date] #12 CometHashAggregate [ss_item_sk,d_date,sum,ss_sales_price] CometProject [ss_item_sk,ss_sales_price,d_date] CometBroadcastHashJoin [ss_item_sk,ss_sales_price,ss_sold_date_sk,d_date_sk,d_date] @@ -98,23 +98,22 @@ TakeOrderedAndProject [item_sk,d_date,web_sales,store_sales,web_cumulative,store ReusedExchange [d_date_sk,d_date] #7 InputAdapter BroadcastExchange #13 - WholeStageCodegen (8) + WholeStageCodegen (9) Project [item_sk,sumss,rk] InputAdapter Window [ss_item_sk,d_date] - WholeStageCodegen (7) + WholeStageCodegen (8) ColumnarToRow InputAdapter CometSort [item_sk,d_date,sumss,ss_item_sk] ReusedExchange [item_sk,d_date,sumss,ss_item_sk] #11 InputAdapter BroadcastExchange #14 - WholeStageCodegen (23) + WholeStageCodegen (29) Project [item_sk,web_sales,store_sales,rk] InputAdapter Window [item_sk,d_date] - WholeStageCodegen (22) - ColumnarToRow + WholeStageCodegen (28) + Sort [item_sk,d_date] InputAdapter - CometSort [item_sk,d_date,web_sales,store_sales] - ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 + ReusedExchange [item_sk,d_date,web_sales,store_sales] #1 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt index 59b4666bf..01a5d7c56 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/explain.txt @@ -11,9 +11,9 @@ TakeOrderedAndProject (45) : : +- Window (24) : : +- * ColumnarToRow (23) : : +- CometSort (22) - : : +- CometColumnarExchange (21) + : : +- CometExchange (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -135,18 +135,18 @@ Input [6]: [i_brand#2, i_category#3, cs_sales_price#6, d_year#10, d_moy#11, cc_n Keys [5]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11] Functions [1]: [partial_sum(UnscaledValue(cs_sales_price#6))] -(19) CometColumnarExchange +(19) CometExchange Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#14] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (20) CometHashAggregate Input [6]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum#14] Keys [5]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11] Functions [1]: [sum(UnscaledValue(cs_sales_price#6))] -(21) CometColumnarExchange +(21) CometExchange Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#15, _w0#16] -Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#3, i_brand#2, cc_name#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (22) CometSort Input [7]: [i_category#3, i_brand#2, cc_name#13, d_year#10, d_moy#11, sum_sales#15, _w0#16] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt index ed9e697f3..539839af7 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q57/simplified.txt @@ -16,9 +16,9 @@ TakeOrderedAndProject [sum_sales,avg_monthly_sales,d_year,i_category,i_brand,d_m ColumnarToRow InputAdapter CometSort [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0] - CometColumnarExchange [i_category,i_brand,cc_name] #1 + CometExchange [i_category,i_brand,cc_name] #1 CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum_sales,_w0,sum,sum(UnscaledValue(cs_sales_price))] - CometColumnarExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 + CometExchange [i_category,i_brand,cc_name,d_year,d_moy] #2 CometHashAggregate [i_category,i_brand,cc_name,d_year,d_moy,sum,cs_sales_price] CometProject [i_brand,i_category,cs_sales_price,d_year,d_moy,cc_name] CometBroadcastHashJoin [i_brand,i_category,cs_call_center_sk,cs_sales_price,d_year,d_moy,cc_call_center_sk,cc_name] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt index bb6a6a8f4..a826e7d78 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/explain.txt @@ -1,97 +1,87 @@ == Physical Plan == -* ColumnarToRow (93) -+- CometTakeOrderedAndProject (92) - +- CometHashAggregate (91) - +- CometColumnarExchange (90) - +- RowToColumnar (89) - +- * HashAggregate (88) - +- Union (87) - :- * HashAggregate (70) - : +- * ColumnarToRow (69) - : +- CometColumnarExchange (68) - : +- RowToColumnar (67) - : +- * HashAggregate (66) - : +- Union (65) - : :- * HashAggregate (22) - : : +- * ColumnarToRow (21) - : : +- CometColumnarExchange (20) - : : +- CometHashAggregate (19) - : : +- CometProject (18) - : : +- CometBroadcastHashJoin (17) - : : :- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometUnion (7) - : : : : :- CometProject (3) - : : : : : +- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometProject (6) - : : : : +- CometFilter (5) - : : : : +- CometScan parquet spark_catalog.default.store_returns (4) - : : : +- CometBroadcastExchange (11) - : : : +- CometProject (10) - : : : +- CometFilter (9) - : : : +- CometScan parquet spark_catalog.default.date_dim (8) - : : +- CometBroadcastExchange (16) - : : +- CometFilter (15) - : : +- CometScan parquet spark_catalog.default.store (14) - : :- * HashAggregate (41) - : : +- * ColumnarToRow (40) - : : +- CometColumnarExchange (39) - : : +- CometHashAggregate (38) - : : +- CometProject (37) - : : +- CometBroadcastHashJoin (36) - : : :- CometProject (32) - : : : +- CometBroadcastHashJoin (31) - : : : :- CometUnion (29) - : : : : :- CometProject (25) - : : : : : +- CometFilter (24) - : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) - : : : : +- CometProject (28) - : : : : +- CometFilter (27) - : : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) - : : : +- ReusedExchange (30) - : : +- CometBroadcastExchange (35) - : : +- CometFilter (34) - : : +- CometScan parquet spark_catalog.default.catalog_page (33) - : +- * HashAggregate (64) - : +- * ColumnarToRow (63) - : +- CometColumnarExchange (62) - : +- CometHashAggregate (61) - : +- CometProject (60) - : +- CometBroadcastHashJoin (59) - : :- CometProject (55) - : : +- CometBroadcastHashJoin (54) - : : :- CometUnion (52) - : : : :- CometProject (44) - : : : : +- CometFilter (43) - : : : : +- CometScan parquet spark_catalog.default.web_sales (42) - : : : +- CometProject (51) - : : : +- CometBroadcastHashJoin (50) - : : : :- CometBroadcastExchange (46) - : : : : +- CometScan parquet spark_catalog.default.web_returns (45) - : : : +- CometProject (49) - : : : +- CometFilter (48) - : : : +- CometScan parquet spark_catalog.default.web_sales (47) - : : +- ReusedExchange (53) - : +- CometBroadcastExchange (58) - : +- CometFilter (57) - : +- CometScan parquet spark_catalog.default.web_site (56) - :- * HashAggregate (78) - : +- * ColumnarToRow (77) - : +- CometColumnarExchange (76) - : +- RowToColumnar (75) - : +- * HashAggregate (74) - : +- * HashAggregate (73) - : +- * ColumnarToRow (72) - : +- ReusedExchange (71) - +- * HashAggregate (86) - +- * ColumnarToRow (85) - +- CometColumnarExchange (84) - +- RowToColumnar (83) - +- * HashAggregate (82) - +- * HashAggregate (81) - +- * ColumnarToRow (80) - +- ReusedExchange (79) +TakeOrderedAndProject (83) ++- * HashAggregate (82) + +- Exchange (81) + +- * HashAggregate (80) + +- Union (79) + :- * HashAggregate (68) + : +- Exchange (67) + : +- * HashAggregate (66) + : +- Union (65) + : :- * HashAggregate (22) + : : +- * ColumnarToRow (21) + : : +- CometExchange (20) + : : +- CometHashAggregate (19) + : : +- CometProject (18) + : : +- CometBroadcastHashJoin (17) + : : :- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometUnion (7) + : : : : :- CometProject (3) + : : : : : +- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometProject (6) + : : : : +- CometFilter (5) + : : : : +- CometScan parquet spark_catalog.default.store_returns (4) + : : : +- CometBroadcastExchange (11) + : : : +- CometProject (10) + : : : +- CometFilter (9) + : : : +- CometScan parquet spark_catalog.default.date_dim (8) + : : +- CometBroadcastExchange (16) + : : +- CometFilter (15) + : : +- CometScan parquet spark_catalog.default.store (14) + : :- * HashAggregate (41) + : : +- * ColumnarToRow (40) + : : +- CometExchange (39) + : : +- CometHashAggregate (38) + : : +- CometProject (37) + : : +- CometBroadcastHashJoin (36) + : : :- CometProject (32) + : : : +- CometBroadcastHashJoin (31) + : : : :- CometUnion (29) + : : : : :- CometProject (25) + : : : : : +- CometFilter (24) + : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (23) + : : : : +- CometProject (28) + : : : : +- CometFilter (27) + : : : : +- CometScan parquet spark_catalog.default.catalog_returns (26) + : : : +- ReusedExchange (30) + : : +- CometBroadcastExchange (35) + : : +- CometFilter (34) + : : +- CometScan parquet spark_catalog.default.catalog_page (33) + : +- * HashAggregate (64) + : +- * ColumnarToRow (63) + : +- CometExchange (62) + : +- CometHashAggregate (61) + : +- CometProject (60) + : +- CometBroadcastHashJoin (59) + : :- CometProject (55) + : : +- CometBroadcastHashJoin (54) + : : :- CometUnion (52) + : : : :- CometProject (44) + : : : : +- CometFilter (43) + : : : : +- CometScan parquet spark_catalog.default.web_sales (42) + : : : +- CometProject (51) + : : : +- CometBroadcastHashJoin (50) + : : : :- CometBroadcastExchange (46) + : : : : +- CometScan parquet spark_catalog.default.web_returns (45) + : : : +- CometProject (49) + : : : +- CometFilter (48) + : : : +- CometScan parquet spark_catalog.default.web_sales (47) + : : +- ReusedExchange (53) + : +- CometBroadcastExchange (58) + : +- CometFilter (57) + : +- CometScan parquet spark_catalog.default.web_site (56) + :- * HashAggregate (73) + : +- Exchange (72) + : +- * HashAggregate (71) + : +- * HashAggregate (70) + : +- ReusedExchange (69) + +- * HashAggregate (78) + +- Exchange (77) + +- * HashAggregate (76) + +- * HashAggregate (75) + +- ReusedExchange (74) (1) Scan parquet spark_catalog.default.store_sales @@ -187,9 +177,9 @@ Input [5]: [sales_price#8, profit#9, return_amt#10, net_loss#11, s_store_id#25] Keys [1]: [s_store_id#25] Functions [4]: [partial_sum(UnscaledValue(sales_price#8)), partial_sum(UnscaledValue(return_amt#10)), partial_sum(UnscaledValue(profit#9)), partial_sum(UnscaledValue(net_loss#11))] -(20) CometColumnarExchange +(20) CometExchange Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] -Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_id#25, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) ColumnarToRow [codegen id : 1] Input [5]: [s_store_id#25, sum#26, sum#27, sum#28, sum#29] @@ -278,9 +268,9 @@ Input [5]: [sales_price#46, profit#47, return_amt#48, net_loss#49, cp_catalog_pa Keys [1]: [cp_catalog_page_id#62] Functions [4]: [partial_sum(UnscaledValue(sales_price#46)), partial_sum(UnscaledValue(return_amt#48)), partial_sum(UnscaledValue(profit#47)), partial_sum(UnscaledValue(net_loss#49))] -(39) CometColumnarExchange +(39) CometExchange Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] -Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cp_catalog_page_id#62, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (40) ColumnarToRow [codegen id : 2] Input [5]: [cp_catalog_page_id#62, sum#63, sum#64, sum#65, sum#66] @@ -388,9 +378,9 @@ Input [5]: [sales_price#83, profit#84, return_amt#85, net_loss#86, web_site_id#1 Keys [1]: [web_site_id#104] Functions [4]: [partial_sum(UnscaledValue(sales_price#83)), partial_sum(UnscaledValue(return_amt#85)), partial_sum(UnscaledValue(profit#84)), partial_sum(UnscaledValue(net_loss#86))] -(62) CometColumnarExchange +(62) CometExchange Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] -Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(web_site_id#104, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (63) ColumnarToRow [codegen id : 3] Input [5]: [web_site_id#104, sum#105, sum#106, sum#107, sum#108] @@ -411,154 +401,126 @@ Functions [3]: [partial_sum(sales#36), partial_sum(returns#37), partial_sum(prof Aggregate Attributes [6]: [sum#118, isEmpty#119, sum#120, isEmpty#121, sum#122, isEmpty#123] Results [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -(67) RowToColumnar +(67) Exchange Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] +Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(68) CometColumnarExchange -Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -Arguments: hashpartitioning(channel#34, id#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(69) ColumnarToRow [codegen id : 5] -Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] - -(70) HashAggregate [codegen id : 5] +(68) HashAggregate [codegen id : 5] Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] Keys [2]: [channel#34, id#35] Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] Results [5]: [channel#34, id#35, cast(sum(sales#36)#130 as decimal(37,2)) AS sales#133, cast(sum(returns#37)#131 as decimal(37,2)) AS returns#134, cast(sum(profit#38)#132 as decimal(38,2)) AS profit#135] -(71) ReusedExchange [Reuses operator id: 68] +(69) ReusedExchange [Reuses operator id: 67] Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -(72) ColumnarToRow [codegen id : 10] -Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] - -(73) HashAggregate [codegen id : 10] +(70) HashAggregate [codegen id : 10] Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] Keys [2]: [channel#34, id#35] Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] Results [4]: [channel#34, sum(sales#36)#130 AS sales#136, sum(returns#37)#131 AS returns#137, sum(profit#38)#132 AS profit#138] -(74) HashAggregate [codegen id : 10] +(71) HashAggregate [codegen id : 10] Input [4]: [channel#34, sales#136, returns#137, profit#138] Keys [1]: [channel#34] Functions [3]: [partial_sum(sales#136), partial_sum(returns#137), partial_sum(profit#138)] Aggregate Attributes [6]: [sum#139, isEmpty#140, sum#141, isEmpty#142, sum#143, isEmpty#144] Results [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] -(75) RowToColumnar +(72) Exchange Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] +Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(76) CometColumnarExchange -Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] -Arguments: hashpartitioning(channel#34, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(77) ColumnarToRow [codegen id : 11] -Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] - -(78) HashAggregate [codegen id : 11] +(73) HashAggregate [codegen id : 11] Input [7]: [channel#34, sum#145, isEmpty#146, sum#147, isEmpty#148, sum#149, isEmpty#150] Keys [1]: [channel#34] Functions [3]: [sum(sales#136), sum(returns#137), sum(profit#138)] Aggregate Attributes [3]: [sum(sales#136)#151, sum(returns#137)#152, sum(profit#138)#153] Results [5]: [channel#34, null AS id#154, sum(sales#136)#151 AS sum(sales)#155, sum(returns#137)#152 AS sum(returns)#156, sum(profit#138)#153 AS sum(profit)#157] -(79) ReusedExchange [Reuses operator id: 68] +(74) ReusedExchange [Reuses operator id: 67] Output [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] -(80) ColumnarToRow [codegen id : 16] -Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] - -(81) HashAggregate [codegen id : 16] +(75) HashAggregate [codegen id : 16] Input [8]: [channel#34, id#35, sum#124, isEmpty#125, sum#126, isEmpty#127, sum#128, isEmpty#129] Keys [2]: [channel#34, id#35] Functions [3]: [sum(sales#36), sum(returns#37), sum(profit#38)] Aggregate Attributes [3]: [sum(sales#36)#130, sum(returns#37)#131, sum(profit#38)#132] Results [3]: [sum(sales#36)#130 AS sales#136, sum(returns#37)#131 AS returns#137, sum(profit#38)#132 AS profit#138] -(82) HashAggregate [codegen id : 16] +(76) HashAggregate [codegen id : 16] Input [3]: [sales#136, returns#137, profit#138] Keys: [] Functions [3]: [partial_sum(sales#136), partial_sum(returns#137), partial_sum(profit#138)] Aggregate Attributes [6]: [sum#158, isEmpty#159, sum#160, isEmpty#161, sum#162, isEmpty#163] Results [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] -(83) RowToColumnar -Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] - -(84) CometColumnarExchange +(77) Exchange Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(85) ColumnarToRow [codegen id : 17] -Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] - -(86) HashAggregate [codegen id : 17] +(78) HashAggregate [codegen id : 17] Input [6]: [sum#164, isEmpty#165, sum#166, isEmpty#167, sum#168, isEmpty#169] Keys: [] Functions [3]: [sum(sales#136), sum(returns#137), sum(profit#138)] Aggregate Attributes [3]: [sum(sales#136)#170, sum(returns#137)#171, sum(profit#138)#172] Results [5]: [null AS channel#173, null AS id#174, sum(sales#136)#170 AS sum(sales)#175, sum(returns#137)#171 AS sum(returns)#176, sum(profit#138)#172 AS sum(profit)#177] -(87) Union +(79) Union -(88) HashAggregate [codegen id : 18] +(80) HashAggregate [codegen id : 18] Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] Aggregate Attributes: [] Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] -(89) RowToColumnar +(81) Exchange Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(90) CometColumnarExchange -Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] -Arguments: hashpartitioning(channel#34, id#35, sales#133, returns#134, profit#135, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(91) CometHashAggregate +(82) HashAggregate [codegen id : 19] Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Keys [5]: [channel#34, id#35, sales#133, returns#134, profit#135] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#34, id#35, sales#133, returns#134, profit#135] -(92) CometTakeOrderedAndProject -Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#34 ASC NULLS FIRST,id#35 ASC NULLS FIRST], output=[channel#34,id#35,sales#133,returns#134,profit#135]), [channel#34, id#35, sales#133, returns#134, profit#135], 100, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] - -(93) ColumnarToRow [codegen id : 19] +(83) TakeOrderedAndProject Input [5]: [channel#34, id#35, sales#133, returns#134, profit#135] +Arguments: 100, [channel#34 ASC NULLS FIRST, id#35 ASC NULLS FIRST], [channel#34, id#35, sales#133, returns#134, profit#135] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (98) -+- * ColumnarToRow (97) - +- CometProject (96) - +- CometFilter (95) - +- CometScan parquet spark_catalog.default.date_dim (94) +BroadcastExchange (88) ++- * ColumnarToRow (87) + +- CometProject (86) + +- CometFilter (85) + +- CometScan parquet spark_catalog.default.date_dim (84) -(94) Scan parquet spark_catalog.default.date_dim +(84) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#22, d_date#23] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-08-18), IsNotNull(d_date_sk)] ReadSchema: struct -(95) CometFilter +(85) CometFilter Input [2]: [d_date_sk#22, d_date#23] Condition : (((isnotnull(d_date#23) AND (d_date#23 >= 1998-08-04)) AND (d_date#23 <= 1998-08-18)) AND isnotnull(d_date_sk#22)) -(96) CometProject +(86) CometProject Input [2]: [d_date_sk#22, d_date#23] Arguments: [d_date_sk#22], [d_date_sk#22] -(97) ColumnarToRow [codegen id : 1] +(87) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#22] -(98) BroadcastExchange +(88) BroadcastExchange Input [1]: [d_date_sk#22] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=8] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt index 00dd3c480..751b3dc24 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q5a/simplified.txt @@ -1,128 +1,118 @@ -WholeStageCodegen (19) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,id,sales,returns,profit] - CometHashAggregate [channel,id,sales,returns,profit] - CometColumnarExchange [channel,id,sales,returns,profit] #1 - RowToColumnar - WholeStageCodegen (18) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id] #2 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_id] #3 - CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,s_store_id] - CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] - CometProject [store_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #6 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [cp_catalog_page_id] #7 - CometHashAggregate [cp_catalog_page_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [page_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #8 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [web_site_id] #9 - CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] - CometProject [sales_price,profit,return_amt,net_loss,web_site_id] - CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] - CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] - CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] - CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] - CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #10 - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] - CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [web_site_sk,web_site_id] #11 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - WholeStageCodegen (11) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel] #12 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #13 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (19) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Exchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [s_store_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,s_store_id] + CometBroadcastHashJoin [store_sk,sales_price,profit,return_amt,net_loss,s_store_sk,s_store_id] + CometProject [store_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [store_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ss_store_sk,ss_sold_date_sk,ss_ext_sales_price,ss_net_profit] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometProject [sr_store_sk,sr_returned_date_sk,sr_return_amt,sr_net_loss] [store_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #6 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #7 + CometHashAggregate [cp_catalog_page_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [page_sk,sales_price,profit,return_amt,net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [page_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [page_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [cs_catalog_page_sk,cs_sold_date_sk,cs_ext_sales_price,cs_net_profit] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [cr_catalog_page_sk,cr_returned_date_sk,cr_return_amount,cr_net_loss] [page_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_catalog_page_sk,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #8 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,sum,sum] [sum(UnscaledValue(sales_price)),sum(UnscaledValue(return_amt)),sum(UnscaledValue(profit)),sum(UnscaledValue(net_loss)),channel,id,sales,returns,profit,sum,sum,sum,sum] + ColumnarToRow + InputAdapter + CometExchange [web_site_id] #9 + CometHashAggregate [web_site_id,sum,sum,sum,sum,sales_price,return_amt,profit,net_loss] + CometProject [sales_price,profit,return_amt,net_loss,web_site_id] + CometBroadcastHashJoin [wsr_web_site_sk,sales_price,profit,return_amt,net_loss,web_site_sk,web_site_id] + CometProject [wsr_web_site_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss,d_date_sk] + CometUnion [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometProject [ws_web_site_sk,ws_sold_date_sk,ws_ext_sales_price,ws_net_profit] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometFilter [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_site_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_web_site_sk,wr_returned_date_sk,wr_return_amt,wr_net_loss] [wsr_web_site_sk,date_sk,sales_price,profit,return_amt,net_loss] + CometBroadcastHashJoin [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk,ws_item_sk,ws_web_site_sk,ws_order_number] + CometBroadcastExchange [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] #10 + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + CometProject [ws_item_sk,ws_web_site_sk,ws_order_number] + CometFilter [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_order_number,ws_sold_date_sk] + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [web_site_sk,web_site_id] #11 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel] #12 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sum(sales),sum(returns),sum(profit),sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange #13 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt index b662f22fb..358d4995e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/explain.txt @@ -3,7 +3,7 @@ +- CometTakeOrderedAndProject (37) +- CometFilter (36) +- CometHashAggregate (35) - +- CometColumnarExchange (34) + +- CometExchange (34) +- CometHashAggregate (33) +- CometProject (32) +- CometBroadcastHashJoin (31) @@ -33,7 +33,7 @@ +- CometBroadcastExchange (27) +- CometFilter (26) +- CometHashAggregate (25) - +- CometColumnarExchange (24) + +- CometExchange (24) +- CometHashAggregate (23) +- CometFilter (22) +- CometScan parquet spark_catalog.default.item (21) @@ -154,9 +154,9 @@ Input [2]: [i_current_price#16, i_category#17] Keys [1]: [i_category#17] Functions [1]: [partial_avg(UnscaledValue(i_current_price#16))] -(24) CometColumnarExchange +(24) CometExchange Input [3]: [i_category#17, sum#18, count#19] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (25) CometHashAggregate Input [3]: [i_category#17, sum#18, count#19] @@ -198,9 +198,9 @@ Input [1]: [ca_state#2] Keys [1]: [ca_state#2] Functions [1]: [partial_count(1)] -(34) CometColumnarExchange +(34) CometExchange Input [2]: [ca_state#2, count#21] -Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(ca_state#2, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (35) CometHashAggregate Input [2]: [ca_state#2, count#21] @@ -253,7 +253,7 @@ Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)) Subquery:2 Hosting operator id = 40 Hosting Expression = Subquery scalar-subquery#11, [id=#12] * ColumnarToRow (50) +- CometHashAggregate (49) - +- CometColumnarExchange (48) + +- CometExchange (48) +- CometHashAggregate (47) +- CometProject (46) +- CometFilter (45) @@ -280,9 +280,9 @@ Input [1]: [d_month_seq#24] Keys [1]: [d_month_seq#24] Functions: [] -(48) CometColumnarExchange +(48) CometExchange Input [1]: [d_month_seq#24] -Arguments: hashpartitioning(d_month_seq#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(d_month_seq#24, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (49) CometHashAggregate Input [1]: [d_month_seq#24] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt index b95744087..580f668ea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q6/simplified.txt @@ -4,7 +4,7 @@ WholeStageCodegen (1) CometTakeOrderedAndProject [state,cnt,ca_state] CometFilter [state,cnt,ca_state] CometHashAggregate [state,cnt,ca_state,count,count(1)] - CometColumnarExchange [ca_state] #1 + CometExchange [ca_state] #1 CometHashAggregate [ca_state,count] CometProject [ca_state] CometBroadcastHashJoin [ca_state,ss_item_sk,i_item_sk] @@ -34,7 +34,7 @@ WholeStageCodegen (1) ColumnarToRow InputAdapter CometHashAggregate [d_month_seq] - CometColumnarExchange [d_month_seq] #5 + CometExchange [d_month_seq] #5 CometHashAggregate [d_month_seq] CometProject [d_month_seq] CometFilter [d_month_seq,d_year,d_moy] @@ -53,7 +53,7 @@ WholeStageCodegen (1) CometBroadcastExchange [avg(i_current_price),i_category] #8 CometFilter [avg(i_current_price),i_category] CometHashAggregate [avg(i_current_price),i_category,sum,count,avg(UnscaledValue(i_current_price))] - CometColumnarExchange [i_category] #9 + CometExchange [i_category] #9 CometHashAggregate [i_category,sum,count,i_current_price] CometFilter [i_current_price,i_category] CometScan parquet spark_catalog.default.item [i_current_price,i_category] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt index 174590641..bd75cbb40 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/explain.txt @@ -7,7 +7,7 @@ +- * SortMergeJoin Inner (171) :- * ColumnarToRow (101) : +- CometSort (100) - : +- CometColumnarExchange (99) + : +- CometExchange (99) : +- CometHashAggregate (98) : +- CometHashAggregate (97) : +- CometProject (96) @@ -43,7 +43,7 @@ : : : : : : : : : : : : : : : :- CometProject (30) : : : : : : : : : : : : : : : : +- CometSortMergeJoin (29) : : : : : : : : : : : : : : : : :- CometSort (10) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (9) + : : : : : : : : : : : : : : : : : +- CometExchange (9) : : : : : : : : : : : : : : : : : +- CometProject (8) : : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (7) : : : : : : : : : : : : : : : : : :- CometBroadcastExchange (3) @@ -56,17 +56,17 @@ : : : : : : : : : : : : : : : : +- CometProject (27) : : : : : : : : : : : : : : : : +- CometFilter (26) : : : : : : : : : : : : : : : : +- CometHashAggregate (25) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (24) + : : : : : : : : : : : : : : : : +- CometExchange (24) : : : : : : : : : : : : : : : : +- CometHashAggregate (23) : : : : : : : : : : : : : : : : +- CometProject (22) : : : : : : : : : : : : : : : : +- CometSortMergeJoin (21) : : : : : : : : : : : : : : : : :- CometSort (15) - : : : : : : : : : : : : : : : : : +- CometColumnarExchange (14) + : : : : : : : : : : : : : : : : : +- CometExchange (14) : : : : : : : : : : : : : : : : : +- CometProject (13) : : : : : : : : : : : : : : : : : +- CometFilter (12) : : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (11) : : : : : : : : : : : : : : : : +- CometSort (20) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (19) + : : : : : : : : : : : : : : : : +- CometExchange (19) : : : : : : : : : : : : : : : : +- CometProject (18) : : : : : : : : : : : : : : : : +- CometFilter (17) : : : : : : : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_returns (16) @@ -108,7 +108,7 @@ : +- CometScan parquet spark_catalog.default.item (91) +- * ColumnarToRow (170) +- CometSort (169) - +- CometColumnarExchange (168) + +- CometExchange (168) +- CometHashAggregate (167) +- CometHashAggregate (166) +- CometProject (165) @@ -144,7 +144,7 @@ : : : : : : : : : : : : : : :- CometProject (118) : : : : : : : : : : : : : : : +- CometSortMergeJoin (117) : : : : : : : : : : : : : : : :- CometSort (111) - : : : : : : : : : : : : : : : : +- CometColumnarExchange (110) + : : : : : : : : : : : : : : : : +- CometExchange (110) : : : : : : : : : : : : : : : : +- CometProject (109) : : : : : : : : : : : : : : : : +- CometBroadcastHashJoin (108) : : : : : : : : : : : : : : : : :- CometBroadcastExchange (104) @@ -217,9 +217,9 @@ Arguments: [ss_item_sk#1, ss_ticket_number#8], [sr_item_sk#14, sr_ticket_number# Input [14]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_ticket_number#8, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12, sr_item_sk#14, sr_ticket_number#15] Arguments: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12], [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -(9) CometColumnarExchange +(9) CometExchange Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] -Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(ss_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (10) CometSort Input [11]: [ss_item_sk#1, ss_customer_sk#2, ss_cdemo_sk#3, ss_hdemo_sk#4, ss_addr_sk#5, ss_store_sk#6, ss_promo_sk#7, ss_wholesale_cost#9, ss_list_price#10, ss_coupon_amt#11, ss_sold_date_sk#12] @@ -240,9 +240,9 @@ Condition : (isnotnull(cs_item_sk#17) AND isnotnull(cs_order_number#18)) Input [4]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19, cs_sold_date_sk#20] Arguments: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19], [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -(14) CometColumnarExchange +(14) CometExchange Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] -Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cs_item_sk#17, cs_order_number#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (15) CometSort Input [3]: [cs_item_sk#17, cs_order_number#18, cs_ext_list_price#19] @@ -263,9 +263,9 @@ Condition : (isnotnull(cr_item_sk#21) AND isnotnull(cr_order_number#22)) Input [6]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25, cr_returned_date_sk#26] Arguments: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25], [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -(19) CometColumnarExchange +(19) CometExchange Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] -Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cr_item_sk#21, cr_order_number#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (20) CometSort Input [5]: [cr_item_sk#21, cr_order_number#22, cr_refunded_cash#23, cr_reversed_charge#24, cr_store_credit#25] @@ -285,9 +285,9 @@ Input [5]: [cs_item_sk#17, cs_ext_list_price#19, cr_refunded_cash#23, cr_reverse Keys [1]: [cs_item_sk#17] Functions [2]: [partial_sum(UnscaledValue(cs_ext_list_price#19)), partial_sum(((cr_refunded_cash#23 + cr_reversed_charge#24) + cr_store_credit#25))] -(24) CometColumnarExchange +(24) CometExchange Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] -Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cs_item_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (25) CometHashAggregate Input [4]: [cs_item_sk#17, sum#27, sum#28, isEmpty#29] @@ -629,9 +629,9 @@ Input [19]: [i_product_name#71, i_item_sk#68, s_store_name#35, s_zip#36, ca_stre Keys [15]: [i_product_name#71, i_item_sk#68, s_store_name#35, s_zip#36, ca_street_number#57, ca_street_name#58, ca_city#59, ca_zip#60, ca_street_number#62, ca_street_name#63, ca_city#64, ca_zip#65, d_year#33, d_year#44, d_year#46] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#9)), sum(UnscaledValue(ss_list_price#10)), sum(UnscaledValue(ss_coupon_amt#11))] -(99) CometColumnarExchange +(99) CometExchange Input [17]: [product_name#76, item_sk#77, store_name#78, store_zip#79, b_street_number#80, b_streen_name#81, b_city#82, b_zip#83, c_street_number#84, c_street_name#85, c_city#86, c_zip#87, syear#88, cnt#89, s1#90, s2#91, s3#92] -Arguments: hashpartitioning(item_sk#77, store_name#78, store_zip#79, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(item_sk#77, store_name#78, store_zip#79, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (100) CometSort Input [17]: [product_name#76, item_sk#77, store_name#78, store_zip#79, b_street_number#80, b_streen_name#81, b_city#82, b_zip#83, c_street_number#84, c_street_name#85, c_city#86, c_zip#87, syear#88, cnt#89, s1#90, s2#91, s3#92] @@ -680,9 +680,9 @@ Arguments: [ss_item_sk#93, ss_ticket_number#100], [sr_item_sk#106, sr_ticket_num Input [14]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_ticket_number#100, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104, sr_item_sk#106, sr_ticket_number#107] Arguments: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104], [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] -(110) CometColumnarExchange +(110) CometExchange Input [11]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] -Arguments: hashpartitioning(ss_item_sk#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(ss_item_sk#93, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (111) CometSort Input [11]: [ss_item_sk#93, ss_customer_sk#94, ss_cdemo_sk#95, ss_hdemo_sk#96, ss_addr_sk#97, ss_store_sk#98, ss_promo_sk#99, ss_wholesale_cost#101, ss_list_price#102, ss_coupon_amt#103, ss_sold_date_sk#104] @@ -919,9 +919,9 @@ Input [19]: [i_product_name#154, i_item_sk#153, s_store_name#120, s_zip#121, ca_ Keys [15]: [i_product_name#154, i_item_sk#153, s_store_name#120, s_zip#121, ca_street_number#142, ca_street_name#143, ca_city#144, ca_zip#145, ca_street_number#147, ca_street_name#148, ca_city#149, ca_zip#150, d_year#118, d_year#129, d_year#131] Functions [4]: [count(1), sum(UnscaledValue(ss_wholesale_cost#101)), sum(UnscaledValue(ss_list_price#102)), sum(UnscaledValue(ss_coupon_amt#103))] -(168) CometColumnarExchange +(168) CometExchange Input [8]: [item_sk#158, store_name#159, store_zip#160, syear#161, cnt#162, s1#163, s2#164, s3#165] -Arguments: hashpartitioning(item_sk#158, store_name#159, store_zip#160, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(item_sk#158, store_name#159, store_zip#160, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (169) CometSort Input [8]: [item_sk#158, store_name#159, store_zip#160, syear#161, cnt#162, s1#163, s2#164, s3#165] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt index 70d0e66f8..911f7fcbd 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q64/simplified.txt @@ -12,7 +12,7 @@ WholeStageCodegen (4) ColumnarToRow InputAdapter CometSort [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #2 + CometExchange [item_sk,store_name,store_zip] #2 CometHashAggregate [product_name,item_sk,store_name,store_zip,b_street_number,b_streen_name,b_city,b_zip,c_street_number,c_street_name,c_city,c_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -48,7 +48,7 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #3 + CometExchange [ss_item_sk] #3 CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #4 @@ -68,17 +68,17 @@ WholeStageCodegen (4) CometProject [cs_item_sk] CometFilter [cs_item_sk,sale,refund] CometHashAggregate [cs_item_sk,sale,refund,sum,sum,isEmpty,sum(UnscaledValue(cs_ext_list_price)),sum(((cr_refunded_cash + cr_reversed_charge) + cr_store_credit))] - CometColumnarExchange [cs_item_sk] #6 + CometExchange [cs_item_sk] #6 CometHashAggregate [cs_item_sk,sum,sum,isEmpty,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometProject [cs_item_sk,cs_ext_list_price,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_ext_list_price,cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometSort [cs_item_sk,cs_order_number,cs_ext_list_price] - CometColumnarExchange [cs_item_sk,cs_order_number] #7 + CometExchange [cs_item_sk,cs_order_number] #7 CometProject [cs_item_sk,cs_order_number,cs_ext_list_price] CometFilter [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometScan parquet spark_catalog.default.catalog_sales [cs_item_sk,cs_order_number,cs_ext_list_price,cs_sold_date_sk] CometSort [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] - CometColumnarExchange [cr_item_sk,cr_order_number] #8 + CometExchange [cr_item_sk,cr_order_number] #8 CometProject [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit] CometFilter [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_refunded_cash,cr_reversed_charge,cr_store_credit,cr_returned_date_sk] @@ -123,7 +123,7 @@ WholeStageCodegen (4) ColumnarToRow InputAdapter CometSort [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3] - CometColumnarExchange [item_sk,store_name,store_zip] #19 + CometExchange [item_sk,store_name,store_zip] #19 CometHashAggregate [item_sk,store_name,store_zip,syear,cnt,s1,s2,s3,i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,count(1),sum(UnscaledValue(ss_wholesale_cost)),sum(UnscaledValue(ss_list_price)),sum(UnscaledValue(ss_coupon_amt))] CometHashAggregate [i_product_name,i_item_sk,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,d_year,d_year,d_year,count,sum,sum,sum,ss_wholesale_cost,ss_list_price,ss_coupon_amt] CometProject [ss_wholesale_cost,ss_list_price,ss_coupon_amt,d_year,d_year,d_year,s_store_name,s_zip,ca_street_number,ca_street_name,ca_city,ca_zip,ca_street_number,ca_street_name,ca_city,ca_zip,i_item_sk,i_product_name] @@ -159,7 +159,7 @@ WholeStageCodegen (4) CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometSortMergeJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,cs_item_sk] CometSort [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk] #20 + CometExchange [ss_item_sk] #20 CometProject [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] CometBroadcastHashJoin [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk,sr_item_sk,sr_ticket_number] CometBroadcastExchange [ss_item_sk,ss_customer_sk,ss_cdemo_sk,ss_hdemo_sk,ss_addr_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk] #21 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt index bc2e260ce..4044e13af 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/explain.txt @@ -4,10 +4,10 @@ TakeOrderedAndProject (68) +- Window (66) +- * ColumnarToRow (65) +- CometSort (64) - +- CometColumnarExchange (63) + +- CometExchange (63) +- CometUnion (62) :- CometHashAggregate (21) - : +- CometColumnarExchange (20) + : +- CometExchange (20) : +- CometHashAggregate (19) : +- CometProject (18) : +- CometBroadcastHashJoin (17) @@ -28,42 +28,42 @@ TakeOrderedAndProject (68) : +- CometFilter (15) : +- CometScan parquet spark_catalog.default.item (14) :- CometHashAggregate (26) - : +- CometColumnarExchange (25) + : +- CometExchange (25) : +- CometHashAggregate (24) : +- CometHashAggregate (23) : +- ReusedExchange (22) :- CometHashAggregate (31) - : +- CometColumnarExchange (30) + : +- CometExchange (30) : +- CometHashAggregate (29) : +- CometHashAggregate (28) : +- ReusedExchange (27) :- CometHashAggregate (36) - : +- CometColumnarExchange (35) + : +- CometExchange (35) : +- CometHashAggregate (34) : +- CometHashAggregate (33) : +- ReusedExchange (32) :- CometHashAggregate (41) - : +- CometColumnarExchange (40) + : +- CometExchange (40) : +- CometHashAggregate (39) : +- CometHashAggregate (38) : +- ReusedExchange (37) :- CometHashAggregate (46) - : +- CometColumnarExchange (45) + : +- CometExchange (45) : +- CometHashAggregate (44) : +- CometHashAggregate (43) : +- ReusedExchange (42) :- CometHashAggregate (51) - : +- CometColumnarExchange (50) + : +- CometExchange (50) : +- CometHashAggregate (49) : +- CometHashAggregate (48) : +- ReusedExchange (47) :- CometHashAggregate (56) - : +- CometColumnarExchange (55) + : +- CometExchange (55) : +- CometHashAggregate (54) : +- CometHashAggregate (53) : +- ReusedExchange (52) +- CometHashAggregate (61) - +- CometColumnarExchange (60) + +- CometExchange (60) +- CometHashAggregate (59) +- CometHashAggregate (58) +- ReusedExchange (57) @@ -162,9 +162,9 @@ Input [10]: [ss_quantity#3, ss_sales_price#4, d_year#9, d_moy#10, d_qoy#11, s_st Keys [8]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13] Functions [1]: [partial_sum(coalesce((ss_sales_price#4 * cast(ss_quantity#3 as decimal(10,0))), 0.00))] -(20) CometColumnarExchange +(20) CometExchange Input [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sum#19, isEmpty#20] -Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (21) CometHashAggregate Input [10]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sum#19, isEmpty#20] @@ -184,9 +184,9 @@ Input [8]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, Keys [7]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10] Functions [1]: [partial_sum(sumsales#23)] -(25) CometColumnarExchange +(25) CometExchange Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, sum#24, isEmpty#25] -Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (26) CometHashAggregate Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, sum#24, isEmpty#25] @@ -206,9 +206,9 @@ Input [7]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, Keys [6]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11] Functions [1]: [partial_sum(sumsales#23)] -(30) CometColumnarExchange +(30) CometExchange Input [8]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, sum#28, isEmpty#29] -Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (31) CometHashAggregate Input [8]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, sum#28, isEmpty#29] @@ -228,9 +228,9 @@ Input [6]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, Keys [5]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9] Functions [1]: [partial_sum(sumsales#23)] -(35) CometColumnarExchange +(35) CometExchange Input [7]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, sum#32, isEmpty#33] -Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (36) CometHashAggregate Input [7]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, sum#32, isEmpty#33] @@ -250,9 +250,9 @@ Input [5]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, sumsales#2 Keys [4]: [i_category#17, i_class#16, i_brand#15, i_product_name#18] Functions [1]: [partial_sum(sumsales#23)] -(40) CometColumnarExchange +(40) CometExchange Input [6]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, sum#36, isEmpty#37] -Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, i_product_name#18, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (41) CometHashAggregate Input [6]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, sum#36, isEmpty#37] @@ -272,9 +272,9 @@ Input [4]: [i_category#17, i_class#16, i_brand#15, sumsales#23] Keys [3]: [i_category#17, i_class#16, i_brand#15] Functions [1]: [partial_sum(sumsales#23)] -(45) CometColumnarExchange +(45) CometExchange Input [5]: [i_category#17, i_class#16, i_brand#15, sum#40, isEmpty#41] -Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(i_category#17, i_class#16, i_brand#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (46) CometHashAggregate Input [5]: [i_category#17, i_class#16, i_brand#15, sum#40, isEmpty#41] @@ -294,9 +294,9 @@ Input [3]: [i_category#17, i_class#16, sumsales#23] Keys [2]: [i_category#17, i_class#16] Functions [1]: [partial_sum(sumsales#23)] -(50) CometColumnarExchange +(50) CometExchange Input [4]: [i_category#17, i_class#16, sum#44, isEmpty#45] -Arguments: hashpartitioning(i_category#17, i_class#16, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(i_category#17, i_class#16, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (51) CometHashAggregate Input [4]: [i_category#17, i_class#16, sum#44, isEmpty#45] @@ -316,9 +316,9 @@ Input [2]: [i_category#17, sumsales#23] Keys [1]: [i_category#17] Functions [1]: [partial_sum(sumsales#23)] -(55) CometColumnarExchange +(55) CometExchange Input [3]: [i_category#17, sum#48, isEmpty#49] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (56) CometHashAggregate Input [3]: [i_category#17, sum#48, isEmpty#49] @@ -338,9 +338,9 @@ Input [1]: [sumsales#23] Keys: [] Functions [1]: [partial_sum(sumsales#23)] -(60) CometColumnarExchange +(60) CometExchange Input [2]: [sum#52, isEmpty#53] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (61) CometHashAggregate Input [2]: [sum#52, isEmpty#53] @@ -358,9 +358,9 @@ Child 6 Input [9]: [i_category#17, i_class#16, i_brand#75, i_product_name#76, d_ Child 7 Input [9]: [i_category#17, i_class#82, i_brand#83, i_product_name#84, d_year#85, d_qoy#86, d_moy#87, s_store_id#88, sumsales#89] Child 8 Input [9]: [i_category#90, i_class#91, i_brand#92, i_product_name#93, d_year#94, d_qoy#95, d_moy#96, s_store_id#97, sumsales#98] -(63) CometColumnarExchange +(63) CometExchange Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#54] -Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(i_category#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (64) CometSort Input [9]: [i_category#17, i_class#16, i_brand#15, i_product_name#18, d_year#9, d_qoy#11, d_moy#10, s_store_id#13, sumsales#54] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt index e631b569c..3216530ac 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q67a/simplified.txt @@ -7,10 +7,10 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ ColumnarToRow InputAdapter CometSort [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] - CometColumnarExchange [i_category] #1 + CometExchange [i_category] #1 CometUnion [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales] CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id] #2 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,ss_sales_price,ss_quantity] CometProject [ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_brand,i_class,i_category,i_product_name] CometBroadcastHashJoin [ss_item_sk,ss_quantity,ss_sales_price,d_year,d_moy,d_qoy,s_store_id,i_item_sk,i_brand,i_class,i_category,i_product_name] @@ -39,42 +39,42 @@ TakeOrderedAndProject [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_ CometFilter [i_item_sk,i_brand,i_class,i_category,i_product_name] CometScan parquet spark_catalog.default.item [i_item_sk,i_brand,i_class,i_category,i_product_name] CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #7 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy] #7 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sum,isEmpty,sumsales] CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,sumsales,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #8 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy] #8 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sum,isEmpty,sumsales] CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,sumsales,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name,d_year] #9 + CometExchange [i_category,i_class,i_brand,i_product_name,d_year] #9 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sum,isEmpty,sumsales] CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,sumsales,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class,i_brand,i_product_name] #10 + CometExchange [i_category,i_class,i_brand,i_product_name] #10 CometHashAggregate [i_category,i_class,i_brand,i_product_name,sum,isEmpty,sumsales] CometHashAggregate [i_category,i_class,i_brand,i_product_name,sumsales,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class,i_brand] #11 + CometExchange [i_category,i_class,i_brand] #11 CometHashAggregate [i_category,i_class,i_brand,sum,isEmpty,sumsales] CometHashAggregate [i_category,i_class,i_brand,sumsales,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category,i_class] #12 + CometExchange [i_category,i_class] #12 CometHashAggregate [i_category,i_class,sum,isEmpty,sumsales] CometHashAggregate [i_category,i_class,sumsales,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange [i_category] #13 + CometExchange [i_category] #13 CometHashAggregate [i_category,sum,isEmpty,sumsales] CometHashAggregate [i_category,sumsales,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 CometHashAggregate [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sumsales,sum,isEmpty,sum(sumsales)] - CometColumnarExchange #14 + CometExchange #14 CometHashAggregate [sum,isEmpty,sumsales] CometHashAggregate [sumsales,i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty,sum(coalesce((ss_sales_price * cast(ss_quantity as decimal(10,0))), 0.00))] ReusedExchange [i_category,i_class,i_brand,i_product_name,d_year,d_qoy,d_moy,s_store_id,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt index 28f77aff2..3b197ad04 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/explain.txt @@ -1,71 +1,61 @@ == Physical Plan == -TakeOrderedAndProject (67) -+- * Project (66) - +- Window (65) - +- * ColumnarToRow (64) - +- CometSort (63) - +- CometColumnarExchange (62) - +- CometHashAggregate (61) - +- CometColumnarExchange (60) - +- RowToColumnar (59) - +- * HashAggregate (58) - +- Union (57) - :- * HashAggregate (40) - : +- * ColumnarToRow (39) - : +- CometColumnarExchange (38) - : +- RowToColumnar (37) - : +- * HashAggregate (36) - : +- * Project (35) - : +- * BroadcastHashJoin Inner BuildRight (34) - : :- * ColumnarToRow (9) - : : +- CometProject (8) - : : +- CometBroadcastHashJoin (7) - : : :- CometFilter (2) - : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : +- CometBroadcastExchange (6) - : : +- CometProject (5) - : : +- CometFilter (4) - : : +- CometScan parquet spark_catalog.default.date_dim (3) - : +- BroadcastExchange (33) - : +- * BroadcastHashJoin LeftSemi BuildRight (32) - : :- * ColumnarToRow (12) - : : +- CometFilter (11) - : : +- CometScan parquet spark_catalog.default.store (10) - : +- BroadcastExchange (31) - : +- * Project (30) - : +- * Filter (29) - : +- Window (28) - : +- * ColumnarToRow (27) - : +- CometSort (26) - : +- CometHashAggregate (25) - : +- CometColumnarExchange (24) - : +- CometHashAggregate (23) - : +- CometProject (22) - : +- CometBroadcastHashJoin (21) - : :- CometProject (19) - : : +- CometBroadcastHashJoin (18) - : : :- CometFilter (14) - : : : +- CometScan parquet spark_catalog.default.store_sales (13) - : : +- CometBroadcastExchange (17) - : : +- CometFilter (16) - : : +- CometScan parquet spark_catalog.default.store (15) - : +- ReusedExchange (20) - :- * HashAggregate (48) - : +- * ColumnarToRow (47) - : +- CometColumnarExchange (46) - : +- RowToColumnar (45) - : +- * HashAggregate (44) - : +- * HashAggregate (43) - : +- * ColumnarToRow (42) - : +- ReusedExchange (41) - +- * HashAggregate (56) - +- * ColumnarToRow (55) - +- CometColumnarExchange (54) - +- RowToColumnar (53) - +- * HashAggregate (52) - +- * HashAggregate (51) - +- * ColumnarToRow (50) - +- ReusedExchange (49) +TakeOrderedAndProject (57) ++- * Project (56) + +- Window (55) + +- * Sort (54) + +- Exchange (53) + +- * HashAggregate (52) + +- Exchange (51) + +- * HashAggregate (50) + +- Union (49) + :- * HashAggregate (38) + : +- Exchange (37) + : +- * HashAggregate (36) + : +- * Project (35) + : +- * BroadcastHashJoin Inner BuildRight (34) + : :- * ColumnarToRow (9) + : : +- CometProject (8) + : : +- CometBroadcastHashJoin (7) + : : :- CometFilter (2) + : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : +- CometBroadcastExchange (6) + : : +- CometProject (5) + : : +- CometFilter (4) + : : +- CometScan parquet spark_catalog.default.date_dim (3) + : +- BroadcastExchange (33) + : +- * BroadcastHashJoin LeftSemi BuildRight (32) + : :- * ColumnarToRow (12) + : : +- CometFilter (11) + : : +- CometScan parquet spark_catalog.default.store (10) + : +- BroadcastExchange (31) + : +- * Project (30) + : +- * Filter (29) + : +- Window (28) + : +- * ColumnarToRow (27) + : +- CometSort (26) + : +- CometHashAggregate (25) + : +- CometExchange (24) + : +- CometHashAggregate (23) + : +- CometProject (22) + : +- CometBroadcastHashJoin (21) + : :- CometProject (19) + : : +- CometBroadcastHashJoin (18) + : : :- CometFilter (14) + : : : +- CometScan parquet spark_catalog.default.store_sales (13) + : : +- CometBroadcastExchange (17) + : : +- CometFilter (16) + : : +- CometScan parquet spark_catalog.default.store (15) + : +- ReusedExchange (20) + :- * HashAggregate (43) + : +- Exchange (42) + : +- * HashAggregate (41) + : +- * HashAggregate (40) + : +- ReusedExchange (39) + +- * HashAggregate (48) + +- Exchange (47) + +- * HashAggregate (46) + +- * HashAggregate (45) + +- ReusedExchange (44) (1) Scan parquet spark_catalog.default.store_sales @@ -178,9 +168,9 @@ Input [2]: [ss_net_profit#11, s_state#15] Keys [1]: [s_state#15] Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#11))] -(24) CometColumnarExchange +(24) CometExchange Input [2]: [s_state#15, sum#17] -Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_state#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (25) CometHashAggregate Input [2]: [s_state#15, sum#17] @@ -237,170 +227,142 @@ Functions [1]: [partial_sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum#20] Results [3]: [s_state#9, s_county#8, sum#21] -(37) RowToColumnar +(37) Exchange Input [3]: [s_state#9, s_county#8, sum#21] +Arguments: hashpartitioning(s_state#9, s_county#8, 5), ENSURE_REQUIREMENTS, [plan_id=4] -(38) CometColumnarExchange -Input [3]: [s_state#9, s_county#8, sum#21] -Arguments: hashpartitioning(s_state#9, s_county#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] - -(39) ColumnarToRow [codegen id : 5] -Input [3]: [s_state#9, s_county#8, sum#21] - -(40) HashAggregate [codegen id : 5] +(38) HashAggregate [codegen id : 5] Input [3]: [s_state#9, s_county#8, sum#21] Keys [2]: [s_state#9, s_county#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] Results [6]: [cast(MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) as decimal(27,2)) AS total_sum#23, s_state#9, s_county#8, 0 AS g_state#24, 0 AS g_county#25, 0 AS lochierarchy#26] -(41) ReusedExchange [Reuses operator id: 38] +(39) ReusedExchange [Reuses operator id: 37] Output [3]: [s_state#9, s_county#8, sum#27] -(42) ColumnarToRow [codegen id : 10] -Input [3]: [s_state#9, s_county#8, sum#27] - -(43) HashAggregate [codegen id : 10] +(40) HashAggregate [codegen id : 10] Input [3]: [s_state#9, s_county#8, sum#27] Keys [2]: [s_state#9, s_county#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] Results [2]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#28, s_state#9] -(44) HashAggregate [codegen id : 10] +(41) HashAggregate [codegen id : 10] Input [2]: [total_sum#28, s_state#9] Keys [1]: [s_state#9] Functions [1]: [partial_sum(total_sum#28)] Aggregate Attributes [2]: [sum#29, isEmpty#30] Results [3]: [s_state#9, sum#31, isEmpty#32] -(45) RowToColumnar +(42) Exchange Input [3]: [s_state#9, sum#31, isEmpty#32] +Arguments: hashpartitioning(s_state#9, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(46) CometColumnarExchange -Input [3]: [s_state#9, sum#31, isEmpty#32] -Arguments: hashpartitioning(s_state#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] - -(47) ColumnarToRow [codegen id : 11] -Input [3]: [s_state#9, sum#31, isEmpty#32] - -(48) HashAggregate [codegen id : 11] +(43) HashAggregate [codegen id : 11] Input [3]: [s_state#9, sum#31, isEmpty#32] Keys [1]: [s_state#9] Functions [1]: [sum(total_sum#28)] Aggregate Attributes [1]: [sum(total_sum#28)#33] Results [6]: [sum(total_sum#28)#33 AS total_sum#34, s_state#9, null AS s_county#35, 0 AS g_state#36, 1 AS g_county#37, 1 AS lochierarchy#38] -(49) ReusedExchange [Reuses operator id: 38] +(44) ReusedExchange [Reuses operator id: 37] Output [3]: [s_state#9, s_county#8, sum#39] -(50) ColumnarToRow [codegen id : 16] -Input [3]: [s_state#9, s_county#8, sum#39] - -(51) HashAggregate [codegen id : 16] +(45) HashAggregate [codegen id : 16] Input [3]: [s_state#9, s_county#8, sum#39] Keys [2]: [s_state#9, s_county#8] Functions [1]: [sum(UnscaledValue(ss_net_profit#2))] Aggregate Attributes [1]: [sum(UnscaledValue(ss_net_profit#2))#22] Results [1]: [MakeDecimal(sum(UnscaledValue(ss_net_profit#2))#22,17,2) AS total_sum#28] -(52) HashAggregate [codegen id : 16] +(46) HashAggregate [codegen id : 16] Input [1]: [total_sum#28] Keys: [] Functions [1]: [partial_sum(total_sum#28)] Aggregate Attributes [2]: [sum#40, isEmpty#41] Results [2]: [sum#42, isEmpty#43] -(53) RowToColumnar -Input [2]: [sum#42, isEmpty#43] - -(54) CometColumnarExchange -Input [2]: [sum#42, isEmpty#43] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] - -(55) ColumnarToRow [codegen id : 17] +(47) Exchange Input [2]: [sum#42, isEmpty#43] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=6] -(56) HashAggregate [codegen id : 17] +(48) HashAggregate [codegen id : 17] Input [2]: [sum#42, isEmpty#43] Keys: [] Functions [1]: [sum(total_sum#28)] Aggregate Attributes [1]: [sum(total_sum#28)#44] Results [6]: [sum(total_sum#28)#44 AS total_sum#45, null AS s_state#46, null AS s_county#47, 1 AS g_state#48, 1 AS g_county#49, 2 AS lochierarchy#50] -(57) Union +(49) Union -(58) HashAggregate [codegen id : 18] +(50) HashAggregate [codegen id : 18] Input [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] Keys [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] Functions: [] Aggregate Attributes: [] Results [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] -(59) RowToColumnar +(51) Exchange Input [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] +Arguments: hashpartitioning(total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26, 5), ENSURE_REQUIREMENTS, [plan_id=7] -(60) CometColumnarExchange -Input [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] -Arguments: hashpartitioning(total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] - -(61) CometHashAggregate +(52) HashAggregate [codegen id : 19] Input [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] Keys [6]: [total_sum#23, s_state#9, s_county#8, g_state#24, g_county#25, lochierarchy#26] Functions: [] +Aggregate Attributes: [] +Results [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, CASE WHEN (g_county#25 = 0) THEN s_state#9 END AS _w0#51] -(62) CometColumnarExchange -Input [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#51] -Arguments: hashpartitioning(lochierarchy#26, _w0#51, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(63) CometSort +(53) Exchange Input [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#51] -Arguments: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#51], [lochierarchy#26 ASC NULLS FIRST, _w0#51 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST] +Arguments: hashpartitioning(lochierarchy#26, _w0#51, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(64) ColumnarToRow [codegen id : 19] +(54) Sort [codegen id : 20] Input [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#51] +Arguments: [lochierarchy#26 ASC NULLS FIRST, _w0#51 ASC NULLS FIRST, total_sum#23 DESC NULLS LAST], false, 0 -(65) Window +(55) Window Input [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#51] Arguments: [rank(total_sum#23) windowspecdefinition(lochierarchy#26, _w0#51, total_sum#23 DESC NULLS LAST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS rank_within_parent#52], [lochierarchy#26, _w0#51], [total_sum#23 DESC NULLS LAST] -(66) Project [codegen id : 20] +(56) Project [codegen id : 21] Output [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, rank_within_parent#52] Input [6]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, _w0#51, rank_within_parent#52] -(67) TakeOrderedAndProject +(57) TakeOrderedAndProject Input [5]: [total_sum#23, s_state#9, s_county#8, lochierarchy#26, rank_within_parent#52] Arguments: 100, [lochierarchy#26 DESC NULLS LAST, CASE WHEN (lochierarchy#26 = 0) THEN s_state#9 END ASC NULLS FIRST, rank_within_parent#52 ASC NULLS FIRST], [total_sum#23, s_state#9, s_county#8, lochierarchy#26, rank_within_parent#52] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#3 IN dynamicpruning#4 -BroadcastExchange (72) -+- * ColumnarToRow (71) - +- CometProject (70) - +- CometFilter (69) - +- CometScan parquet spark_catalog.default.date_dim (68) +BroadcastExchange (62) ++- * ColumnarToRow (61) + +- CometProject (60) + +- CometFilter (59) + +- CometScan parquet spark_catalog.default.date_dim (58) -(68) Scan parquet spark_catalog.default.date_dim +(58) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#5, d_month_seq#6] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_month_seq), GreaterThanOrEqual(d_month_seq,1212), LessThanOrEqual(d_month_seq,1223), IsNotNull(d_date_sk)] ReadSchema: struct -(69) CometFilter +(59) CometFilter Input [2]: [d_date_sk#5, d_month_seq#6] Condition : (((isnotnull(d_month_seq#6) AND (d_month_seq#6 >= 1212)) AND (d_month_seq#6 <= 1223)) AND isnotnull(d_date_sk#5)) -(70) CometProject +(60) CometProject Input [2]: [d_date_sk#5, d_month_seq#6] Arguments: [d_date_sk#5], [d_date_sk#5] -(71) ColumnarToRow [codegen id : 1] +(61) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#5] -(72) BroadcastExchange +(62) BroadcastExchange Input [1]: [d_date_sk#5] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=9] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt index 16b9573cd..a1d9ab518 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q70a/simplified.txt @@ -1,102 +1,94 @@ TakeOrderedAndProject [lochierarchy,s_state,rank_within_parent,total_sum,s_county] - WholeStageCodegen (20) + WholeStageCodegen (21) Project [total_sum,s_state,s_county,lochierarchy,rank_within_parent] InputAdapter Window [total_sum,lochierarchy,_w0] - WholeStageCodegen (19) - ColumnarToRow + WholeStageCodegen (20) + Sort [lochierarchy,_w0,total_sum] InputAdapter - CometSort [total_sum,s_state,s_county,lochierarchy,_w0] - CometColumnarExchange [lochierarchy,_w0] #1 - CometHashAggregate [total_sum,s_state,s_county,lochierarchy,_w0,g_state,g_county] - CometColumnarExchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 - RowToColumnar + Exchange [lochierarchy,_w0] #1 + WholeStageCodegen (19) + HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] [_w0] + InputAdapter + Exchange [total_sum,s_state,s_county,g_state,g_county,lochierarchy] #2 WholeStageCodegen (18) HashAggregate [total_sum,s_state,s_county,g_state,g_county,lochierarchy] InputAdapter Union WholeStageCodegen (5) HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,g_state,g_county,lochierarchy,sum] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_state,s_county] #3 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] - Project [ss_net_profit,s_county,s_state] - BroadcastHashJoin [ss_store_sk,s_store_sk] - ColumnarToRow - InputAdapter - CometProject [ss_store_sk,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_month_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] - InputAdapter - BroadcastExchange #6 - WholeStageCodegen (3) - BroadcastHashJoin [s_state,s_state] - ColumnarToRow - InputAdapter - CometFilter [s_store_sk,s_county,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] - InputAdapter - BroadcastExchange #7 - WholeStageCodegen (2) - Project [s_state] - Filter [ranking] - InputAdapter - Window [_w0,s_state] - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometSort [s_state,_w0] - CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] - CometColumnarExchange [s_state] #8 - CometHashAggregate [s_state,sum,ss_net_profit] - CometProject [ss_net_profit,s_state] - CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] - CometProject [ss_net_profit,ss_sold_date_sk,s_state] - CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] - CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometBroadcastExchange [s_store_sk,s_state] #9 - CometFilter [s_store_sk,s_state] - CometScan parquet spark_catalog.default.store [s_store_sk,s_state] - ReusedExchange [d_date_sk] #5 + InputAdapter + Exchange [s_state,s_county] #3 + WholeStageCodegen (4) + HashAggregate [s_state,s_county,ss_net_profit] [sum,sum] + Project [ss_net_profit,s_county,s_state] + BroadcastHashJoin [ss_store_sk,s_store_sk] + ColumnarToRow + InputAdapter + CometProject [ss_store_sk,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_month_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_month_seq] + InputAdapter + BroadcastExchange #6 + WholeStageCodegen (3) + BroadcastHashJoin [s_state,s_state] + ColumnarToRow + InputAdapter + CometFilter [s_store_sk,s_county,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_county,s_state] + InputAdapter + BroadcastExchange #7 + WholeStageCodegen (2) + Project [s_state] + Filter [ranking] + InputAdapter + Window [_w0,s_state] + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometSort [s_state,_w0] + CometHashAggregate [s_state,_w0,sum,sum(UnscaledValue(ss_net_profit))] + CometExchange [s_state] #8 + CometHashAggregate [s_state,sum,ss_net_profit] + CometProject [ss_net_profit,s_state] + CometBroadcastHashJoin [ss_net_profit,ss_sold_date_sk,s_state,d_date_sk] + CometProject [ss_net_profit,ss_sold_date_sk,s_state] + CometBroadcastHashJoin [ss_store_sk,ss_net_profit,ss_sold_date_sk,s_store_sk,s_state] + CometFilter [ss_store_sk,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_net_profit,ss_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometBroadcastExchange [s_store_sk,s_state] #9 + CometFilter [s_store_sk,s_state] + CometScan parquet spark_catalog.default.store [s_store_sk,s_state] + ReusedExchange [d_date_sk] #5 WholeStageCodegen (11) HashAggregate [s_state,sum,isEmpty] [sum(total_sum),total_sum,s_county,g_state,g_county,lochierarchy,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_state] #10 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 + InputAdapter + Exchange [s_state] #10 + WholeStageCodegen (10) + HashAggregate [s_state,total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 WholeStageCodegen (17) HashAggregate [sum,isEmpty] [sum(total_sum),total_sum,s_state,s_county,g_state,g_county,lochierarchy,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #11 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] - HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] - ColumnarToRow - InputAdapter - ReusedExchange [s_state,s_county,sum] #3 + InputAdapter + Exchange #11 + WholeStageCodegen (16) + HashAggregate [total_sum] [sum,isEmpty,sum,isEmpty] + HashAggregate [s_state,s_county,sum] [sum(UnscaledValue(ss_net_profit)),total_sum,sum] + InputAdapter + ReusedExchange [s_state,s_county,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt index 53e1bb5a6..276e36240 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/explain.txt @@ -1,72 +1,71 @@ == Physical Plan == -* ColumnarToRow (68) -+- CometTakeOrderedAndProject (67) - +- CometHashAggregate (66) - +- CometColumnarExchange (65) - +- CometHashAggregate (64) - +- CometProject (63) - +- CometSortMergeJoin (62) - :- CometSort (56) - : +- CometColumnarExchange (55) - : +- RowToColumnar (54) - : +- * Project (53) - : +- * BroadcastHashJoin LeftOuter BuildRight (52) - : :- * Project (47) - : : +- * BroadcastHashJoin Inner BuildRight (46) - : : :- * ColumnarToRow (41) - : : : +- CometProject (40) - : : : +- CometBroadcastHashJoin (39) - : : : :- CometProject (35) - : : : : +- CometBroadcastHashJoin (34) - : : : : :- CometProject (29) - : : : : : +- CometBroadcastHashJoin (28) - : : : : : :- CometProject (23) - : : : : : : +- CometBroadcastHashJoin (22) - : : : : : : :- CometProject (17) - : : : : : : : +- CometBroadcastHashJoin (16) - : : : : : : : :- CometProject (12) - : : : : : : : : +- CometBroadcastHashJoin (11) - : : : : : : : : :- CometProject (7) - : : : : : : : : : +- CometBroadcastHashJoin (6) - : : : : : : : : : :- CometFilter (2) - : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) - : : : : : : : : : +- CometBroadcastExchange (5) - : : : : : : : : : +- CometFilter (4) - : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) - : : : : : : : : +- CometBroadcastExchange (10) - : : : : : : : : +- CometFilter (9) - : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) - : : : : : : : +- CometBroadcastExchange (15) - : : : : : : : +- CometFilter (14) - : : : : : : : +- CometScan parquet spark_catalog.default.item (13) - : : : : : : +- CometBroadcastExchange (21) - : : : : : : +- CometProject (20) - : : : : : : +- CometFilter (19) - : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) - : : : : : +- CometBroadcastExchange (27) - : : : : : +- CometProject (26) - : : : : : +- CometFilter (25) - : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) - : : : : +- CometBroadcastExchange (33) - : : : : +- CometProject (32) - : : : : +- CometFilter (31) - : : : : +- CometScan parquet spark_catalog.default.date_dim (30) - : : : +- CometBroadcastExchange (38) - : : : +- CometFilter (37) - : : : +- CometScan parquet spark_catalog.default.date_dim (36) - : : +- BroadcastExchange (45) - : : +- * ColumnarToRow (44) - : : +- CometFilter (43) - : : +- CometScan parquet spark_catalog.default.date_dim (42) - : +- BroadcastExchange (51) - : +- * ColumnarToRow (50) - : +- CometFilter (49) - : +- CometScan parquet spark_catalog.default.promotion (48) - +- CometSort (61) - +- CometColumnarExchange (60) - +- CometProject (59) - +- CometFilter (58) - +- CometScan parquet spark_catalog.default.catalog_returns (57) +TakeOrderedAndProject (67) ++- * HashAggregate (66) + +- Exchange (65) + +- * HashAggregate (64) + +- * Project (63) + +- * SortMergeJoin LeftOuter (62) + :- * Sort (55) + : +- Exchange (54) + : +- * Project (53) + : +- * BroadcastHashJoin LeftOuter BuildRight (52) + : :- * Project (47) + : : +- * BroadcastHashJoin Inner BuildRight (46) + : : :- * ColumnarToRow (41) + : : : +- CometProject (40) + : : : +- CometBroadcastHashJoin (39) + : : : :- CometProject (35) + : : : : +- CometBroadcastHashJoin (34) + : : : : :- CometProject (29) + : : : : : +- CometBroadcastHashJoin (28) + : : : : : :- CometProject (23) + : : : : : : +- CometBroadcastHashJoin (22) + : : : : : : :- CometProject (17) + : : : : : : : +- CometBroadcastHashJoin (16) + : : : : : : : :- CometProject (12) + : : : : : : : : +- CometBroadcastHashJoin (11) + : : : : : : : : :- CometProject (7) + : : : : : : : : : +- CometBroadcastHashJoin (6) + : : : : : : : : : :- CometFilter (2) + : : : : : : : : : : +- CometScan parquet spark_catalog.default.catalog_sales (1) + : : : : : : : : : +- CometBroadcastExchange (5) + : : : : : : : : : +- CometFilter (4) + : : : : : : : : : +- CometScan parquet spark_catalog.default.inventory (3) + : : : : : : : : +- CometBroadcastExchange (10) + : : : : : : : : +- CometFilter (9) + : : : : : : : : +- CometScan parquet spark_catalog.default.warehouse (8) + : : : : : : : +- CometBroadcastExchange (15) + : : : : : : : +- CometFilter (14) + : : : : : : : +- CometScan parquet spark_catalog.default.item (13) + : : : : : : +- CometBroadcastExchange (21) + : : : : : : +- CometProject (20) + : : : : : : +- CometFilter (19) + : : : : : : +- CometScan parquet spark_catalog.default.customer_demographics (18) + : : : : : +- CometBroadcastExchange (27) + : : : : : +- CometProject (26) + : : : : : +- CometFilter (25) + : : : : : +- CometScan parquet spark_catalog.default.household_demographics (24) + : : : : +- CometBroadcastExchange (33) + : : : : +- CometProject (32) + : : : : +- CometFilter (31) + : : : : +- CometScan parquet spark_catalog.default.date_dim (30) + : : : +- CometBroadcastExchange (38) + : : : +- CometFilter (37) + : : : +- CometScan parquet spark_catalog.default.date_dim (36) + : : +- BroadcastExchange (45) + : : +- * ColumnarToRow (44) + : : +- CometFilter (43) + : : +- CometScan parquet spark_catalog.default.date_dim (42) + : +- BroadcastExchange (51) + : +- * ColumnarToRow (50) + : +- CometFilter (49) + : +- CometScan parquet spark_catalog.default.promotion (48) + +- * ColumnarToRow (61) + +- CometSort (60) + +- CometExchange (59) + +- CometProject (58) + +- CometFilter (57) + +- CometScan parquet spark_catalog.default.catalog_returns (56) (1) Scan parquet spark_catalog.default.catalog_sales @@ -321,99 +320,101 @@ Join condition: None Output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_promo_sk#5, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, p_promo_sk#30] -(54) RowToColumnar +(54) Exchange Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] +Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, [plan_id=3] -(55) CometColumnarExchange +(55) Sort [codegen id : 4] Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: hashpartitioning(cs_item_sk#4, cs_order_number#6, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST], false, 0 -(56) CometSort -Input [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Arguments: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [cs_item_sk#4 ASC NULLS FIRST, cs_order_number#6 ASC NULLS FIRST] - -(57) Scan parquet spark_catalog.default.catalog_returns +(56) Scan parquet spark_catalog.default.catalog_returns Output [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Batched: true Location [not included in comparison]/{warehouse_dir}/catalog_returns] PushedFilters: [IsNotNull(cr_item_sk), IsNotNull(cr_order_number)] ReadSchema: struct -(58) CometFilter +(57) CometFilter Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Condition : (isnotnull(cr_item_sk#31) AND isnotnull(cr_order_number#32)) -(59) CometProject +(58) CometProject Input [3]: [cr_item_sk#31, cr_order_number#32, cr_returned_date_sk#33] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31, cr_order_number#32] -(60) CometColumnarExchange +(59) CometExchange Input [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(cr_item_sk#31, cr_order_number#32, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] -(61) CometSort +(60) CometSort Input [2]: [cr_item_sk#31, cr_order_number#32] Arguments: [cr_item_sk#31, cr_order_number#32], [cr_item_sk#31 ASC NULLS FIRST, cr_order_number#32 ASC NULLS FIRST] -(62) CometSortMergeJoin -Left output [5]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -Right output [2]: [cr_item_sk#31, cr_order_number#32] -Arguments: [cs_item_sk#4, cs_order_number#6], [cr_item_sk#31, cr_order_number#32], LeftOuter +(61) ColumnarToRow [codegen id : 5] +Input [2]: [cr_item_sk#31, cr_order_number#32] -(63) CometProject +(62) SortMergeJoin [codegen id : 6] +Left keys [2]: [cs_item_sk#4, cs_order_number#6] +Right keys [2]: [cr_item_sk#31, cr_order_number#32] +Join type: LeftOuter +Join condition: None + +(63) Project [codegen id : 6] +Output [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Input [7]: [cs_item_sk#4, cs_order_number#6, w_warehouse_name#15, i_item_desc#17, d_week_seq#24, cr_item_sk#31, cr_order_number#32] -Arguments: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24], [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] -(64) CometHashAggregate +(64) HashAggregate [codegen id : 6] Input [3]: [w_warehouse_name#15, i_item_desc#17, d_week_seq#24] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [partial_count(1)] +Aggregate Attributes [1]: [count#34] +Results [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] -(65) CometColumnarExchange -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] -Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +(65) Exchange +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] +Arguments: hashpartitioning(i_item_desc#17, w_warehouse_name#15, d_week_seq#24, 5), ENSURE_REQUIREMENTS, [plan_id=5] -(66) CometHashAggregate -Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#34] +(66) HashAggregate [codegen id : 7] +Input [4]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count#35] Keys [3]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24] Functions [1]: [count(1)] +Aggregate Attributes [1]: [count(1)#36] +Results [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, count(1)#36 AS no_promo#37, count(1)#36 AS promo#38, count(1)#36 AS total_cnt#39] -(67) CometTakeOrderedAndProject -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[total_cnt#37 DESC NULLS LAST,i_item_desc#17 ASC NULLS FIRST,w_warehouse_name#15 ASC NULLS FIRST,d_week_seq#24 ASC NULLS FIRST], output=[i_item_desc#17,w_warehouse_name#15,d_week_seq#24,no_promo#35,promo#36,total_cnt#37]), [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37], 100, [total_cnt#37 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] - -(68) ColumnarToRow [codegen id : 4] -Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#35, promo#36, total_cnt#37] +(67) TakeOrderedAndProject +Input [6]: [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] +Arguments: 100, [total_cnt#39 DESC NULLS LAST, i_item_desc#17 ASC NULLS FIRST, w_warehouse_name#15 ASC NULLS FIRST, d_week_seq#24 ASC NULLS FIRST], [i_item_desc#17, w_warehouse_name#15, d_week_seq#24, no_promo#37, promo#38, total_cnt#39] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = cs_sold_date_sk#8 IN dynamicpruning#9 -BroadcastExchange (73) -+- * ColumnarToRow (72) - +- CometProject (71) - +- CometFilter (70) - +- CometScan parquet spark_catalog.default.date_dim (69) +BroadcastExchange (72) ++- * ColumnarToRow (71) + +- CometProject (70) + +- CometFilter (69) + +- CometScan parquet spark_catalog.default.date_dim (68) -(69) Scan parquet spark_catalog.default.date_dim +(68) Scan parquet spark_catalog.default.date_dim Output [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_date_sk), IsNotNull(d_week_seq), IsNotNull(d_date)] ReadSchema: struct -(70) CometFilter +(69) CometFilter Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Condition : ((((isnotnull(d_year#25) AND (d_year#25 = 2001)) AND isnotnull(d_date_sk#22)) AND isnotnull(d_week_seq#24)) AND isnotnull(d_date#23)) -(71) CometProject +(70) CometProject Input [4]: [d_date_sk#22, d_date#23, d_week_seq#24, d_year#25] Arguments: [d_date_sk#22, d_date#23, d_week_seq#24], [d_date_sk#22, d_date#23, d_week_seq#24] -(72) ColumnarToRow [codegen id : 1] +(71) ColumnarToRow [codegen id : 1] Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] -(73) BroadcastExchange +(72) BroadcastExchange Input [3]: [d_date_sk#22, d_date#23, d_week_seq#24] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=6] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt index 1022e1a90..1efc2e5d0 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q72/simplified.txt @@ -1,86 +1,92 @@ -WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt] - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo,total_cnt,count,count(1)] - CometColumnarExchange [i_item_desc,w_warehouse_name,d_week_seq] #1 - CometHashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] - CometProject [w_warehouse_name,i_item_desc,d_week_seq] - CometSortMergeJoin [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq,cr_item_sk,cr_order_number] - CometSort [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - CometColumnarExchange [cs_item_sk,cs_order_number] #2 - RowToColumnar - WholeStageCodegen (3) - Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_promo_sk,p_promo_sk] - Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] - BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] - ColumnarToRow - InputAdapter - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] - CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] - CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] - CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] - CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #3 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 - CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] - CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 - CometFilter [w_warehouse_sk,w_warehouse_name] - CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] - CometBroadcastExchange [i_item_sk,i_item_desc] #6 - CometFilter [i_item_sk,i_item_desc] - CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] - CometBroadcastExchange [cd_demo_sk] #7 - CometProject [cd_demo_sk] - CometFilter [cd_demo_sk,cd_marital_status] - CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] - CometBroadcastExchange [hd_demo_sk] #8 - CometProject [hd_demo_sk] - CometFilter [hd_demo_sk,hd_buy_potential] - CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] - CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 - CometProject [d_date_sk,d_date,d_week_seq] - CometFilter [d_date_sk,d_date,d_week_seq,d_year] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] - CometBroadcastExchange [d_date_sk,d_week_seq] #10 - CometFilter [d_date_sk,d_week_seq] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] +TakeOrderedAndProject [total_cnt,i_item_desc,w_warehouse_name,d_week_seq,no_promo,promo] + WholeStageCodegen (7) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq,count] [count(1),no_promo,promo,total_cnt,count] + InputAdapter + Exchange [i_item_desc,w_warehouse_name,d_week_seq] #1 + WholeStageCodegen (6) + HashAggregate [i_item_desc,w_warehouse_name,d_week_seq] [count,count] + Project [w_warehouse_name,i_item_desc,d_week_seq] + SortMergeJoin [cs_item_sk,cs_order_number,cr_item_sk,cr_order_number] + InputAdapter + WholeStageCodegen (4) + Sort [cs_item_sk,cs_order_number] + InputAdapter + Exchange [cs_item_sk,cs_order_number] #2 + WholeStageCodegen (3) + Project [cs_item_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_promo_sk,p_promo_sk] + Project [cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_week_seq] + BroadcastHashJoin [cs_ship_date_sk,d_date_sk,d_date,d_date] + ColumnarToRow + InputAdapter + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq,d_date_sk,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,inv_date_sk,w_warehouse_name,i_item_desc,d_date,d_week_seq] + CometBroadcastHashJoin [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,d_date_sk,d_date,d_week_seq] + CometProject [cs_ship_date_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,hd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc,cd_demo_sk] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_desc] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name,i_item_sk,i_item_desc] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_date_sk,w_warehouse_name] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk,w_warehouse_sk,w_warehouse_name] + CometProject [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_sold_date_sk,inv_warehouse_sk,inv_date_sk] + CometBroadcastHashJoin [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk,inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometFilter [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_ship_date_sk,cs_bill_cdemo_sk,cs_bill_hdemo_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_quantity,cs_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #3 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] #4 + CometFilter [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometScan parquet spark_catalog.default.inventory [inv_item_sk,inv_warehouse_sk,inv_quantity_on_hand,inv_date_sk] + CometBroadcastExchange [w_warehouse_sk,w_warehouse_name] #5 + CometFilter [w_warehouse_sk,w_warehouse_name] + CometScan parquet spark_catalog.default.warehouse [w_warehouse_sk,w_warehouse_name] + CometBroadcastExchange [i_item_sk,i_item_desc] #6 + CometFilter [i_item_sk,i_item_desc] + CometScan parquet spark_catalog.default.item [i_item_sk,i_item_desc] + CometBroadcastExchange [cd_demo_sk] #7 + CometProject [cd_demo_sk] + CometFilter [cd_demo_sk,cd_marital_status] + CometScan parquet spark_catalog.default.customer_demographics [cd_demo_sk,cd_marital_status] + CometBroadcastExchange [hd_demo_sk] #8 + CometProject [hd_demo_sk] + CometFilter [hd_demo_sk,hd_buy_potential] + CometScan parquet spark_catalog.default.household_demographics [hd_demo_sk,hd_buy_potential] + CometBroadcastExchange [d_date_sk,d_date,d_week_seq] #9 + CometProject [d_date_sk,d_date,d_week_seq] + CometFilter [d_date_sk,d_date,d_week_seq,d_year] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date,d_week_seq,d_year] + CometBroadcastExchange [d_date_sk,d_week_seq] #10 + CometFilter [d_date_sk,d_week_seq] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_week_seq] + InputAdapter + BroadcastExchange #11 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] InputAdapter - BroadcastExchange #11 - WholeStageCodegen (1) + BroadcastExchange #12 + WholeStageCodegen (2) ColumnarToRow InputAdapter - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - InputAdapter - BroadcastExchange #12 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometFilter [p_promo_sk] - CometScan parquet spark_catalog.default.promotion [p_promo_sk] - CometSort [cr_item_sk,cr_order_number] - CometColumnarExchange [cr_item_sk,cr_order_number] #13 - CometProject [cr_item_sk,cr_order_number] - CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometFilter [p_promo_sk] + CometScan parquet spark_catalog.default.promotion [p_promo_sk] + InputAdapter + WholeStageCodegen (5) + ColumnarToRow + InputAdapter + CometSort [cr_item_sk,cr_order_number] + CometExchange [cr_item_sk,cr_order_number] #13 + CometProject [cr_item_sk,cr_order_number] + CometFilter [cr_item_sk,cr_order_number,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_returned_date_sk] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt index e92f554c5..4072b2277 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/explain.txt @@ -8,7 +8,7 @@ : :- CometBroadcastHashJoin (33) : : :- CometFilter (16) : : : +- CometHashAggregate (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometHashAggregate (13) : : : +- CometProject (12) : : : +- CometBroadcastHashJoin (11) @@ -24,7 +24,7 @@ : : : +- CometScan parquet spark_catalog.default.date_dim (8) : : +- CometBroadcastExchange (32) : : +- CometHashAggregate (31) - : : +- CometColumnarExchange (30) + : : +- CometExchange (30) : : +- CometHashAggregate (29) : : +- CometProject (28) : : +- CometBroadcastHashJoin (27) @@ -41,7 +41,7 @@ : +- CometBroadcastExchange (48) : +- CometFilter (47) : +- CometHashAggregate (46) - : +- CometColumnarExchange (45) + : +- CometExchange (45) : +- CometHashAggregate (44) : +- CometProject (43) : +- CometBroadcastHashJoin (42) @@ -55,7 +55,7 @@ : +- ReusedExchange (41) +- CometBroadcastExchange (64) +- CometHashAggregate (63) - +- CometColumnarExchange (62) + +- CometExchange (62) +- CometHashAggregate (61) +- CometProject (60) +- CometBroadcastHashJoin (59) @@ -134,9 +134,9 @@ Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, ss_net_paid#6, d_yea Keys [4]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#6))] -(14) CometColumnarExchange +(14) CometExchange Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#11] -Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometHashAggregate Input [5]: [c_customer_id#2, c_first_name#3, c_last_name#4, d_year#10, sum#11] @@ -212,9 +212,9 @@ Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, ss_net_paid#19, d Keys [4]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23] Functions [1]: [partial_sum(UnscaledValue(ss_net_paid#19))] -(30) CometColumnarExchange +(30) CometExchange Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, sum#24] -Arguments: hashpartitioning(c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (31) CometHashAggregate Input [5]: [c_customer_id#15, c_first_name#16, c_last_name#17, d_year#23, sum#24] @@ -283,9 +283,9 @@ Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, ws_net_paid#34, d Keys [4]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#34))] -(45) CometColumnarExchange +(45) CometExchange Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, sum#39] -Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (46) CometHashAggregate Input [5]: [c_customer_id#30, c_first_name#31, c_last_name#32, d_year#38, sum#39] @@ -362,9 +362,9 @@ Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, ws_net_paid#47, d Keys [4]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#47))] -(62) CometColumnarExchange +(62) CometExchange Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, sum#52] -Arguments: hashpartitioning(c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (63) CometHashAggregate Input [5]: [c_customer_id#43, c_first_name#44, c_last_name#45, d_year#51, sum#52] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt index 0b288df94..631a82f1e 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q74/simplified.txt @@ -9,7 +9,7 @@ WholeStageCodegen (1) CometBroadcastHashJoin [customer_id,year_total,customer_id,customer_first_name,customer_last_name,year_total] CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #1 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -32,7 +32,7 @@ WholeStageCodegen (1) CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometBroadcastExchange [customer_id,customer_first_name,customer_last_name,year_total] #5 CometHashAggregate [customer_id,customer_first_name,customer_last_name,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ss_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #6 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ss_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ss_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ss_net_paid,ss_sold_date_sk,d_date_sk,d_year] @@ -56,7 +56,7 @@ WholeStageCodegen (1) CometBroadcastExchange [customer_id,year_total] #10 CometFilter [customer_id,year_total] CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #11 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] @@ -71,7 +71,7 @@ WholeStageCodegen (1) ReusedExchange [d_date_sk,d_year] #4 CometBroadcastExchange [customer_id,year_total] #13 CometHashAggregate [customer_id,year_total,c_customer_id,c_first_name,c_last_name,d_year,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 + CometExchange [c_customer_id,c_first_name,c_last_name,d_year] #14 CometHashAggregate [c_customer_id,c_first_name,c_last_name,d_year,sum,ws_net_paid] CometProject [c_customer_id,c_first_name,c_last_name,ws_net_paid,d_year] CometBroadcastHashJoin [c_customer_id,c_first_name,c_last_name,ws_net_paid,ws_sold_date_sk,d_date_sk,d_year] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt index 2aeff8de1..68deab557 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/explain.txt @@ -4,19 +4,19 @@ TakeOrderedAndProject (125) +- * SortMergeJoin Inner (123) :- * ColumnarToRow (67) : +- CometSort (66) - : +- CometColumnarExchange (65) + : +- CometExchange (65) : +- CometFilter (64) : +- CometHashAggregate (63) - : +- CometColumnarExchange (62) + : +- CometExchange (62) : +- CometHashAggregate (61) : +- CometHashAggregate (60) - : +- CometColumnarExchange (59) + : +- CometExchange (59) : +- CometHashAggregate (58) : +- CometUnion (57) : :- CometProject (22) : : +- CometSortMergeJoin (21) : : :- CometSort (15) - : : : +- CometColumnarExchange (14) + : : : +- CometExchange (14) : : : +- CometProject (13) : : : +- CometBroadcastHashJoin (12) : : : :- CometProject (8) @@ -31,14 +31,14 @@ TakeOrderedAndProject (125) : : : +- CometFilter (10) : : : +- CometScan parquet spark_catalog.default.date_dim (9) : : +- CometSort (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometProject (18) : : +- CometFilter (17) : : +- CometScan parquet spark_catalog.default.catalog_returns (16) : :- CometProject (39) : : +- CometSortMergeJoin (38) : : :- CometSort (32) - : : : +- CometColumnarExchange (31) + : : : +- CometExchange (31) : : : +- CometProject (30) : : : +- CometBroadcastHashJoin (29) : : : :- CometProject (27) @@ -48,14 +48,14 @@ TakeOrderedAndProject (125) : : : : +- ReusedExchange (25) : : : +- ReusedExchange (28) : : +- CometSort (37) - : : +- CometColumnarExchange (36) + : : +- CometExchange (36) : : +- CometProject (35) : : +- CometFilter (34) : : +- CometScan parquet spark_catalog.default.store_returns (33) : +- CometProject (56) : +- CometSortMergeJoin (55) : :- CometSort (49) - : : +- CometColumnarExchange (48) + : : +- CometExchange (48) : : +- CometProject (47) : : +- CometBroadcastHashJoin (46) : : :- CometProject (44) @@ -65,25 +65,25 @@ TakeOrderedAndProject (125) : : : +- ReusedExchange (42) : : +- ReusedExchange (45) : +- CometSort (54) - : +- CometColumnarExchange (53) + : +- CometExchange (53) : +- CometProject (52) : +- CometFilter (51) : +- CometScan parquet spark_catalog.default.web_returns (50) +- * ColumnarToRow (122) +- CometSort (121) - +- CometColumnarExchange (120) + +- CometExchange (120) +- CometFilter (119) +- CometHashAggregate (118) - +- CometColumnarExchange (117) + +- CometExchange (117) +- CometHashAggregate (116) +- CometHashAggregate (115) - +- CometColumnarExchange (114) + +- CometExchange (114) +- CometHashAggregate (113) +- CometUnion (112) :- CometProject (83) : +- CometSortMergeJoin (82) : :- CometSort (79) - : : +- CometColumnarExchange (78) + : : +- CometExchange (78) : : +- CometProject (77) : : +- CometBroadcastHashJoin (76) : : :- CometProject (72) @@ -99,7 +99,7 @@ TakeOrderedAndProject (125) :- CometProject (97) : +- CometSortMergeJoin (96) : :- CometSort (93) - : : +- CometColumnarExchange (92) + : : +- CometExchange (92) : : +- CometProject (91) : : +- CometBroadcastHashJoin (90) : : :- CometProject (88) @@ -113,7 +113,7 @@ TakeOrderedAndProject (125) +- CometProject (111) +- CometSortMergeJoin (110) :- CometSort (107) - : +- CometColumnarExchange (106) + : +- CometExchange (106) : +- CometProject (105) : +- CometBroadcastHashJoin (104) : :- CometProject (102) @@ -190,9 +190,9 @@ Arguments: [cs_sold_date_sk#5], [d_date_sk#13], Inner, BuildRight Input [11]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, cs_sold_date_sk#5, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_date_sk#13, d_year#14] Arguments: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14], [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -(14) CometColumnarExchange +(14) CometExchange Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] -Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(cs_order_number#2, cs_item_sk#1, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (15) CometSort Input [9]: [cs_item_sk#1, cs_order_number#2, cs_quantity#3, cs_ext_sales_price#4, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, d_year#14] @@ -213,9 +213,9 @@ Condition : (isnotnull(cr_order_number#16) AND isnotnull(cr_item_sk#15)) Input [5]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18, cr_returned_date_sk#19] Arguments: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18], [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -(19) CometColumnarExchange +(19) CometExchange Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] -Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(cr_order_number#16, cr_item_sk#15, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (20) CometSort Input [4]: [cr_item_sk#15, cr_order_number#16, cr_return_quantity#17, cr_return_amount#18] @@ -266,9 +266,9 @@ Arguments: [ss_sold_date_sk#26], [d_date_sk#33], Inner, BuildRight Input [11]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, ss_sold_date_sk#26, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_date_sk#33, d_year#34] Arguments: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34], [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -(31) CometColumnarExchange +(31) CometExchange Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] -Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(ss_ticket_number#23, ss_item_sk#22, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (32) CometSort Input [9]: [ss_item_sk#22, ss_ticket_number#23, ss_quantity#24, ss_ext_sales_price#25, i_brand_id#29, i_class_id#30, i_category_id#31, i_manufact_id#32, d_year#34] @@ -289,9 +289,9 @@ Condition : (isnotnull(sr_ticket_number#36) AND isnotnull(sr_item_sk#35)) Input [5]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38, sr_returned_date_sk#39] Arguments: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38], [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -(36) CometColumnarExchange +(36) CometExchange Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] -Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(sr_ticket_number#36, sr_item_sk#35, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (37) CometSort Input [4]: [sr_item_sk#35, sr_ticket_number#36, sr_return_quantity#37, sr_return_amt#38] @@ -342,9 +342,9 @@ Arguments: [ws_sold_date_sk#46], [d_date_sk#53], Inner, BuildRight Input [11]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, ws_sold_date_sk#46, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_date_sk#53, d_year#54] Arguments: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54], [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -(48) CometColumnarExchange +(48) CometExchange Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] -Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(ws_order_number#43, ws_item_sk#42, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (49) CometSort Input [9]: [ws_item_sk#42, ws_order_number#43, ws_quantity#44, ws_ext_sales_price#45, i_brand_id#49, i_class_id#50, i_category_id#51, i_manufact_id#52, d_year#54] @@ -365,9 +365,9 @@ Condition : (isnotnull(wr_order_number#56) AND isnotnull(wr_item_sk#55)) Input [5]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58, wr_returned_date_sk#59] Arguments: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58], [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -(53) CometColumnarExchange +(53) CometExchange Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] -Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(wr_order_number#56, wr_item_sk#55, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (54) CometSort Input [4]: [wr_item_sk#55, wr_order_number#56, wr_return_quantity#57, wr_return_amt#58] @@ -392,9 +392,9 @@ Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_ Keys [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] Functions: [] -(59) CometColumnarExchange +(59) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (60) CometHashAggregate Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#20, sales_amt#21] @@ -406,9 +406,9 @@ Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_ Keys [5]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -(62) CometColumnarExchange +(62) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] -Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] +Arguments: hashpartitioning(d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=8] (63) CometHashAggregate Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sum#62, sum#63] @@ -419,9 +419,9 @@ Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] Condition : isnotnull(sales_cnt#64) -(65) CometColumnarExchange +(65) CometExchange Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] -Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] +Arguments: hashpartitioning(i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=9] (66) CometSort Input [7]: [d_year#14, i_brand_id#8, i_class_id#9, i_category_id#10, i_manufact_id#12, sales_cnt#64, sales_amt#65] @@ -478,9 +478,9 @@ Arguments: [cs_sold_date_sk#70], [d_date_sk#77], Inner, BuildRight Input [11]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, cs_sold_date_sk#70, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_date_sk#77, d_year#78] Arguments: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78], [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -(78) CometColumnarExchange +(78) CometExchange Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] -Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: hashpartitioning(cs_order_number#67, cs_item_sk#66, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=10] (79) CometSort Input [9]: [cs_item_sk#66, cs_order_number#67, cs_quantity#68, cs_ext_sales_price#69, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, d_year#78] @@ -538,9 +538,9 @@ Arguments: [ss_sold_date_sk#87], [d_date_sk#94], Inner, BuildRight Input [11]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, ss_sold_date_sk#87, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_date_sk#94, d_year#95] Arguments: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95], [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -(92) CometColumnarExchange +(92) CometExchange Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] -Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] +Arguments: hashpartitioning(ss_ticket_number#84, ss_item_sk#83, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=11] (93) CometSort Input [9]: [ss_item_sk#83, ss_ticket_number#84, ss_quantity#85, ss_ext_sales_price#86, i_brand_id#90, i_class_id#91, i_category_id#92, i_manufact_id#93, d_year#95] @@ -598,9 +598,9 @@ Arguments: [ws_sold_date_sk#104], [d_date_sk#111], Inner, BuildRight Input [11]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, ws_sold_date_sk#104, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_date_sk#111, d_year#112] Arguments: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112], [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -(106) CometColumnarExchange +(106) CometExchange Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] -Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=12] +Arguments: hashpartitioning(ws_order_number#101, ws_item_sk#100, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=12] (107) CometSort Input [9]: [ws_item_sk#100, ws_order_number#101, ws_quantity#102, ws_ext_sales_price#103, i_brand_id#107, i_class_id#108, i_category_id#109, i_manufact_id#110, d_year#112] @@ -632,9 +632,9 @@ Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufac Keys [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] Functions: [] -(114) CometColumnarExchange +(114) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=13] +Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=13] (115) CometHashAggregate Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#20, sales_amt#21] @@ -646,9 +646,9 @@ Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufac Keys [5]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76] Functions [2]: [partial_sum(sales_cnt#20), partial_sum(UnscaledValue(sales_amt#21))] -(117) CometColumnarExchange +(117) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] -Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=14] +Arguments: hashpartitioning(d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=14] (118) CometHashAggregate Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sum#62, sum#117] @@ -659,9 +659,9 @@ Functions [2]: [sum(sales_cnt#20), sum(UnscaledValue(sales_amt#21))] Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] Condition : isnotnull(sales_cnt#118) -(120) CometColumnarExchange +(120) CometExchange Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] -Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=15] +Arguments: hashpartitioning(i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=15] (121) CometSort Input [7]: [d_year#78, i_brand_id#73, i_class_id#74, i_category_id#75, i_manufact_id#76, sales_cnt#118, sales_amt#119] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt index 9b187bca1..60df69dea 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q75/simplified.txt @@ -7,19 +7,19 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #1 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #2 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #3 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #4 + CometExchange [cs_order_number,cs_item_sk] #4 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -41,14 +41,14 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i CometFilter [d_date_sk,d_year] CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_year] CometSort [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] - CometColumnarExchange [cr_order_number,cr_item_sk] #8 + CometExchange [cr_order_number,cr_item_sk] #8 CometProject [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometFilter [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount,cr_returned_date_sk] CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #9 + CometExchange [ss_ticket_number,ss_item_sk] #9 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -59,14 +59,14 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 ReusedExchange [d_date_sk,d_year] #7 CometSort [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] - CometColumnarExchange [sr_ticket_number,sr_item_sk] #10 + CometExchange [sr_ticket_number,sr_item_sk] #10 CometProject [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometFilter [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt,sr_returned_date_sk] CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #11 + CometExchange [ws_order_number,ws_item_sk] #11 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -77,7 +77,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ReusedExchange [i_item_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] #6 ReusedExchange [d_date_sk,d_year] #7 CometSort [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] - CometColumnarExchange [wr_order_number,wr_item_sk] #12 + CometExchange [wr_order_number,wr_item_sk] #12 CometProject [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometFilter [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt,wr_returned_date_sk] @@ -86,19 +86,19 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i ColumnarToRow InputAdapter CometSort [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 + CometExchange [i_brand_id,i_class_id,i_category_id,i_manufact_id] #13 CometFilter [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt,sum,sum,sum(sales_cnt),sum(UnscaledValue(sales_amt))] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id] #14 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sum,sum,sales_cnt,sales_amt] CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] - CometColumnarExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 + CometExchange [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] #15 CometHashAggregate [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometUnion [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometProject [cs_quantity,cr_return_quantity,cs_ext_sales_price,cr_return_amount] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,cr_item_sk,cr_order_number,cr_return_quantity,cr_return_amount] CometSort [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [cs_order_number,cs_item_sk] #16 + CometExchange [cs_order_number,cs_item_sk] #16 CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [cs_item_sk,cs_order_number,cs_quantity,cs_ext_sales_price,cs_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -121,7 +121,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i CometProject [ss_quantity,sr_return_quantity,ss_ext_sales_price,sr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,sr_item_sk,sr_ticket_number,sr_return_quantity,sr_return_amt] CometSort [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ss_ticket_number,ss_item_sk] #19 + CometExchange [ss_ticket_number,ss_item_sk] #19 CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ss_item_sk,ss_ticket_number,ss_quantity,ss_ext_sales_price,ss_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] @@ -136,7 +136,7 @@ TakeOrderedAndProject [sales_cnt_diff,sales_amt_diff,prev_year,year,i_brand_id,i CometProject [ws_quantity,wr_return_quantity,ws_ext_sales_price,wr_return_amt] [d_year,i_brand_id,i_class_id,i_category_id,i_manufact_id,sales_cnt,sales_amt] CometSortMergeJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year,wr_item_sk,wr_order_number,wr_return_quantity,wr_return_amt] CometSort [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] - CometColumnarExchange [ws_order_number,ws_item_sk] #20 + CometExchange [ws_order_number,ws_item_sk] #20 CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_year] CometBroadcastHashJoin [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id,d_date_sk,d_year] CometProject [ws_item_sk,ws_order_number,ws_quantity,ws_ext_sales_price,ws_sold_date_sk,i_brand_id,i_class_id,i_category_id,i_manufact_id] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt index ec37a5df5..48b11768d 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/explain.txt @@ -1,111 +1,101 @@ == Physical Plan == -* ColumnarToRow (107) -+- CometTakeOrderedAndProject (106) - +- CometHashAggregate (105) - +- CometColumnarExchange (104) - +- RowToColumnar (103) - +- * HashAggregate (102) - +- Union (101) - :- * HashAggregate (84) - : +- * ColumnarToRow (83) - : +- CometColumnarExchange (82) - : +- RowToColumnar (81) - : +- * HashAggregate (80) - : +- Union (79) - : :- * ColumnarToRow (31) - : : +- CometProject (30) - : : +- CometBroadcastHashJoin (29) - : : :- CometHashAggregate (16) - : : : +- CometColumnarExchange (15) - : : : +- CometHashAggregate (14) - : : : +- CometProject (13) - : : : +- CometBroadcastHashJoin (12) - : : : :- CometProject (8) - : : : : +- CometBroadcastHashJoin (7) - : : : : :- CometFilter (2) - : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) - : : : : +- CometBroadcastExchange (6) - : : : : +- CometProject (5) - : : : : +- CometFilter (4) - : : : : +- CometScan parquet spark_catalog.default.date_dim (3) - : : : +- CometBroadcastExchange (11) - : : : +- CometFilter (10) - : : : +- CometScan parquet spark_catalog.default.store (9) - : : +- CometBroadcastExchange (28) - : : +- CometHashAggregate (27) - : : +- CometColumnarExchange (26) - : : +- CometHashAggregate (25) - : : +- CometProject (24) - : : +- CometBroadcastHashJoin (23) - : : :- CometProject (21) - : : : +- CometBroadcastHashJoin (20) - : : : :- CometFilter (18) - : : : : +- CometScan parquet spark_catalog.default.store_returns (17) - : : : +- ReusedExchange (19) - : : +- ReusedExchange (22) - : :- * Project (50) - : : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) - : : :- BroadcastExchange (40) - : : : +- * ColumnarToRow (39) - : : : +- CometHashAggregate (38) - : : : +- CometColumnarExchange (37) - : : : +- CometHashAggregate (36) - : : : +- CometProject (35) - : : : +- CometBroadcastHashJoin (34) - : : : :- CometScan parquet spark_catalog.default.catalog_sales (32) - : : : +- ReusedExchange (33) - : : +- * ColumnarToRow (48) - : : +- CometHashAggregate (47) - : : +- CometColumnarExchange (46) - : : +- CometHashAggregate (45) - : : +- CometProject (44) - : : +- CometBroadcastHashJoin (43) - : : :- CometScan parquet spark_catalog.default.catalog_returns (41) - : : +- ReusedExchange (42) - : +- * ColumnarToRow (78) - : +- CometProject (77) - : +- CometBroadcastHashJoin (76) - : :- CometHashAggregate (63) - : : +- CometColumnarExchange (62) - : : +- CometHashAggregate (61) - : : +- CometProject (60) - : : +- CometBroadcastHashJoin (59) - : : :- CometProject (55) - : : : +- CometBroadcastHashJoin (54) - : : : :- CometFilter (52) - : : : : +- CometScan parquet spark_catalog.default.web_sales (51) - : : : +- ReusedExchange (53) - : : +- CometBroadcastExchange (58) - : : +- CometFilter (57) - : : +- CometScan parquet spark_catalog.default.web_page (56) - : +- CometBroadcastExchange (75) - : +- CometHashAggregate (74) - : +- CometColumnarExchange (73) - : +- CometHashAggregate (72) - : +- CometProject (71) - : +- CometBroadcastHashJoin (70) - : :- CometProject (68) - : : +- CometBroadcastHashJoin (67) - : : :- CometFilter (65) - : : : +- CometScan parquet spark_catalog.default.web_returns (64) - : : +- ReusedExchange (66) - : +- ReusedExchange (69) - :- * HashAggregate (92) - : +- * ColumnarToRow (91) - : +- CometColumnarExchange (90) - : +- RowToColumnar (89) - : +- * HashAggregate (88) - : +- * HashAggregate (87) - : +- * ColumnarToRow (86) - : +- ReusedExchange (85) - +- * HashAggregate (100) - +- * ColumnarToRow (99) - +- CometColumnarExchange (98) - +- RowToColumnar (97) - +- * HashAggregate (96) - +- * HashAggregate (95) - +- * ColumnarToRow (94) - +- ReusedExchange (93) +TakeOrderedAndProject (97) ++- * HashAggregate (96) + +- Exchange (95) + +- * HashAggregate (94) + +- Union (93) + :- * HashAggregate (82) + : +- Exchange (81) + : +- * HashAggregate (80) + : +- Union (79) + : :- * ColumnarToRow (31) + : : +- CometProject (30) + : : +- CometBroadcastHashJoin (29) + : : :- CometHashAggregate (16) + : : : +- CometExchange (15) + : : : +- CometHashAggregate (14) + : : : +- CometProject (13) + : : : +- CometBroadcastHashJoin (12) + : : : :- CometProject (8) + : : : : +- CometBroadcastHashJoin (7) + : : : : :- CometFilter (2) + : : : : : +- CometScan parquet spark_catalog.default.store_sales (1) + : : : : +- CometBroadcastExchange (6) + : : : : +- CometProject (5) + : : : : +- CometFilter (4) + : : : : +- CometScan parquet spark_catalog.default.date_dim (3) + : : : +- CometBroadcastExchange (11) + : : : +- CometFilter (10) + : : : +- CometScan parquet spark_catalog.default.store (9) + : : +- CometBroadcastExchange (28) + : : +- CometHashAggregate (27) + : : +- CometExchange (26) + : : +- CometHashAggregate (25) + : : +- CometProject (24) + : : +- CometBroadcastHashJoin (23) + : : :- CometProject (21) + : : : +- CometBroadcastHashJoin (20) + : : : :- CometFilter (18) + : : : : +- CometScan parquet spark_catalog.default.store_returns (17) + : : : +- ReusedExchange (19) + : : +- ReusedExchange (22) + : :- * Project (50) + : : +- * BroadcastNestedLoopJoin Inner BuildLeft (49) + : : :- BroadcastExchange (40) + : : : +- * ColumnarToRow (39) + : : : +- CometHashAggregate (38) + : : : +- CometExchange (37) + : : : +- CometHashAggregate (36) + : : : +- CometProject (35) + : : : +- CometBroadcastHashJoin (34) + : : : :- CometScan parquet spark_catalog.default.catalog_sales (32) + : : : +- ReusedExchange (33) + : : +- * ColumnarToRow (48) + : : +- CometHashAggregate (47) + : : +- CometExchange (46) + : : +- CometHashAggregate (45) + : : +- CometProject (44) + : : +- CometBroadcastHashJoin (43) + : : :- CometScan parquet spark_catalog.default.catalog_returns (41) + : : +- ReusedExchange (42) + : +- * ColumnarToRow (78) + : +- CometProject (77) + : +- CometBroadcastHashJoin (76) + : :- CometHashAggregate (63) + : : +- CometExchange (62) + : : +- CometHashAggregate (61) + : : +- CometProject (60) + : : +- CometBroadcastHashJoin (59) + : : :- CometProject (55) + : : : +- CometBroadcastHashJoin (54) + : : : :- CometFilter (52) + : : : : +- CometScan parquet spark_catalog.default.web_sales (51) + : : : +- ReusedExchange (53) + : : +- CometBroadcastExchange (58) + : : +- CometFilter (57) + : : +- CometScan parquet spark_catalog.default.web_page (56) + : +- CometBroadcastExchange (75) + : +- CometHashAggregate (74) + : +- CometExchange (73) + : +- CometHashAggregate (72) + : +- CometProject (71) + : +- CometBroadcastHashJoin (70) + : :- CometProject (68) + : : +- CometBroadcastHashJoin (67) + : : :- CometFilter (65) + : : : +- CometScan parquet spark_catalog.default.web_returns (64) + : : +- ReusedExchange (66) + : +- ReusedExchange (69) + :- * HashAggregate (87) + : +- Exchange (86) + : +- * HashAggregate (85) + : +- * HashAggregate (84) + : +- ReusedExchange (83) + +- * HashAggregate (92) + +- Exchange (91) + +- * HashAggregate (90) + +- * HashAggregate (89) + +- ReusedExchange (88) (1) Scan parquet spark_catalog.default.store_sales @@ -177,9 +167,9 @@ Input [3]: [ss_ext_sales_price#2, ss_net_profit#3, s_store_sk#8] Keys [1]: [s_store_sk#8] Functions [2]: [partial_sum(UnscaledValue(ss_ext_sales_price#2)), partial_sum(UnscaledValue(ss_net_profit#3))] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [s_store_sk#8, sum#9, sum#10] -Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(s_store_sk#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [s_store_sk#8, sum#9, sum#10] @@ -227,9 +217,9 @@ Input [3]: [sr_return_amt#12, sr_net_loss#13, s_store_sk#17] Keys [1]: [s_store_sk#17] Functions [2]: [partial_sum(UnscaledValue(sr_return_amt#12)), partial_sum(UnscaledValue(sr_net_loss#13))] -(26) CometColumnarExchange +(26) CometExchange Input [3]: [s_store_sk#17, sum#18, sum#19] -Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(s_store_sk#17, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (27) CometHashAggregate Input [3]: [s_store_sk#17, sum#18, sum#19] @@ -276,9 +266,9 @@ Input [3]: [cs_call_center_sk#28, cs_ext_sales_price#29, cs_net_profit#30] Keys [1]: [cs_call_center_sk#28] Functions [2]: [partial_sum(UnscaledValue(cs_ext_sales_price#29)), partial_sum(UnscaledValue(cs_net_profit#30))] -(37) CometColumnarExchange +(37) CometExchange Input [3]: [cs_call_center_sk#28, sum#34, sum#35] -Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: hashpartitioning(cs_call_center_sk#28, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (38) CometHashAggregate Input [3]: [cs_call_center_sk#28, sum#34, sum#35] @@ -316,9 +306,9 @@ Input [2]: [cr_return_amount#38, cr_net_loss#39] Keys: [] Functions [2]: [partial_sum(UnscaledValue(cr_return_amount#38)), partial_sum(UnscaledValue(cr_net_loss#39))] -(46) CometColumnarExchange +(46) CometExchange Input [2]: [sum#43, sum#44] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (47) CometHashAggregate Input [2]: [sum#43, sum#44] @@ -389,9 +379,9 @@ Input [3]: [ws_ext_sales_price#51, ws_net_profit#52, wp_web_page_sk#56] Keys [1]: [wp_web_page_sk#56] Functions [2]: [partial_sum(UnscaledValue(ws_ext_sales_price#51)), partial_sum(UnscaledValue(ws_net_profit#52))] -(62) CometColumnarExchange +(62) CometExchange Input [3]: [wp_web_page_sk#56, sum#57, sum#58] -Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=6] +Arguments: hashpartitioning(wp_web_page_sk#56, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=6] (63) CometHashAggregate Input [3]: [wp_web_page_sk#56, sum#57, sum#58] @@ -439,9 +429,9 @@ Input [3]: [wr_return_amt#60, wr_net_loss#61, wp_web_page_sk#65] Keys [1]: [wp_web_page_sk#65] Functions [2]: [partial_sum(UnscaledValue(wr_return_amt#60)), partial_sum(UnscaledValue(wr_net_loss#61))] -(73) CometColumnarExchange +(73) CometExchange Input [3]: [wp_web_page_sk#65, sum#66, sum#67] -Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=7] +Arguments: hashpartitioning(wp_web_page_sk#65, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=7] (74) CometHashAggregate Input [3]: [wp_web_page_sk#65, sum#66, sum#67] @@ -473,154 +463,126 @@ Functions [3]: [partial_sum(sales#22), partial_sum(returns#26), partial_sum(prof Aggregate Attributes [6]: [sum#76, isEmpty#77, sum#78, isEmpty#79, sum#80, isEmpty#81] Results [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -(81) RowToColumnar +(81) Exchange Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] +Arguments: hashpartitioning(channel#24, id#25, 5), ENSURE_REQUIREMENTS, [plan_id=8] -(82) CometColumnarExchange -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -Arguments: hashpartitioning(channel#24, id#25, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=8] - -(83) ColumnarToRow [codegen id : 6] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] - -(84) HashAggregate [codegen id : 6] +(82) HashAggregate [codegen id : 6] Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] Keys [2]: [channel#24, id#25] Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] Results [5]: [channel#24, id#25, cast(sum(sales#22)#88 as decimal(37,2)) AS sales#91, cast(sum(returns#26)#89 as decimal(37,2)) AS returns#92, cast(sum(profit#27)#90 as decimal(38,2)) AS profit#93] -(85) ReusedExchange [Reuses operator id: 82] +(83) ReusedExchange [Reuses operator id: 81] Output [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -(86) ColumnarToRow [codegen id : 12] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] - -(87) HashAggregate [codegen id : 12] +(84) HashAggregate [codegen id : 12] Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] Keys [2]: [channel#24, id#25] Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] Results [4]: [channel#24, sum(sales#22)#88 AS sales#94, sum(returns#26)#89 AS returns#95, sum(profit#27)#90 AS profit#96] -(88) HashAggregate [codegen id : 12] +(85) HashAggregate [codegen id : 12] Input [4]: [channel#24, sales#94, returns#95, profit#96] Keys [1]: [channel#24] Functions [3]: [partial_sum(sales#94), partial_sum(returns#95), partial_sum(profit#96)] Aggregate Attributes [6]: [sum#97, isEmpty#98, sum#99, isEmpty#100, sum#101, isEmpty#102] Results [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -(89) RowToColumnar +(86) Exchange Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] +Arguments: hashpartitioning(channel#24, 5), ENSURE_REQUIREMENTS, [plan_id=9] -(90) CometColumnarExchange -Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] -Arguments: hashpartitioning(channel#24, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=9] - -(91) ColumnarToRow [codegen id : 13] -Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] - -(92) HashAggregate [codegen id : 13] +(87) HashAggregate [codegen id : 13] Input [7]: [channel#24, sum#103, isEmpty#104, sum#105, isEmpty#106, sum#107, isEmpty#108] Keys [1]: [channel#24] Functions [3]: [sum(sales#94), sum(returns#95), sum(profit#96)] Aggregate Attributes [3]: [sum(sales#94)#109, sum(returns#95)#110, sum(profit#96)#111] Results [5]: [channel#24, null AS id#112, sum(sales#94)#109 AS sales#113, sum(returns#95)#110 AS returns#114, sum(profit#96)#111 AS profit#115] -(93) ReusedExchange [Reuses operator id: 82] +(88) ReusedExchange [Reuses operator id: 81] Output [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] -(94) ColumnarToRow [codegen id : 19] -Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] - -(95) HashAggregate [codegen id : 19] +(89) HashAggregate [codegen id : 19] Input [8]: [channel#24, id#25, sum#82, isEmpty#83, sum#84, isEmpty#85, sum#86, isEmpty#87] Keys [2]: [channel#24, id#25] Functions [3]: [sum(sales#22), sum(returns#26), sum(profit#27)] Aggregate Attributes [3]: [sum(sales#22)#88, sum(returns#26)#89, sum(profit#27)#90] Results [3]: [sum(sales#22)#88 AS sales#94, sum(returns#26)#89 AS returns#95, sum(profit#27)#90 AS profit#96] -(96) HashAggregate [codegen id : 19] +(90) HashAggregate [codegen id : 19] Input [3]: [sales#94, returns#95, profit#96] Keys: [] Functions [3]: [partial_sum(sales#94), partial_sum(returns#95), partial_sum(profit#96)] Aggregate Attributes [6]: [sum#116, isEmpty#117, sum#118, isEmpty#119, sum#120, isEmpty#121] Results [6]: [sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127] -(97) RowToColumnar -Input [6]: [sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127] - -(98) CometColumnarExchange +(91) Exchange Input [6]: [sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=10] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=10] -(99) ColumnarToRow [codegen id : 20] -Input [6]: [sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127] - -(100) HashAggregate [codegen id : 20] +(92) HashAggregate [codegen id : 20] Input [6]: [sum#122, isEmpty#123, sum#124, isEmpty#125, sum#126, isEmpty#127] Keys: [] Functions [3]: [sum(sales#94), sum(returns#95), sum(profit#96)] Aggregate Attributes [3]: [sum(sales#94)#128, sum(returns#95)#129, sum(profit#96)#130] Results [5]: [null AS channel#131, null AS id#132, sum(sales#94)#128 AS sales#133, sum(returns#95)#129 AS returns#134, sum(profit#96)#130 AS profit#135] -(101) Union +(93) Union -(102) HashAggregate [codegen id : 21] +(94) HashAggregate [codegen id : 21] Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] Keys [5]: [channel#24, id#25, sales#91, returns#92, profit#93] Functions: [] Aggregate Attributes: [] Results [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -(103) RowToColumnar +(95) Exchange Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] +Arguments: hashpartitioning(channel#24, id#25, sales#91, returns#92, profit#93, 5), ENSURE_REQUIREMENTS, [plan_id=11] -(104) CometColumnarExchange -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Arguments: hashpartitioning(channel#24, id#25, sales#91, returns#92, profit#93, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=11] - -(105) CometHashAggregate +(96) HashAggregate [codegen id : 22] Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] Keys [5]: [channel#24, id#25, sales#91, returns#92, profit#93] Functions: [] +Aggregate Attributes: [] +Results [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -(106) CometTakeOrderedAndProject -Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] -Arguments: TakeOrderedAndProject(limit=100, orderBy=[channel#24 ASC NULLS FIRST,id#25 ASC NULLS FIRST], output=[channel#24,id#25,sales#91,returns#92,profit#93]), [channel#24, id#25, sales#91, returns#92, profit#93], 100, [channel#24 ASC NULLS FIRST, id#25 ASC NULLS FIRST], [channel#24, id#25, sales#91, returns#92, profit#93] - -(107) ColumnarToRow [codegen id : 22] +(97) TakeOrderedAndProject Input [5]: [channel#24, id#25, sales#91, returns#92, profit#93] +Arguments: 100, [channel#24 ASC NULLS FIRST, id#25 ASC NULLS FIRST], [channel#24, id#25, sales#91, returns#92, profit#93] ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = ss_sold_date_sk#4 IN dynamicpruning#5 -BroadcastExchange (112) -+- * ColumnarToRow (111) - +- CometProject (110) - +- CometFilter (109) - +- CometScan parquet spark_catalog.default.date_dim (108) +BroadcastExchange (102) ++- * ColumnarToRow (101) + +- CometProject (100) + +- CometFilter (99) + +- CometScan parquet spark_catalog.default.date_dim (98) -(108) Scan parquet spark_catalog.default.date_dim +(98) Scan parquet spark_catalog.default.date_dim Output [2]: [d_date_sk#6, d_date#7] Batched: true Location [not included in comparison]/{warehouse_dir}/date_dim] PushedFilters: [IsNotNull(d_date), GreaterThanOrEqual(d_date,1998-08-04), LessThanOrEqual(d_date,1998-09-03), IsNotNull(d_date_sk)] ReadSchema: struct -(109) CometFilter +(99) CometFilter Input [2]: [d_date_sk#6, d_date#7] Condition : (((isnotnull(d_date#7) AND (d_date#7 >= 1998-08-04)) AND (d_date#7 <= 1998-09-03)) AND isnotnull(d_date_sk#6)) -(110) CometProject +(100) CometProject Input [2]: [d_date_sk#6, d_date#7] Arguments: [d_date_sk#6], [d_date_sk#6] -(111) ColumnarToRow [codegen id : 1] +(101) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#6] -(112) BroadcastExchange +(102) BroadcastExchange Input [1]: [d_date_sk#6] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=12] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt index 6e5506d68..01cd2f085 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q77a/simplified.txt @@ -1,145 +1,135 @@ -WholeStageCodegen (22) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,id,sales,returns,profit] - CometHashAggregate [channel,id,sales,returns,profit] - CometColumnarExchange [channel,id,sales,returns,profit] #1 - RowToColumnar - WholeStageCodegen (21) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (6) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id] #2 - RowToColumnar - WholeStageCodegen (5) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (22) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Exchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (21) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (6) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel,id] #2 + WholeStageCodegen (5) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + ColumnarToRow InputAdapter - Union - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [s_store_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] - CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] - CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] - CometColumnarExchange [s_store_sk] #3 - CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] - CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] - CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] - CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] - CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #4 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [d_date_sk] #5 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk] #6 - CometFilter [s_store_sk] - CometScan parquet spark_catalog.default.store [s_store_sk] - CometBroadcastExchange [s_store_sk,returns,profit_loss] #7 - CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] - CometColumnarExchange [s_store_sk] #8 - CometHashAggregate [s_store_sk,sum,sum,sr_return_amt,sr_net_loss] - CometProject [sr_return_amt,sr_net_loss,s_store_sk] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] - CometProject [sr_store_sk,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] - CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [s_store_sk] #6 - WholeStageCodegen (3) - Project [cs_call_center_sk,sales,returns,profit,profit_loss] - BroadcastNestedLoopJoin - InputAdapter - BroadcastExchange #9 - WholeStageCodegen (2) - ColumnarToRow - InputAdapter - CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] - CometColumnarExchange [cs_call_center_sk] #10 - CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] - CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] - CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ColumnarToRow - InputAdapter - CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] - CometColumnarExchange #11 - CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] - CometProject [cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometProject [s_store_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] + CometBroadcastHashJoin [s_store_sk,sales,profit,s_store_sk,returns,profit_loss] + CometHashAggregate [s_store_sk,sales,profit,sum,sum,sum(UnscaledValue(ss_ext_sales_price)),sum(UnscaledValue(ss_net_profit))] + CometExchange [s_store_sk] #3 + CometHashAggregate [s_store_sk,sum,sum,ss_ext_sales_price,ss_net_profit] + CometProject [ss_ext_sales_price,ss_net_profit,s_store_sk] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,s_store_sk] + CometProject [ss_store_sk,ss_ext_sales_price,ss_net_profit] + CometBroadcastHashJoin [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,d_date_sk] + CometFilter [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_store_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #4 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [d_date_sk] #5 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk] #6 + CometFilter [s_store_sk] + CometScan parquet spark_catalog.default.store [s_store_sk] + CometBroadcastExchange [s_store_sk,returns,profit_loss] #7 + CometHashAggregate [s_store_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(sr_return_amt)),sum(UnscaledValue(sr_net_loss))] + CometExchange [s_store_sk] #8 + CometHashAggregate [s_store_sk,sum,sum,sr_return_amt,sr_net_loss] + CometProject [sr_return_amt,sr_net_loss,s_store_sk] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,s_store_sk] + CometProject [sr_store_sk,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk,d_date_sk] + CometFilter [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_store_sk,sr_return_amt,sr_net_loss,sr_returned_date_sk] ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - WholeStageCodegen (4) - ColumnarToRow - InputAdapter - CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] - CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] - CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] - CometColumnarExchange [wp_web_page_sk] #12 - CometHashAggregate [wp_web_page_sk,sum,sum,ws_ext_sales_price,ws_net_profit] - CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] - CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] - CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] - CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - CometBroadcastExchange [wp_web_page_sk] #13 - CometFilter [wp_web_page_sk] - CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] - CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #14 - CometHashAggregate [wp_web_page_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] - CometColumnarExchange [wp_web_page_sk] #15 - CometHashAggregate [wp_web_page_sk,sum,sum,wr_return_amt,wr_net_loss] - CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] - CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] - CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedSubquery [d_date_sk] #1 - ReusedExchange [d_date_sk] #5 - ReusedExchange [wp_web_page_sk] #13 - WholeStageCodegen (13) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel] #16 - RowToColumnar - WholeStageCodegen (12) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (20) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #17 - RowToColumnar - WholeStageCodegen (19) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + ReusedExchange [d_date_sk] #5 + ReusedExchange [s_store_sk] #6 + WholeStageCodegen (3) + Project [cs_call_center_sk,sales,returns,profit,profit_loss] + BroadcastNestedLoopJoin + InputAdapter + BroadcastExchange #9 + WholeStageCodegen (2) + ColumnarToRow + InputAdapter + CometHashAggregate [cs_call_center_sk,sales,profit,sum,sum,sum(UnscaledValue(cs_ext_sales_price)),sum(UnscaledValue(cs_net_profit))] + CometExchange [cs_call_center_sk] #10 + CometHashAggregate [cs_call_center_sk,sum,sum,cs_ext_sales_price,cs_net_profit] + CometProject [cs_call_center_sk,cs_ext_sales_price,cs_net_profit] + CometBroadcastHashJoin [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_call_center_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 ColumnarToRow InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + CometHashAggregate [returns,profit_loss,sum,sum,sum(UnscaledValue(cr_return_amount)),sum(UnscaledValue(cr_net_loss))] + CometExchange #11 + CometHashAggregate [sum,sum,cr_return_amount,cr_net_loss] + CometProject [cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cr_return_amount,cr_net_loss,cr_returned_date_sk,d_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + WholeStageCodegen (4) + ColumnarToRow + InputAdapter + CometProject [wp_web_page_sk,returns,profit,profit_loss] [channel,id,sales,returns,profit] + CometBroadcastHashJoin [wp_web_page_sk,sales,profit,wp_web_page_sk,returns,profit_loss] + CometHashAggregate [wp_web_page_sk,sales,profit,sum,sum,sum(UnscaledValue(ws_ext_sales_price)),sum(UnscaledValue(ws_net_profit))] + CometExchange [wp_web_page_sk] #12 + CometHashAggregate [wp_web_page_sk,sum,sum,ws_ext_sales_price,ws_net_profit] + CometProject [ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,wp_web_page_sk] + CometProject [ws_web_page_sk,ws_ext_sales_price,ws_net_profit] + CometBroadcastHashJoin [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,d_date_sk] + CometFilter [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_web_page_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + CometBroadcastExchange [wp_web_page_sk] #13 + CometFilter [wp_web_page_sk] + CometScan parquet spark_catalog.default.web_page [wp_web_page_sk] + CometBroadcastExchange [wp_web_page_sk,returns,profit_loss] #14 + CometHashAggregate [wp_web_page_sk,returns,profit_loss,sum,sum,sum(UnscaledValue(wr_return_amt)),sum(UnscaledValue(wr_net_loss))] + CometExchange [wp_web_page_sk] #15 + CometHashAggregate [wp_web_page_sk,sum,sum,wr_return_amt,wr_net_loss] + CometProject [wr_return_amt,wr_net_loss,wp_web_page_sk] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wp_web_page_sk] + CometProject [wr_web_page_sk,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk,d_date_sk] + CometFilter [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_web_page_sk,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedSubquery [d_date_sk] #1 + ReusedExchange [d_date_sk] #5 + ReusedExchange [wp_web_page_sk] #13 + WholeStageCodegen (13) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel] #16 + WholeStageCodegen (12) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (20) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange #17 + WholeStageCodegen (19) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt index 9304cb85f..ac26f8d51 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q78/explain.txt @@ -7,7 +7,7 @@ TakeOrderedAndProject (67) : +- CometSortMergeJoin (42) : :- CometSort (21) : : +- CometHashAggregate (20) - : : +- CometColumnarExchange (19) + : : +- CometExchange (19) : : +- CometHashAggregate (18) : : +- CometProject (17) : : +- CometBroadcastHashJoin (16) @@ -15,11 +15,11 @@ TakeOrderedAndProject (67) : : : +- CometFilter (11) : : : +- CometSortMergeJoin (10) : : : :- CometSort (4) - : : : : +- CometColumnarExchange (3) + : : : : +- CometExchange (3) : : : : +- CometFilter (2) : : : : +- CometScan parquet spark_catalog.default.store_sales (1) : : : +- CometSort (9) - : : : +- CometColumnarExchange (8) + : : : +- CometExchange (8) : : : +- CometProject (7) : : : +- CometFilter (6) : : : +- CometScan parquet spark_catalog.default.store_returns (5) @@ -29,7 +29,7 @@ TakeOrderedAndProject (67) : +- CometSort (41) : +- CometFilter (40) : +- CometHashAggregate (39) - : +- CometColumnarExchange (38) + : +- CometExchange (38) : +- CometHashAggregate (37) : +- CometProject (36) : +- CometBroadcastHashJoin (35) @@ -37,11 +37,11 @@ TakeOrderedAndProject (67) : : +- CometFilter (32) : : +- CometSortMergeJoin (31) : : :- CometSort (25) - : : : +- CometColumnarExchange (24) + : : : +- CometExchange (24) : : : +- CometFilter (23) : : : +- CometScan parquet spark_catalog.default.web_sales (22) : : +- CometSort (30) - : : +- CometColumnarExchange (29) + : : +- CometExchange (29) : : +- CometProject (28) : : +- CometFilter (27) : : +- CometScan parquet spark_catalog.default.web_returns (26) @@ -49,7 +49,7 @@ TakeOrderedAndProject (67) +- CometSort (63) +- CometFilter (62) +- CometHashAggregate (61) - +- CometColumnarExchange (60) + +- CometExchange (60) +- CometHashAggregate (59) +- CometProject (58) +- CometBroadcastHashJoin (57) @@ -57,11 +57,11 @@ TakeOrderedAndProject (67) : +- CometFilter (54) : +- CometSortMergeJoin (53) : :- CometSort (47) - : : +- CometColumnarExchange (46) + : : +- CometExchange (46) : : +- CometFilter (45) : : +- CometScan parquet spark_catalog.default.catalog_sales (44) : +- CometSort (52) - : +- CometColumnarExchange (51) + : +- CometExchange (51) : +- CometProject (50) : +- CometFilter (49) : +- CometScan parquet spark_catalog.default.catalog_returns (48) @@ -80,9 +80,9 @@ ReadSchema: struct -(127) CometFilter +(117) CometFilter Input [2]: [d_date_sk#14, d_date#15] Condition : (((isnotnull(d_date#15) AND (d_date#15 >= 1998-08-04)) AND (d_date#15 <= 1998-09-03)) AND isnotnull(d_date_sk#14)) -(128) CometProject +(118) CometProject Input [2]: [d_date_sk#14, d_date#15] Arguments: [d_date_sk#14], [d_date_sk#14] -(129) ColumnarToRow [codegen id : 1] +(119) ColumnarToRow [codegen id : 1] Input [1]: [d_date_sk#14] -(130) BroadcastExchange +(120) BroadcastExchange Input [1]: [d_date_sk#14] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=14] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt index 210c9b6d2..aa61bbd57 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q80a/simplified.txt @@ -1,157 +1,147 @@ -WholeStageCodegen (19) - ColumnarToRow - InputAdapter - CometTakeOrderedAndProject [channel,id,sales,returns,profit] - CometHashAggregate [channel,id,sales,returns,profit] - CometColumnarExchange [channel,id,sales,returns,profit] #1 - RowToColumnar - WholeStageCodegen (18) - HashAggregate [channel,id,sales,returns,profit] - InputAdapter - Union - WholeStageCodegen (5) - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel,id] #2 - RowToColumnar - WholeStageCodegen (4) - HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - InputAdapter - Union - WholeStageCodegen (1) - HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [s_store_id] #3 - CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] - CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] - CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] - CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] - CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] - CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] - CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometColumnarExchange [ss_item_sk,ss_ticket_number] #4 - CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] - SubqueryBroadcast [d_date_sk] #1 - BroadcastExchange #5 - WholeStageCodegen (1) - ColumnarToRow - InputAdapter - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometColumnarExchange [sr_item_sk,sr_ticket_number] #6 - CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] - CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] - CometBroadcastExchange [d_date_sk] #7 - CometProject [d_date_sk] - CometFilter [d_date_sk,d_date] - CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] - CometBroadcastExchange [s_store_sk,s_store_id] #8 - CometFilter [s_store_sk,s_store_id] - CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] - CometBroadcastExchange [i_item_sk] #9 - CometProject [i_item_sk] - CometFilter [i_item_sk,i_current_price] - CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] - CometBroadcastExchange [p_promo_sk] #10 - CometProject [p_promo_sk] - CometFilter [p_promo_sk,p_channel_tv] - CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] - WholeStageCodegen (2) - HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [cp_catalog_page_id] #11 - CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] - CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] - CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] - CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] - CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] - CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] - CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometColumnarExchange [cs_item_sk,cs_order_number] #12 - CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometColumnarExchange [cr_item_sk,cr_order_number] #13 - CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] - CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #14 - CometFilter [cp_catalog_page_sk,cp_catalog_page_id] - CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 - WholeStageCodegen (3) - HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [web_site_id] #15 - CometHashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] - CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] - CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] - CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] - CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] - CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] - CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometColumnarExchange [ws_item_sk,ws_order_number] #16 - CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] - ReusedSubquery [d_date_sk] #1 - CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometColumnarExchange [wr_item_sk,wr_order_number] #17 - CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] - CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] - ReusedExchange [d_date_sk] #7 - CometBroadcastExchange [web_site_sk,web_site_id] #18 - CometFilter [web_site_sk,web_site_id] - CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] - ReusedExchange [i_item_sk] #9 - ReusedExchange [p_promo_sk] #10 - WholeStageCodegen (11) - HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange [channel] #19 - RowToColumnar - WholeStageCodegen (10) - HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 - WholeStageCodegen (17) - HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - CometColumnarExchange #20 - RowToColumnar - WholeStageCodegen (16) - HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] - HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] - ColumnarToRow - InputAdapter - ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 +TakeOrderedAndProject [channel,id,sales,returns,profit] + WholeStageCodegen (19) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Exchange [channel,id,sales,returns,profit] #1 + WholeStageCodegen (18) + HashAggregate [channel,id,sales,returns,profit] + InputAdapter + Union + WholeStageCodegen (5) + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel,id] #2 + WholeStageCodegen (4) + HashAggregate [channel,id,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Union + WholeStageCodegen (1) + HashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ss_ext_sales_price)),sum(coalesce(cast(sr_return_amt as decimal(12,2)), 0.00)),sum((ss_net_profit - coalesce(cast(sr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [s_store_id] #3 + CometHashAggregate [s_store_id,sum,sum,isEmpty,sum,isEmpty,ss_ext_sales_price,sr_return_amt,ss_net_profit,sr_net_loss] + CometProject [ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,p_promo_sk] + CometProject [ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id,i_item_sk] + CometProject [ss_item_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_id] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss,s_store_sk,s_store_id] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,sr_return_amt,sr_net_loss] + CometBroadcastHashJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss,d_date_sk] + CometProject [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_return_amt,sr_net_loss] + CometSortMergeJoin [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk,sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometSort [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometExchange [ss_item_sk,ss_ticket_number] #4 + CometFilter [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + CometScan parquet spark_catalog.default.store_sales [ss_item_sk,ss_store_sk,ss_promo_sk,ss_ticket_number,ss_ext_sales_price,ss_net_profit,ss_sold_date_sk] + SubqueryBroadcast [d_date_sk] #1 + BroadcastExchange #5 + WholeStageCodegen (1) + ColumnarToRow + InputAdapter + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometSort [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometExchange [sr_item_sk,sr_ticket_number] #6 + CometProject [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss] + CometFilter [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometScan parquet spark_catalog.default.store_returns [sr_item_sk,sr_ticket_number,sr_return_amt,sr_net_loss,sr_returned_date_sk] + CometBroadcastExchange [d_date_sk] #7 + CometProject [d_date_sk] + CometFilter [d_date_sk,d_date] + CometScan parquet spark_catalog.default.date_dim [d_date_sk,d_date] + CometBroadcastExchange [s_store_sk,s_store_id] #8 + CometFilter [s_store_sk,s_store_id] + CometScan parquet spark_catalog.default.store [s_store_sk,s_store_id] + CometBroadcastExchange [i_item_sk] #9 + CometProject [i_item_sk] + CometFilter [i_item_sk,i_current_price] + CometScan parquet spark_catalog.default.item [i_item_sk,i_current_price] + CometBroadcastExchange [p_promo_sk] #10 + CometProject [p_promo_sk] + CometFilter [p_promo_sk,p_channel_tv] + CometScan parquet spark_catalog.default.promotion [p_promo_sk,p_channel_tv] + WholeStageCodegen (2) + HashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(cs_ext_sales_price)),sum(coalesce(cast(cr_return_amount as decimal(12,2)), 0.00)),sum((cs_net_profit - coalesce(cast(cr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [cp_catalog_page_id] #11 + CometHashAggregate [cp_catalog_page_id,sum,sum,isEmpty,sum,isEmpty,cs_ext_sales_price,cr_return_amount,cs_net_profit,cr_net_loss] + CometProject [cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,p_promo_sk] + CometProject [cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id,i_item_sk] + CometProject [cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_id] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss,cp_catalog_page_sk,cp_catalog_page_id] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cr_return_amount,cr_net_loss] + CometBroadcastHashJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss,d_date_sk] + CometProject [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_return_amount,cr_net_loss] + CometSortMergeJoin [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk,cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometSort [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometExchange [cs_item_sk,cs_order_number] #12 + CometFilter [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + CometScan parquet spark_catalog.default.catalog_sales [cs_catalog_page_sk,cs_item_sk,cs_promo_sk,cs_order_number,cs_ext_sales_price,cs_net_profit,cs_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometExchange [cr_item_sk,cr_order_number] #13 + CometProject [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss] + CometFilter [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + CometScan parquet spark_catalog.default.catalog_returns [cr_item_sk,cr_order_number,cr_return_amount,cr_net_loss,cr_returned_date_sk] + ReusedExchange [d_date_sk] #7 + CometBroadcastExchange [cp_catalog_page_sk,cp_catalog_page_id] #14 + CometFilter [cp_catalog_page_sk,cp_catalog_page_id] + CometScan parquet spark_catalog.default.catalog_page [cp_catalog_page_sk,cp_catalog_page_id] + ReusedExchange [i_item_sk] #9 + ReusedExchange [p_promo_sk] #10 + WholeStageCodegen (3) + HashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty] [sum(UnscaledValue(ws_ext_sales_price)),sum(coalesce(cast(wr_return_amt as decimal(12,2)), 0.00)),sum((ws_net_profit - coalesce(cast(wr_net_loss as decimal(12,2)), 0.00))),channel,id,sales,returns,profit,sum,sum,isEmpty,sum,isEmpty] + ColumnarToRow + InputAdapter + CometExchange [web_site_id] #15 + CometHashAggregate [web_site_id,sum,sum,isEmpty,sum,isEmpty,ws_ext_sales_price,wr_return_amt,ws_net_profit,wr_net_loss] + CometProject [ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,p_promo_sk] + CometProject [ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id,i_item_sk] + CometProject [ws_item_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_id] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss,web_site_sk,web_site_id] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,wr_return_amt,wr_net_loss] + CometBroadcastHashJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss,d_date_sk] + CometProject [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_return_amt,wr_net_loss] + CometSortMergeJoin [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk,wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometSort [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometExchange [ws_item_sk,ws_order_number] #16 + CometFilter [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + CometScan parquet spark_catalog.default.web_sales [ws_item_sk,ws_web_site_sk,ws_promo_sk,ws_order_number,ws_ext_sales_price,ws_net_profit,ws_sold_date_sk] + ReusedSubquery [d_date_sk] #1 + CometSort [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometExchange [wr_item_sk,wr_order_number] #17 + CometProject [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss] + CometFilter [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + CometScan parquet spark_catalog.default.web_returns [wr_item_sk,wr_order_number,wr_return_amt,wr_net_loss,wr_returned_date_sk] + ReusedExchange [d_date_sk] #7 + CometBroadcastExchange [web_site_sk,web_site_id] #18 + CometFilter [web_site_sk,web_site_id] + CometScan parquet spark_catalog.default.web_site [web_site_sk,web_site_id] + ReusedExchange [i_item_sk] #9 + ReusedExchange [p_promo_sk] #10 + WholeStageCodegen (11) + HashAggregate [channel,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange [channel] #19 + WholeStageCodegen (10) + HashAggregate [channel,sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 + WholeStageCodegen (17) + HashAggregate [sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),channel,id,sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + Exchange #20 + WholeStageCodegen (16) + HashAggregate [sales,returns,profit] [sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty,sum,isEmpty] + HashAggregate [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] [sum(sales),sum(returns),sum(profit),sales,returns,profit,sum,isEmpty,sum,isEmpty,sum,isEmpty] + InputAdapter + ReusedExchange [channel,id,sum,isEmpty,sum,isEmpty,sum,isEmpty] #2 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt index 7a0d1b7ad..d6850a031 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/explain.txt @@ -4,13 +4,13 @@ TakeOrderedAndProject (36) +- Window (34) +- * ColumnarToRow (33) +- CometSort (32) - +- CometColumnarExchange (31) + +- CometExchange (31) +- CometHashAggregate (30) - +- CometColumnarExchange (29) + +- CometExchange (29) +- CometHashAggregate (28) +- CometUnion (27) :- CometHashAggregate (16) - : +- CometColumnarExchange (15) + : +- CometExchange (15) : +- CometHashAggregate (14) : +- CometProject (13) : +- CometBroadcastHashJoin (12) @@ -26,12 +26,12 @@ TakeOrderedAndProject (36) : +- CometFilter (10) : +- CometScan parquet spark_catalog.default.item (9) :- CometHashAggregate (21) - : +- CometColumnarExchange (20) + : +- CometExchange (20) : +- CometHashAggregate (19) : +- CometHashAggregate (18) : +- ReusedExchange (17) +- CometHashAggregate (26) - +- CometColumnarExchange (25) + +- CometExchange (25) +- CometHashAggregate (24) +- CometHashAggregate (23) +- ReusedExchange (22) @@ -106,9 +106,9 @@ Input [3]: [ws_net_paid#2, i_class#8, i_category#9] Keys [2]: [i_category#9, i_class#8] Functions [1]: [partial_sum(UnscaledValue(ws_net_paid#2))] -(15) CometColumnarExchange +(15) CometExchange Input [3]: [i_category#9, i_class#8, sum#10] -Arguments: hashpartitioning(i_category#9, i_class#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_category#9, i_class#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [3]: [i_category#9, i_class#8, sum#10] @@ -128,9 +128,9 @@ Input [2]: [total_sum#12, i_category#9] Keys [1]: [i_category#9] Functions [1]: [partial_sum(total_sum#12)] -(20) CometColumnarExchange +(20) CometExchange Input [3]: [i_category#9, sum#13, isEmpty#14] -Arguments: hashpartitioning(i_category#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_category#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (21) CometHashAggregate Input [3]: [i_category#9, sum#13, isEmpty#14] @@ -150,9 +150,9 @@ Input [1]: [total_sum#12] Keys: [] Functions [1]: [partial_sum(total_sum#12)] -(25) CometColumnarExchange +(25) CometExchange Input [2]: [sum#16, isEmpty#17] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=3] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=3] (26) CometHashAggregate Input [2]: [sum#16, isEmpty#17] @@ -169,18 +169,18 @@ Input [6]: [total_sum#18, i_category#9, i_class#8, g_category#19, g_class#20, lo Keys [6]: [total_sum#18, i_category#9, i_class#8, g_category#19, g_class#20, lochierarchy#21] Functions: [] -(29) CometColumnarExchange +(29) CometExchange Input [6]: [total_sum#18, i_category#9, i_class#8, g_category#19, g_class#20, lochierarchy#21] -Arguments: hashpartitioning(total_sum#18, i_category#9, i_class#8, g_category#19, g_class#20, lochierarchy#21, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=4] +Arguments: hashpartitioning(total_sum#18, i_category#9, i_class#8, g_category#19, g_class#20, lochierarchy#21, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=4] (30) CometHashAggregate Input [6]: [total_sum#18, i_category#9, i_class#8, g_category#19, g_class#20, lochierarchy#21] Keys [6]: [total_sum#18, i_category#9, i_class#8, g_category#19, g_class#20, lochierarchy#21] Functions: [] -(31) CometColumnarExchange +(31) CometExchange Input [5]: [total_sum#18, i_category#9, i_class#8, lochierarchy#21, _w0#33] -Arguments: hashpartitioning(lochierarchy#21, _w0#33, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=5] +Arguments: hashpartitioning(lochierarchy#21, _w0#33, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=5] (32) CometSort Input [5]: [total_sum#18, i_category#9, i_class#8, lochierarchy#21, _w0#33] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt index 255e4acc3..ae161e5ae 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q86a/simplified.txt @@ -7,13 +7,13 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl ColumnarToRow InputAdapter CometSort [total_sum,i_category,i_class,lochierarchy,_w0] - CometColumnarExchange [lochierarchy,_w0] #1 + CometExchange [lochierarchy,_w0] #1 CometHashAggregate [total_sum,i_category,i_class,lochierarchy,_w0,g_category,g_class] - CometColumnarExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 + CometExchange [total_sum,i_category,i_class,g_category,g_class,lochierarchy] #2 CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy] CometUnion [total_sum,i_category,i_class,g_category,g_class,lochierarchy] CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,sum(UnscaledValue(ws_net_paid))] - CometColumnarExchange [i_category,i_class] #3 + CometExchange [i_category,i_class] #3 CometHashAggregate [i_category,i_class,sum,ws_net_paid] CometProject [ws_net_paid,i_class,i_category] CometBroadcastHashJoin [ws_item_sk,ws_net_paid,i_item_sk,i_class,i_category] @@ -37,12 +37,12 @@ TakeOrderedAndProject [lochierarchy,i_category,rank_within_parent,total_sum,i_cl CometFilter [i_item_sk,i_class,i_category] CometScan parquet spark_catalog.default.item [i_item_sk,i_class,i_category] CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] - CometColumnarExchange [i_category] #7 + CometExchange [i_category] #7 CometHashAggregate [i_category,sum,isEmpty,total_sum] CometHashAggregate [total_sum,i_category,i_class,sum,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 CometHashAggregate [total_sum,i_category,i_class,g_category,g_class,lochierarchy,sum,isEmpty,sum(total_sum)] - CometColumnarExchange #8 + CometExchange #8 CometHashAggregate [sum,isEmpty,total_sum] CometHashAggregate [total_sum,i_category,i_class,sum,sum(UnscaledValue(ws_net_paid))] ReusedExchange [i_category,i_class,sum] #3 diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt index 8b0facb13..e85baa1ee 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/explain.txt @@ -7,9 +7,9 @@ +- Window (20) +- * ColumnarToRow (19) +- CometSort (18) - +- CometColumnarExchange (17) + +- CometExchange (17) +- CometHashAggregate (16) - +- CometColumnarExchange (15) + +- CometExchange (15) +- CometHashAggregate (14) +- CometProject (13) +- CometBroadcastHashJoin (12) @@ -95,18 +95,18 @@ Input [6]: [ss_ext_sales_price#2, i_item_id#6, i_item_desc#7, i_current_price#8, Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [partial_sum(UnscaledValue(ss_ext_sales_price#2))] -(15) CometColumnarExchange +(15) CometExchange Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] -Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=1] +Arguments: hashpartitioning(i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=1] (16) CometHashAggregate Input [6]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, sum#13] Keys [5]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8] Functions [1]: [sum(UnscaledValue(ss_ext_sales_price#2))] -(17) CometColumnarExchange +(17) CometExchange Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] -Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometColumnarShuffle, [plan_id=2] +Arguments: hashpartitioning(i_class#9, 5), ENSURE_REQUIREMENTS, CometNativeShuffle, [plan_id=2] (18) CometSort Input [7]: [i_item_id#6, i_item_desc#7, i_category#10, i_class#9, i_current_price#8, itemrevenue#14, _w0#15] diff --git a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt index cbc817472..989ade02c 100644 --- a/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt +++ b/spark/src/test/resources/tpcds-plan-stability/approved-plans-v2_7/q98/simplified.txt @@ -12,9 +12,9 @@ WholeStageCodegen (3) ColumnarToRow InputAdapter CometSort [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0] - CometColumnarExchange [i_class] #2 + CometExchange [i_class] #2 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,itemrevenue,_w0,sum,sum(UnscaledValue(ss_ext_sales_price))] - CometColumnarExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 + CometExchange [i_item_id,i_item_desc,i_category,i_class,i_current_price] #3 CometHashAggregate [i_item_id,i_item_desc,i_category,i_class,i_current_price,sum,ss_ext_sales_price] CometProject [ss_ext_sales_price,i_item_id,i_item_desc,i_current_price,i_class,i_category] CometBroadcastHashJoin [ss_ext_sales_price,ss_sold_date_sk,i_item_id,i_item_desc,i_current_price,i_class,i_category,d_date_sk]