diff --git a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q2.txt b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q2.txt index 0966f808d..c7e8a5e46 100644 --- a/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q2.txt +++ b/dev/auron-it/src/main/resources/tpcds-plan-stability/spark-3.5/q2.txt @@ -1,145 +1,144 @@ == Physical Plan == -AdaptiveSparkPlan (136) +AdaptiveSparkPlan (135) +- == Final Plan == - NativeSort (77) - +- InputAdapter (76) - +- AQEShuffleRead (75) - +- ShuffleQueryStage (74), Statistics(X) - +- NativeShuffleExchange (73) - +- ConvertToNative (72) - +- * Project (71) - +- NativeSortMergeJoin Inner (70) - :- NativeProject (45) - : +- NativeSortMergeJoin Inner (44) - : :- NativeSort (34) - : : +- NativeProject (33) - : : +- NativeHashAggregate (32) - : : +- InputAdapter (31) - : : +- AQEShuffleRead (30) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- NativeShuffleExchange (28) - : : +- NativeHashAggregate (27) - : : +- NativeProject (26) - : : +- NativeProject (25) - : : +- NativeSortMergeJoin Inner (24) - : : :- NativeSort (15) - : : : +- InputAdapter (14) - : : : +- AQEShuffleRead (13) - : : : +- ShuffleQueryStage (12), Statistics(X) - : : : +- NativeShuffleExchange (11) - : : : +- InputAdapter (10) - : : : +- NativeUnion (9) - : : : :- NativeProject (4) - : : : : +- NativeFilter (3) - : : : : +- InputAdapter (2) - : : : : +- NativeParquetScan (1) - : : : +- NativeProject (8) - : : : +- NativeFilter (7) - : : : +- InputAdapter (6) - : : : +- NativeParquetScan (5) - : : +- NativeSort (23) - : : +- InputAdapter (22) - : : +- AQEShuffleRead (21) - : : +- ShuffleQueryStage (20), Statistics(X) - : : +- NativeShuffleExchange (19) - : : +- NativeFilter (18) - : : +- InputAdapter (17) - : : +- NativeParquetScan (16) - : +- NativeSort (43) - : +- InputAdapter (42) - : +- AQEShuffleRead (41) - : +- ShuffleQueryStage (40), Statistics(X) - : +- NativeShuffleExchange (39) - : +- NativeProject (38) - : +- NativeFilter (37) - : +- InputAdapter (36) - : +- NativeParquetScan (35) - +- NativeSort (69) - +- InputAdapter (68) - +- AQEShuffleRead (67) - +- ShuffleQueryStage (66), Statistics(X) - +- NativeShuffleExchange (65) - +- NativeProject (64) - +- NativeSortMergeJoin Inner (63) - :- NativeSort (53) - : +- NativeProject (52) - : +- NativeHashAggregate (51) - : +- InputAdapter (50) - : +- InputAdapter (49) - : +- AQEShuffleRead (48) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ReusedExchange (46) - +- NativeSort (62) - +- InputAdapter (61) - +- AQEShuffleRead (60) - +- ShuffleQueryStage (59), Statistics(X) - +- NativeShuffleExchange (58) - +- NativeProject (57) - +- NativeFilter (56) - +- InputAdapter (55) - +- NativeParquetScan (54) + NativeSort (76) + +- InputAdapter (75) + +- AQEShuffleRead (74) + +- ShuffleQueryStage (73), Statistics(X) + +- NativeShuffleExchange (72) + +- NativeProject (71) + +- NativeSortMergeJoin Inner (70) + :- NativeProject (45) + : +- NativeSortMergeJoin Inner (44) + : :- NativeSort (34) + : : +- NativeProject (33) + : : +- NativeHashAggregate (32) + : : +- InputAdapter (31) + : : +- AQEShuffleRead (30) + : : +- ShuffleQueryStage (29), Statistics(X) + : : +- NativeShuffleExchange (28) + : : +- NativeHashAggregate (27) + : : +- NativeProject (26) + : : +- NativeProject (25) + : : +- NativeSortMergeJoin Inner (24) + : : :- NativeSort (15) + : : : +- InputAdapter (14) + : : : +- AQEShuffleRead (13) + : : : +- ShuffleQueryStage (12), Statistics(X) + : : : +- NativeShuffleExchange (11) + : : : +- InputAdapter (10) + : : : +- NativeUnion (9) + : : : :- NativeProject (4) + : : : : +- NativeFilter (3) + : : : : +- InputAdapter (2) + : : : : +- NativeParquetScan (1) + : : : +- NativeProject (8) + : : : +- NativeFilter (7) + : : : +- InputAdapter (6) + : : : +- NativeParquetScan (5) + : : +- NativeSort (23) + : : +- InputAdapter (22) + : : +- AQEShuffleRead (21) + : : +- ShuffleQueryStage (20), Statistics(X) + : : +- NativeShuffleExchange (19) + : : +- NativeFilter (18) + : : +- InputAdapter (17) + : : +- NativeParquetScan (16) + : +- NativeSort (43) + : +- InputAdapter (42) + : +- AQEShuffleRead (41) + : +- ShuffleQueryStage (40), Statistics(X) + : +- NativeShuffleExchange (39) + : +- NativeProject (38) + : +- NativeFilter (37) + : +- InputAdapter (36) + : +- NativeParquetScan (35) + +- NativeSort (69) + +- InputAdapter (68) + +- AQEShuffleRead (67) + +- ShuffleQueryStage (66), Statistics(X) + +- NativeShuffleExchange (65) + +- NativeProject (64) + +- NativeSortMergeJoin Inner (63) + :- NativeSort (53) + : +- NativeProject (52) + : +- NativeHashAggregate (51) + : +- InputAdapter (50) + : +- InputAdapter (49) + : +- AQEShuffleRead (48) + : +- ShuffleQueryStage (47), Statistics(X) + : +- ReusedExchange (46) + +- NativeSort (62) + +- InputAdapter (61) + +- AQEShuffleRead (60) + +- ShuffleQueryStage (59), Statistics(X) + +- NativeShuffleExchange (58) + +- NativeProject (57) + +- NativeFilter (56) + +- InputAdapter (55) + +- NativeParquetScan (54) +- == Initial Plan == - Sort (135) - +- Exchange (134) - +- Project (133) - +- SortMergeJoin Inner (132) - :- Project (103) - : +- SortMergeJoin Inner (102) - : :- Sort (96) - : : +- HashAggregate (95) - : : +- Exchange (94) - : : +- HashAggregate (93) - : : +- Project (92) - : : +- SortMergeJoin Inner (91) - : : :- Sort (86) - : : : +- Exchange (85) - : : : +- Union (84) - : : : :- Project (80) - : : : : +- Filter (79) - : : : : +- Scan parquet (78) - : : : +- Project (83) - : : : +- Filter (82) - : : : +- Scan parquet (81) - : : +- Sort (90) - : : +- Exchange (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Sort (101) - : +- Exchange (100) - : +- Project (99) - : +- Filter (98) - : +- Scan parquet (97) - +- Sort (131) - +- Exchange (130) - +- Project (129) - +- SortMergeJoin Inner (128) - :- Sort (122) - : +- HashAggregate (121) - : +- Exchange (120) - : +- HashAggregate (119) - : +- Project (118) - : +- SortMergeJoin Inner (117) - : :- Sort (112) - : : +- Exchange (111) - : : +- Union (110) - : : :- Project (106) - : : : +- Filter (105) - : : : +- Scan parquet (104) - : : +- Project (109) - : : +- Filter (108) - : : +- Scan parquet (107) - : +- Sort (116) - : +- Exchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- Sort (127) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) - - -(78) Scan parquet + Sort (134) + +- Exchange (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Project (102) + : +- SortMergeJoin Inner (101) + : :- Sort (95) + : : +- HashAggregate (94) + : : +- Exchange (93) + : : +- HashAggregate (92) + : : +- Project (91) + : : +- SortMergeJoin Inner (90) + : : :- Sort (85) + : : : +- Exchange (84) + : : : +- Union (83) + : : : :- Project (79) + : : : : +- Filter (78) + : : : : +- Scan parquet (77) + : : : +- Project (82) + : : : +- Filter (81) + : : : +- Scan parquet (80) + : : +- Sort (89) + : : +- Exchange (88) + : : +- Filter (87) + : : +- Scan parquet (86) + : +- Sort (100) + : +- Exchange (99) + : +- Project (98) + : +- Filter (97) + : +- Scan parquet (96) + +- Sort (130) + +- Exchange (129) + +- Project (128) + +- SortMergeJoin Inner (127) + :- Sort (121) + : +- HashAggregate (120) + : +- Exchange (119) + : +- HashAggregate (118) + : +- Project (117) + : +- SortMergeJoin Inner (116) + : :- Sort (111) + : : +- Exchange (110) + : : +- Union (109) + : : :- Project (105) + : : : +- Filter (104) + : : : +- Scan parquet (103) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Sort (115) + : +- Exchange (114) + : +- Filter (113) + : +- Scan parquet (112) + +- Sort (126) + +- Exchange (125) + +- Project (124) + +- Filter (123) + +- Scan parquet (122) + + +(77) Scan parquet Output [2]: [ws_sold_date_sk#1, ws_ext_sales_price#2] Batched: true Location: InMemoryFileIndex [file:/] @@ -158,7 +157,7 @@ Condition : isnotnull(ws_sold_date_sk#1) Output [2]: [ws_sold_date_sk#1 AS sold_date_sk#3, ws_ext_sales_price#2 AS sales_price#4] Input [2]: [#1#1, #2#2] -(81) Scan parquet +(80) Scan parquet Output [2]: [cs_sold_date_sk#5, cs_ext_sales_price#6] Batched: true Location: InMemoryFileIndex [file:/] @@ -203,7 +202,7 @@ Input [2]: [#3#3, #4#4] Input [2]: [#3#3, #4#4] Arguments: [sold_date_sk#3 ASC NULLS FIRST], false -(87) Scan parquet +(86) Scan parquet Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location: InMemoryFileIndex [file:/] @@ -288,7 +287,7 @@ Input [8]: [d_week_seq#10, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday) Input [8]: [d_week_seq#10, sun_sales#34, mon_sales#35, tue_sales#36, wed_sales#37, thu_sales#38, fri_sales#39, sat_sales#40] Arguments: [d_week_seq#10 ASC NULLS FIRST], false -(97) Scan parquet +(96) Scan parquet Output [2]: [d_week_seq#41, d_year#42] Batched: true Location: InMemoryFileIndex [file:/] @@ -369,7 +368,7 @@ Input [8]: [d_week_seq#51, sum(UnscaledValue(CASE WHEN (d_day_name#66 = Sunday) Input [8]: [d_week_seq#51, sun_sales#67, mon_sales#68, tue_sales#69, wed_sales#70, thu_sales#71, fri_sales#72, sat_sales#73] Arguments: [d_week_seq#51 ASC NULLS FIRST], false -(123) Scan parquet +(122) Scan parquet Output [2]: [d_week_seq#74, d_year#75] Batched: true Location: InMemoryFileIndex [file:/] @@ -442,307 +441,304 @@ Right keys [1]: [(d_week_seq2#76 - 53)] Join type: Inner Join condition: None -(71) Project [codegen id : X] +(71) NativeProject Output [8]: [d_week_seq1#43, round((sun_sales1#44 / sun_sales2#77), 2) AS round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1#45 / mon_sales2#78), 2) AS round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1#46 / tue_sales2#79), 2) AS round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1#47 / wed_sales2#80), 2) AS round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1#48 / thu_sales2#81), 2) AS round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1#49 / fri_sales2#82), 2) AS round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1#50 / sat_sales2#83), 2) AS round((sat_sales1 / sat_sales2), 2)#90] Input [16]: [d_week_seq1#43, sun_sales1#44, mon_sales1#45, tue_sales1#46, wed_sales1#47, thu_sales1#48, fri_sales1#49, sat_sales1#50, d_week_seq2#76, sun_sales2#77, mon_sales2#78, tue_sales2#79, wed_sales2#80, thu_sales2#81, fri_sales2#82, sat_sales2#83] -(72) ConvertToNative -Input [8]: [d_week_seq1#43, round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1 / sat_sales2), 2)#90] - -(73) NativeShuffleExchange +(72) NativeShuffleExchange Input [8]: [d_week_seq1#43, round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1 / sat_sales2), 2)#90] Arguments: rangepartitioning(d_week_seq1#43 ASC NULLS FIRST, 100), ENSURE_REQUIREMENTS, [plan_id=7] -(74) ShuffleQueryStage +(73) ShuffleQueryStage Output [8]: [d_week_seq1#43, round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1 / sat_sales2), 2)#90] Arguments: X -(75) AQEShuffleRead +(74) AQEShuffleRead Input [8]: [d_week_seq1#43, round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1 / sat_sales2), 2)#90] Arguments: coalesced -(76) InputAdapter +(75) InputAdapter Input [8]: [d_week_seq1#43, round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1 / sat_sales2), 2)#90] -(77) NativeSort +(76) NativeSort Input [8]: [d_week_seq1#43, round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1 / sat_sales2), 2)#90] Arguments: [d_week_seq1#43 ASC NULLS FIRST], true -(78) Scan parquet +(77) Scan parquet Output [2]: [ws_sold_date_sk#1, ws_ext_sales_price#2] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(79) Filter +(78) Filter Input [2]: [ws_sold_date_sk#1, ws_ext_sales_price#2] Condition : isnotnull(ws_sold_date_sk#1) -(80) Project +(79) Project Output [2]: [ws_sold_date_sk#1 AS sold_date_sk#3, ws_ext_sales_price#2 AS sales_price#4] Input [2]: [ws_sold_date_sk#1, ws_ext_sales_price#2] -(81) Scan parquet +(80) Scan parquet Output [2]: [cs_sold_date_sk#5, cs_ext_sales_price#6] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(cs_sold_date_sk)] ReadSchema: struct -(82) Filter +(81) Filter Input [2]: [cs_sold_date_sk#5, cs_ext_sales_price#6] Condition : isnotnull(cs_sold_date_sk#5) -(83) Project +(82) Project Output [2]: [cs_sold_date_sk#5 AS sold_date_sk#7, cs_ext_sales_price#6 AS sales_price#8] Input [2]: [cs_sold_date_sk#5, cs_ext_sales_price#6] -(84) Union +(83) Union -(85) Exchange +(84) Exchange Input [2]: [sold_date_sk#3, sales_price#4] Arguments: hashpartitioning(sold_date_sk#3, 100), ENSURE_REQUIREMENTS, [plan_id=8] -(86) Sort +(85) Sort Input [2]: [sold_date_sk#3, sales_price#4] Arguments: [sold_date_sk#3 ASC NULLS FIRST], false, 0 -(87) Scan parquet +(86) Scan parquet Output [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] ReadSchema: struct -(88) Filter +(87) Filter Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Condition : (isnotnull(d_date_sk#9) AND isnotnull(d_week_seq#10)) -(89) Exchange +(88) Exchange Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Arguments: hashpartitioning(d_date_sk#9, 100), ENSURE_REQUIREMENTS, [plan_id=9] -(90) Sort +(89) Sort Input [3]: [d_date_sk#9, d_week_seq#10, d_day_name#11] Arguments: [d_date_sk#9 ASC NULLS FIRST], false, 0 -(91) SortMergeJoin +(90) SortMergeJoin Left keys [1]: [sold_date_sk#3] Right keys [1]: [d_date_sk#9] Join type: Inner Join condition: None -(92) Project +(91) Project Output [3]: [sales_price#4, d_week_seq#10, d_day_name#11] Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#9, d_week_seq#10, d_day_name#11] -(93) HashAggregate +(92) HashAggregate Input [3]: [sales_price#4, d_week_seq#10, d_day_name#11] Keys [1]: [d_week_seq#10] Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday) THEN sales_price#4 END))] Aggregate Attributes [7]: [sum#19, sum#20, sum#21, sum#22, sum#23, sum#24, sum#25] Results [8]: [d_week_seq#10, sum#91, sum#92, sum#93, sum#94, sum#95, sum#96, sum#97] -(94) Exchange +(93) Exchange Input [8]: [d_week_seq#10, sum#91, sum#92, sum#93, sum#94, sum#95, sum#96, sum#97] Arguments: hashpartitioning(d_week_seq#10, 100), ENSURE_REQUIREMENTS, [plan_id=10] -(95) HashAggregate +(94) HashAggregate Input [8]: [d_week_seq#10, sum#91, sum#92, sum#93, sum#94, sum#95, sum#96, sum#97] Keys [1]: [d_week_seq#10] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday) THEN sales_price#4 END))] Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday) THEN sales_price#4 END))#32, sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday) THEN sales_price#4 END))#33] Results [8]: [d_week_seq#10, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Sunday) THEN sales_price#4 END))#27,17,2) AS sun_sales#34, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Monday) THEN sales_price#4 END))#28,17,2) AS mon_sales#35, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Tuesday) THEN sales_price#4 END))#29,17,2) AS tue_sales#36, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Wednesday) THEN sales_price#4 END))#30,17,2) AS wed_sales#37, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Thursday) THEN sales_price#4 END))#31,17,2) AS thu_sales#38, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Friday) THEN sales_price#4 END))#32,17,2) AS fri_sales#39, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#11 = Saturday) THEN sales_price#4 END))#33,17,2) AS sat_sales#40] -(96) Sort +(95) Sort Input [8]: [d_week_seq#10, sun_sales#34, mon_sales#35, tue_sales#36, wed_sales#37, thu_sales#38, fri_sales#39, sat_sales#40] Arguments: [d_week_seq#10 ASC NULLS FIRST], false, 0 -(97) Scan parquet +(96) Scan parquet Output [2]: [d_week_seq#41, d_year#42] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2001), IsNotNull(d_week_seq)] ReadSchema: struct -(98) Filter +(97) Filter Input [2]: [d_week_seq#41, d_year#42] Condition : ((isnotnull(d_year#42) AND (d_year#42 = 2001)) AND isnotnull(d_week_seq#41)) -(99) Project +(98) Project Output [1]: [d_week_seq#41] Input [2]: [d_week_seq#41, d_year#42] -(100) Exchange +(99) Exchange Input [1]: [d_week_seq#41] Arguments: hashpartitioning(d_week_seq#41, 100), ENSURE_REQUIREMENTS, [plan_id=11] -(101) Sort +(100) Sort Input [1]: [d_week_seq#41] Arguments: [d_week_seq#41 ASC NULLS FIRST], false, 0 -(102) SortMergeJoin +(101) SortMergeJoin Left keys [1]: [d_week_seq#10] Right keys [1]: [d_week_seq#41] Join type: Inner Join condition: None -(103) Project +(102) Project Output [8]: [d_week_seq#10 AS d_week_seq1#43, sun_sales#34 AS sun_sales1#44, mon_sales#35 AS mon_sales1#45, tue_sales#36 AS tue_sales1#46, wed_sales#37 AS wed_sales1#47, thu_sales#38 AS thu_sales1#48, fri_sales#39 AS fri_sales1#49, sat_sales#40 AS sat_sales1#50] Input [9]: [d_week_seq#10, sun_sales#34, mon_sales#35, tue_sales#36, wed_sales#37, thu_sales#38, fri_sales#39, sat_sales#40, d_week_seq#41] -(104) Scan parquet +(103) Scan parquet Output [2]: [ws_sold_date_sk#98, ws_ext_sales_price#99] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(ws_sold_date_sk)] ReadSchema: struct -(105) Filter +(104) Filter Input [2]: [ws_sold_date_sk#98, ws_ext_sales_price#99] Condition : isnotnull(ws_sold_date_sk#98) -(106) Project +(105) Project Output [2]: [ws_sold_date_sk#98 AS sold_date_sk#3, ws_ext_sales_price#99 AS sales_price#4] Input [2]: [ws_sold_date_sk#98, ws_ext_sales_price#99] -(107) Scan parquet +(106) Scan parquet Output [2]: [cs_sold_date_sk#100, cs_ext_sales_price#101] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(cs_sold_date_sk)] ReadSchema: struct -(108) Filter +(107) Filter Input [2]: [cs_sold_date_sk#100, cs_ext_sales_price#101] Condition : isnotnull(cs_sold_date_sk#100) -(109) Project +(108) Project Output [2]: [cs_sold_date_sk#100 AS sold_date_sk#7, cs_ext_sales_price#101 AS sales_price#8] Input [2]: [cs_sold_date_sk#100, cs_ext_sales_price#101] -(110) Union +(109) Union -(111) Exchange +(110) Exchange Input [2]: [sold_date_sk#3, sales_price#4] Arguments: hashpartitioning(sold_date_sk#3, 100), ENSURE_REQUIREMENTS, [plan_id=12] -(112) Sort +(111) Sort Input [2]: [sold_date_sk#3, sales_price#4] Arguments: [sold_date_sk#3 ASC NULLS FIRST], false, 0 -(113) Scan parquet +(112) Scan parquet Output [3]: [d_date_sk#102, d_week_seq#51, d_day_name#66] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_date_sk), IsNotNull(d_week_seq)] ReadSchema: struct -(114) Filter +(113) Filter Input [3]: [d_date_sk#102, d_week_seq#51, d_day_name#66] Condition : (isnotnull(d_date_sk#102) AND isnotnull(d_week_seq#51)) -(115) Exchange +(114) Exchange Input [3]: [d_date_sk#102, d_week_seq#51, d_day_name#66] Arguments: hashpartitioning(d_date_sk#102, 100), ENSURE_REQUIREMENTS, [plan_id=13] -(116) Sort +(115) Sort Input [3]: [d_date_sk#102, d_week_seq#51, d_day_name#66] Arguments: [d_date_sk#102 ASC NULLS FIRST], false, 0 -(117) SortMergeJoin +(116) SortMergeJoin Left keys [1]: [sold_date_sk#3] Right keys [1]: [d_date_sk#102] Join type: Inner Join condition: None -(118) Project +(117) Project Output [3]: [sales_price#4, d_week_seq#51, d_day_name#66] Input [5]: [sold_date_sk#3, sales_price#4, d_date_sk#102, d_week_seq#51, d_day_name#66] -(119) HashAggregate +(118) HashAggregate Input [3]: [sales_price#4, d_week_seq#51, d_day_name#66] Keys [1]: [d_week_seq#51] Functions [7]: [partial_sum(UnscaledValue(CASE WHEN (d_day_name#66 = Sunday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#66 = Monday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#66 = Tuesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#66 = Wednesday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#66 = Thursday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#66 = Friday) THEN sales_price#4 END)), partial_sum(UnscaledValue(CASE WHEN (d_day_name#66 = Saturday) THEN sales_price#4 END))] Aggregate Attributes [7]: [sum#103, sum#104, sum#105, sum#106, sum#107, sum#108, sum#109] Results [8]: [d_week_seq#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] -(120) Exchange +(119) Exchange Input [8]: [d_week_seq#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] Arguments: hashpartitioning(d_week_seq#51, 100), ENSURE_REQUIREMENTS, [plan_id=14] -(121) HashAggregate +(120) HashAggregate Input [8]: [d_week_seq#51, sum#52, sum#53, sum#54, sum#55, sum#56, sum#57, sum#58] Keys [1]: [d_week_seq#51] Functions [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#66 = Sunday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#66 = Monday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#66 = Tuesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#66 = Wednesday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#66 = Thursday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#66 = Friday) THEN sales_price#4 END)), sum(UnscaledValue(CASE WHEN (d_day_name#66 = Saturday) THEN sales_price#4 END))] Aggregate Attributes [7]: [sum(UnscaledValue(CASE WHEN (d_day_name#66 = Sunday) THEN sales_price#4 END))#27, sum(UnscaledValue(CASE WHEN (d_day_name#66 = Monday) THEN sales_price#4 END))#28, sum(UnscaledValue(CASE WHEN (d_day_name#66 = Tuesday) THEN sales_price#4 END))#29, sum(UnscaledValue(CASE WHEN (d_day_name#66 = Wednesday) THEN sales_price#4 END))#30, sum(UnscaledValue(CASE WHEN (d_day_name#66 = Thursday) THEN sales_price#4 END))#31, sum(UnscaledValue(CASE WHEN (d_day_name#66 = Friday) THEN sales_price#4 END))#32, sum(UnscaledValue(CASE WHEN (d_day_name#66 = Saturday) THEN sales_price#4 END))#33] Results [8]: [d_week_seq#51, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#66 = Sunday) THEN sales_price#4 END))#27,17,2) AS sun_sales#67, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#66 = Monday) THEN sales_price#4 END))#28,17,2) AS mon_sales#68, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#66 = Tuesday) THEN sales_price#4 END))#29,17,2) AS tue_sales#69, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#66 = Wednesday) THEN sales_price#4 END))#30,17,2) AS wed_sales#70, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#66 = Thursday) THEN sales_price#4 END))#31,17,2) AS thu_sales#71, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#66 = Friday) THEN sales_price#4 END))#32,17,2) AS fri_sales#72, MakeDecimal(sum(UnscaledValue(CASE WHEN (d_day_name#66 = Saturday) THEN sales_price#4 END))#33,17,2) AS sat_sales#73] -(122) Sort +(121) Sort Input [8]: [d_week_seq#51, sun_sales#67, mon_sales#68, tue_sales#69, wed_sales#70, thu_sales#71, fri_sales#72, sat_sales#73] Arguments: [d_week_seq#51 ASC NULLS FIRST], false, 0 -(123) Scan parquet +(122) Scan parquet Output [2]: [d_week_seq#74, d_year#75] Batched: true Location: InMemoryFileIndex [file:/] PushedFilters: [IsNotNull(d_year), EqualTo(d_year,2002), IsNotNull(d_week_seq)] ReadSchema: struct -(124) Filter +(123) Filter Input [2]: [d_week_seq#74, d_year#75] Condition : ((isnotnull(d_year#75) AND (d_year#75 = 2002)) AND isnotnull(d_week_seq#74)) -(125) Project +(124) Project Output [1]: [d_week_seq#74] Input [2]: [d_week_seq#74, d_year#75] -(126) Exchange +(125) Exchange Input [1]: [d_week_seq#74] Arguments: hashpartitioning(d_week_seq#74, 100), ENSURE_REQUIREMENTS, [plan_id=15] -(127) Sort +(126) Sort Input [1]: [d_week_seq#74] Arguments: [d_week_seq#74 ASC NULLS FIRST], false, 0 -(128) SortMergeJoin +(127) SortMergeJoin Left keys [1]: [d_week_seq#51] Right keys [1]: [d_week_seq#74] Join type: Inner Join condition: None -(129) Project +(128) Project Output [8]: [d_week_seq#51 AS d_week_seq2#76, sun_sales#67 AS sun_sales2#77, mon_sales#68 AS mon_sales2#78, tue_sales#69 AS tue_sales2#79, wed_sales#70 AS wed_sales2#80, thu_sales#71 AS thu_sales2#81, fri_sales#72 AS fri_sales2#82, sat_sales#73 AS sat_sales2#83] Input [9]: [d_week_seq#51, sun_sales#67, mon_sales#68, tue_sales#69, wed_sales#70, thu_sales#71, fri_sales#72, sat_sales#73, d_week_seq#74] -(130) Exchange +(129) Exchange Input [8]: [d_week_seq2#76, sun_sales2#77, mon_sales2#78, tue_sales2#79, wed_sales2#80, thu_sales2#81, fri_sales2#82, sat_sales2#83] Arguments: hashpartitioning((d_week_seq2#76 - 53), 100), ENSURE_REQUIREMENTS, [plan_id=16] -(131) Sort +(130) Sort Input [8]: [d_week_seq2#76, sun_sales2#77, mon_sales2#78, tue_sales2#79, wed_sales2#80, thu_sales2#81, fri_sales2#82, sat_sales2#83] Arguments: [(d_week_seq2#76 - 53) ASC NULLS FIRST], false, 0 -(132) SortMergeJoin +(131) SortMergeJoin Left keys [1]: [d_week_seq1#43] Right keys [1]: [(d_week_seq2#76 - 53)] Join type: Inner Join condition: None -(133) Project +(132) Project Output [8]: [d_week_seq1#43, round((sun_sales1#44 / sun_sales2#77), 2) AS round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1#45 / mon_sales2#78), 2) AS round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1#46 / tue_sales2#79), 2) AS round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1#47 / wed_sales2#80), 2) AS round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1#48 / thu_sales2#81), 2) AS round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1#49 / fri_sales2#82), 2) AS round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1#50 / sat_sales2#83), 2) AS round((sat_sales1 / sat_sales2), 2)#90] Input [16]: [d_week_seq1#43, sun_sales1#44, mon_sales1#45, tue_sales1#46, wed_sales1#47, thu_sales1#48, fri_sales1#49, sat_sales1#50, d_week_seq2#76, sun_sales2#77, mon_sales2#78, tue_sales2#79, wed_sales2#80, thu_sales2#81, fri_sales2#82, sat_sales2#83] -(134) Exchange +(133) Exchange Input [8]: [d_week_seq1#43, round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1 / sat_sales2), 2)#90] Arguments: rangepartitioning(d_week_seq1#43 ASC NULLS FIRST, 100), ENSURE_REQUIREMENTS, [plan_id=17] -(135) Sort +(134) Sort Input [8]: [d_week_seq1#43, round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1 / sat_sales2), 2)#90] Arguments: [d_week_seq1#43 ASC NULLS FIRST], true, 0 -(136) AdaptiveSparkPlan +(135) AdaptiveSparkPlan Output [8]: [d_week_seq1#43, round((sun_sales1 / sun_sales2), 2)#84, round((mon_sales1 / mon_sales2), 2)#85, round((tue_sales1 / tue_sales2), 2)#86, round((wed_sales1 / wed_sales2), 2)#87, round((thu_sales1 / thu_sales2), 2)#88, round((fri_sales1 / fri_sales2), 2)#89, round((sat_sales1 / sat_sales2), 2)#90] Arguments: isFinalPlan=true diff --git a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronExpressionSuite.scala b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronExpressionSuite.scala index b0006d363..d40c95524 100644 --- a/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronExpressionSuite.scala +++ b/spark-extension-shims-spark/src/test/scala/org/apache/auron/AuronExpressionSuite.scala @@ -20,6 +20,20 @@ import org.apache.spark.sql.AuronQueryTest class AuronExpressionSuite extends AuronQueryTest with BaseAuronSQLSuite { + test("preserve literal parameters during expression fallback") { + withSQLConf( + "spark.auron.decimal.arithOp.enabled" -> "false", + "spark.auron.udf.singleChildFallback.enabled" -> "true") { + withTable("literal_fallback") { + sql("CREATE TABLE literal_fallback(a DECIMAL(17,2), b DECIMAL(17,2)) USING parquet") + sql("""INSERT INTO literal_fallback VALUES + |(123.45, 7.00), (-123.45, 7.00), (1.00, 8.00), (NULL, 7.00), (1.00, NULL) + |""".stripMargin) + checkSparkAnswerAndOperator("SELECT round(a / b, 2) FROM literal_fallback") + } + } + } + test("EqualNullSafe") { withTable("t1") { sql("CREATE TABLE t1(id INT, flag BOOLEAN) USING parquet") diff --git a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala index 6ad0931bd..ce3f4b92f 100644 --- a/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala +++ b/spark-extension/src/main/scala/org/apache/spark/sql/auron/NativeConverters.scala @@ -355,16 +355,19 @@ object NativeConverters extends Logging { numInconvertibleChildren match { case 0 => convertExprWithFallback(sparkExpr, isPruningExpr = false, fallbackToError) case 1 => - val childrenConverted = sparkExpr.mapChildren { child => - try { - val converted = - convertExprWithFallback(child, isPruningExpr = false, fallbackToError) - Shims.get.createNativeExprWrapper(converted, child.dataType, child.nullable) - } catch { - case _: NotImplementedError => - val fallbacked = convertExpr(child) - Shims.get.createNativeExprWrapper(fallbacked, child.dataType, child.nullable) - } + val childrenConverted = sparkExpr.mapChildren { + // Preserve literal parameters when wrapping child expressions. + case literal: Literal => literal + case child => + try { + val converted = + convertExprWithFallback(child, isPruningExpr = false, fallbackToError) + Shims.get.createNativeExprWrapper(converted, child.dataType, child.nullable) + } catch { + case _: NotImplementedError => + val fallbacked = convertExpr(child) + Shims.get.createNativeExprWrapper(fallbacked, child.dataType, child.nullable) + } } convertExprWithFallback(childrenConverted, isPruningExpr = false, fallbackToError) case _ =>