From ad500322f3706abe1f0edccb9ca814ed9d9f9482 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 20 Aug 2026 16:25:26 +0800 Subject: [PATCH 1/2] fix: tree-style explain for memory table shows wrong row count --- datafusion/datasource/src/memory.rs | 7 ++++++- .../sqllogictest/test_files/explain.slt | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/datafusion/datasource/src/memory.rs b/datafusion/datasource/src/memory.rs index 2370ed87a2954..a4f51c28f83ac 100644 --- a/datafusion/datasource/src/memory.rs +++ b/datafusion/datasource/src/memory.rs @@ -129,7 +129,12 @@ impl DataSource for MemorySourceConfig { } } DisplayFormatType::TreeRender => { - let total_rows = self.partitions.iter().map(|b| b.len()).sum::(); + let total_rows = self + .partitions + .iter() + .flatten() + .map(RecordBatch::num_rows) + .sum::(); let total_bytes: usize = self .partitions .iter() diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.slt index b6837002086ad..b7fced521a01d 100644 --- a/datafusion/sqllogictest/test_files/explain.slt +++ b/datafusion/sqllogictest/test_files/explain.slt @@ -839,3 +839,23 @@ physical_plan statement ok reset datafusion.sql_parser.dialect; + +# Test `DataSourceExec` tree-style explain +statement ok +CREATE TABLE test_memtable AS +SELECT * +FROM generate_series(100); + +query TT +EXPLAIN FORMAT TREE +SELECT * +FROM test_memtable; +---- +physical_plan +01)┌───────────────────────────┐ +02)│ DataSourceExec │ +03)│ -------------------- │ +04)│ bytes: 65632 │ +05)│ format: memory │ +06)│ rows: 101 │ +07)└───────────────────────────┘ From d75e726116128ed3ba9822e258c881168ce557b5 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Thu, 20 Aug 2026 16:46:06 +0800 Subject: [PATCH 2/2] fix existing tests --- .../test_files/aggregate_memory_spill.slt | 2 +- .../test_files/array/init_data.slt.part | 1 - .../sqllogictest/test_files/array_add.slt | 2 +- .../sqllogictest/test_files/array_product.slt | 2 +- .../test_files/array_subtract.slt | 2 +- .../sqllogictest/test_files/explain.slt | 20 ---------------- .../sqllogictest/test_files/explain_tree.slt | 24 +++++++++---------- datafusion/sqllogictest/test_files/insert.slt | 2 +- .../test_files/join_is_not_distinct_from.slt | 2 +- datafusion/sqllogictest/test_files/limit.slt | 2 +- .../test_files/monotonic_projection_test.slt | 1 - .../test_files/ordered_aggregate_spill.slt | 2 +- .../sqllogictest/test_files/repartition.slt | 2 +- .../test_files/spark/math/hypot.slt | 2 +- datafusion/sqllogictest/test_files/struct.slt | 1 - .../sqllogictest/test_files/window_topn.slt | 2 +- 16 files changed, 23 insertions(+), 46 deletions(-) diff --git a/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt b/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt index 3dbf880fd1fa9..79c6cb7a5155e 100644 --- a/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt +++ b/datafusion/sqllogictest/test_files/aggregate_memory_spill.slt @@ -209,7 +209,7 @@ FROM ( 100000 5000050000 # Assert spill happened in the final aggregation. -# In multi-partitions configuration, 'spilled_rows' is not deterministic, so assert +# In multi-partitions configuration, 'spilled_rows' is not deterministic, so assert # the unit to be 'K' query TT EXPLAIN ANALYZE diff --git a/datafusion/sqllogictest/test_files/array/init_data.slt.part b/datafusion/sqllogictest/test_files/array/init_data.slt.part index bb8d76809f816..06d029303022b 100644 --- a/datafusion/sqllogictest/test_files/array/init_data.slt.part +++ b/datafusion/sqllogictest/test_files/array/init_data.slt.part @@ -688,4 +688,3 @@ AS arrow_cast(column4, 'FixedSizeList(3, Float64)') AS column4 FROM arrays_distance_table ; - diff --git a/datafusion/sqllogictest/test_files/array_add.slt b/datafusion/sqllogictest/test_files/array_add.slt index e13f6acd269cb..acef9984fdbcc 100644 --- a/datafusion/sqllogictest/test_files/array_add.slt +++ b/datafusion/sqllogictest/test_files/array_add.slt @@ -234,4 +234,4 @@ select array_add(array_add(a, b), c) from (values ---- [111.0, 222.0] NULL -[111.0, NULL] \ No newline at end of file +[111.0, NULL] diff --git a/datafusion/sqllogictest/test_files/array_product.slt b/datafusion/sqllogictest/test_files/array_product.slt index ba60360d1c1a9..a904282358dff 100644 --- a/datafusion/sqllogictest/test_files/array_product.slt +++ b/datafusion/sqllogictest/test_files/array_product.slt @@ -142,4 +142,4 @@ select list_product(column1) from (values ) as t(column1); ---- 6 -NULL \ No newline at end of file +NULL diff --git a/datafusion/sqllogictest/test_files/array_subtract.slt b/datafusion/sqllogictest/test_files/array_subtract.slt index 4a680c93aae95..28edd6ba9b0ed 100644 --- a/datafusion/sqllogictest/test_files/array_subtract.slt +++ b/datafusion/sqllogictest/test_files/array_subtract.slt @@ -234,4 +234,4 @@ select array_subtract(array_subtract(a, b), c) from (values ---- [89.0, 178.0] NULL -[89.0, NULL] \ No newline at end of file +[89.0, NULL] diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.slt index b7fced521a01d..b6837002086ad 100644 --- a/datafusion/sqllogictest/test_files/explain.slt +++ b/datafusion/sqllogictest/test_files/explain.slt @@ -839,23 +839,3 @@ physical_plan statement ok reset datafusion.sql_parser.dialect; - -# Test `DataSourceExec` tree-style explain -statement ok -CREATE TABLE test_memtable AS -SELECT * -FROM generate_series(100); - -query TT -EXPLAIN FORMAT TREE -SELECT * -FROM test_memtable; ----- -physical_plan -01)┌───────────────────────────┐ -02)│ DataSourceExec │ -03)│ -------------------- │ -04)│ bytes: 65632 │ -05)│ format: memory │ -06)│ rows: 101 │ -07)└───────────────────────────┘ diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt index 4e0397bb41e2e..fcac86aa21a2e 100644 --- a/datafusion/sqllogictest/test_files/explain_tree.slt +++ b/datafusion/sqllogictest/test_files/explain_tree.slt @@ -270,7 +270,7 @@ physical_plan 08)│ -------------------- │ 09)│ bytes: 1024 │ 10)│ format: memory │ -11)│ rows: 2 │ +11)│ rows: 6 │ 12)└───────────────────────────┘ # 2 Joins @@ -331,7 +331,7 @@ physical_plan 09)│ -------------------- ││ -------------------- │ 10)│ bytes: 512 ││ on: ├──────────────┐ 11)│ format: memory ││ (int_col = int_col) │ │ -12)│ rows: 1 ││ │ │ +12)│ rows: 3 ││ │ │ 13)└───────────────────────────┘└─────────────┬─────────────┘ │ 14)-----------------------------┌─────────────┴─────────────┐┌─────────────┴─────────────┐ 15)-----------------------------│ DataSourceExec ││ RepartitionExec │ @@ -570,7 +570,7 @@ physical_plan 09)│ -------------------- │ 10)│ bytes: 512 │ 11)│ format: memory │ -12)│ rows: 1 │ +12)│ rows: 3 │ 13)└───────────────────────────┘ # Query with filter on json @@ -927,7 +927,7 @@ physical_plan 15)│ -------------------- │ 16)│ bytes: 512 │ 17)│ format: memory │ -18)│ rows: 1 │ +18)│ rows: 3 │ 19)└───────────────────────────┘ # Query with projection on json @@ -1177,7 +1177,7 @@ physical_plan 13)│ -------------------- ││ -------------------- │ 14)│ bytes: 5932 ││ bytes: 5932 │ 15)│ format: memory ││ format: memory │ -16)│ rows: 1 ││ rows: 1 │ +16)│ rows: 4 ││ rows: 4 │ 17)└───────────────────────────┘└───────────────────────────┘ statement ok @@ -1276,7 +1276,7 @@ physical_plan 44)│ -------------------- ││ -------------------- │ 45)│ bytes: 288 ││ bytes: 280 │ 46)│ format: memory ││ format: memory │ -47)│ rows: 1 ││ rows: 1 │ +47)│ rows: 3 ││ rows: 3 │ 48)└───────────────────────────┘└───────────────────────────┘ # Test explain tree for UnionExec @@ -1295,14 +1295,14 @@ physical_plan 06)│ -------------------- ││ -------------------- │ 07)│ bytes: 288 ││ id: CAST(id AS Int32) │ 08)│ format: memory ││ name: name │ -09)│ rows: 1 ││ │ +09)│ rows: 3 ││ │ 10)└───────────────────────────┘└─────────────┬─────────────┘ 11)-----------------------------┌─────────────┴─────────────┐ 12)-----------------------------│ DataSourceExec │ 13)-----------------------------│ -------------------- │ 14)-----------------------------│ bytes: 280 │ 15)-----------------------------│ format: memory │ -16)-----------------------------│ rows: 1 │ +16)-----------------------------│ rows: 3 │ 17)-----------------------------└───────────────────────────┘ # cleanup @@ -1646,7 +1646,7 @@ physical_plan 13)│ -------------------- │ 14)│ bytes: 2576 │ 15)│ format: memory │ -16)│ rows: 1 │ +16)│ rows: 3 │ 17)└───────────────────────────┘ query TT @@ -1669,7 +1669,7 @@ physical_plan 13)│ -------------------- │ 14)│ bytes: 2576 │ 15)│ format: memory │ -16)│ rows: 1 │ +16)│ rows: 3 │ 17)└───────────────────────────┘ query TT @@ -1692,7 +1692,7 @@ physical_plan 13)│ -------------------- │ 14)│ bytes: 2576 │ 15)│ format: memory │ -16)│ rows: 1 │ +16)│ rows: 3 │ 17)└───────────────────────────┘ @@ -1754,7 +1754,7 @@ physical_plan 48)│ -------------------- │ 49)│ bytes: 160 │ 50)│ format: memory │ -51)│ rows: 1 │ +51)│ rows: 10 │ 52)└───────────────────────────┘ # clean up diff --git a/datafusion/sqllogictest/test_files/insert.slt b/datafusion/sqllogictest/test_files/insert.slt index c807f73d60fe0..ddb954bcfebc6 100644 --- a/datafusion/sqllogictest/test_files/insert.slt +++ b/datafusion/sqllogictest/test_files/insert.slt @@ -463,7 +463,7 @@ drop table unsigned_bigint_test # Config reset -# The SLT runner sets `target_partitions` to 4 instead of using the default, so +# The SLT runner sets `target_partitions` to 4 instead of using the default, so # reset it explicitly. statement ok set datafusion.execution.target_partitions = 4; diff --git a/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt b/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt index 1b6f2e4c86385..eaafd70588772 100644 --- a/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt +++ b/datafusion/sqllogictest/test_files/join_is_not_distinct_from.slt @@ -113,7 +113,7 @@ physical_plan 17)│ -------------------- ││ -------------------- │ 18)│ bytes: 288 ││ bytes: 288 │ 19)│ format: memory ││ format: memory │ -20)│ rows: 1 ││ rows: 1 │ +20)│ rows: 4 ││ rows: 4 │ 21)└───────────────────────────┘└───────────────────────────┘ statement ok diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt index 58a655c02b2fc..c1e4b054c318f 100644 --- a/datafusion/sqllogictest/test_files/limit.slt +++ b/datafusion/sqllogictest/test_files/limit.slt @@ -401,7 +401,7 @@ physical_plan 09)│ -------------------- │ 10)│ bytes: 160 │ 11)│ format: memory │ -12)│ rows: 1 │ +12)│ rows: 10 │ 13)└───────────────────────────┘ # generate BIGINT data from 1 to 1000 in multiple partitions diff --git a/datafusion/sqllogictest/test_files/monotonic_projection_test.slt b/datafusion/sqllogictest/test_files/monotonic_projection_test.slt index 71e5fbc08e3eb..92e3744d40d9a 100644 --- a/datafusion/sqllogictest/test_files/monotonic_projection_test.slt +++ b/datafusion/sqllogictest/test_files/monotonic_projection_test.slt @@ -314,4 +314,3 @@ DROP TABLE t2_int_float; statement ok DROP TABLE t3_int_bigint; - diff --git a/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt b/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt index 2c53c94144eb3..67fa20101dbfe 100644 --- a/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt +++ b/datafusion/sqllogictest/test_files/ordered_aggregate_spill.slt @@ -73,7 +73,7 @@ GROUP BY v1 # ================================================================================== # Input is partially ordered by group keys (input order by (a), query is 'group by a,b') -# +# # Try different memory limits, ensure result is the same, but spill count differ # HACK: check `spilled_bytes=x KB` to ensure it has spilled. If it has not spilled, diff --git a/datafusion/sqllogictest/test_files/repartition.slt b/datafusion/sqllogictest/test_files/repartition.slt index cf913caefc525..7cec3b5844112 100644 --- a/datafusion/sqllogictest/test_files/repartition.slt +++ b/datafusion/sqllogictest/test_files/repartition.slt @@ -140,7 +140,7 @@ FROM t1 WHERE ((false > (v1 = v1)) IS DISTINCT FROM true); # Config reset -# The SLT runner sets `target_partitions` to 4 instead of using the default, so +# The SLT runner sets `target_partitions` to 4 instead of using the default, so # reset it explicitly. statement ok set datafusion.execution.target_partitions = 4; diff --git a/datafusion/sqllogictest/test_files/spark/math/hypot.slt b/datafusion/sqllogictest/test_files/spark/math/hypot.slt index 564b34add8b9f..519383d023cdc 100644 --- a/datafusion/sqllogictest/test_files/spark/math/hypot.slt +++ b/datafusion/sqllogictest/test_files/spark/math/hypot.slt @@ -132,4 +132,4 @@ NaN query R SELECT hypot('Infinity'::double, '-Infinity'::double); ---- -Infinity \ No newline at end of file +Infinity diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt index a0e47a8691f34..941bab7c2d866 100644 --- a/datafusion/sqllogictest/test_files/struct.slt +++ b/datafusion/sqllogictest/test_files/struct.slt @@ -1732,4 +1732,3 @@ drop view leaf_view; statement ok drop table leaf_base; - diff --git a/datafusion/sqllogictest/test_files/window_topn.slt b/datafusion/sqllogictest/test_files/window_topn.slt index 8f65e496b502d..a50b2d9e7e7b6 100644 --- a/datafusion/sqllogictest/test_files/window_topn.slt +++ b/datafusion/sqllogictest/test_files/window_topn.slt @@ -283,7 +283,7 @@ physical_plan 54)│ -------------------- │ 55)│ bytes: 480 │ 56)│ format: memory │ -57)│ rows: 1 │ +57)│ rows: 10 │ 58)└───────────────────────────┘ statement ok