Merge remote-tracking branch 'origin/main' into fix/exact-hash-table-accounting-primitive

Author: mohitgurav20

.github/workflows/rust.yml

@@ -700,10 +700,12 @@ jobs:
           rust-version: stable
       - name: Run sqllogictest
         # TODO: Right now several tests are failing in Substrait round-trip mode, so this
-        #  command cannot be run for all the .slt files. Run it for just one that works (limit.slt)
+        #  command cannot be run for all the .slt files. Run a supported subset
         #  until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed
         #  and this command can be run without filters.
-        run: cargo xtask ci step test substrait
+        run: |
+          cargo xtask ci step test substrait
+          cargo xtask ci step test substrait-optimized
 
   #  Temporarily commenting out the Windows flow, the reason is enormously slow running build
   #  Waiting for new Windows 2025 github runner

AGENTS.md

@@ -4,12 +4,19 @@
 
 - [Quick Start Setup](docs/source/contributor-guide/development_environment.md#quick-start)
 - [Testing Quick Start](docs/source/contributor-guide/testing.md#testing-quick-start)
+- [Open Contribution and Assigning tickets](docs/source/contributor-guide/index.md#open-contribution-and-assigning-tickets)
 - [Choosing What Kind of Test to Write](docs/source/contributor-guide/testing.md#choosing-what-kind-of-test-to-write)
 - [Before Submitting a PR](docs/source/contributor-guide/index.md#before-submitting-a-pr)
 - [Reviewing Pull Requests](docs/source/contributor-guide/pr_review.md)
 - [Contributor Guide](docs/source/contributor-guide/index.md)
 - [Architecture Guide](docs/source/contributor-guide/architecture.md)
 
+## Before Starting Work
+
+Before you start work on an issue, you MUST follow the instructions in
+[Open Contribution and Assigning tickets](docs/source/contributor-guide/index.md#open-contribution-and-assigning-tickets). You must ensure duplicate work is not being created.
+Do not create multiple PRs if this is your first time contributing.
+
 ## Before Committing
 
 Before committing any changes, you MUST follow the instructions in

datafusion/common/benches/with_hashes.rs

@@ -18,9 +18,10 @@
 //! Benchmarks for `with_hashes` function
 
 use arrow::array::{
-    Array, ArrayRef, ArrowPrimitiveType, DictionaryArray, GenericStringArray, Int32Array,
-    Int64Array, ListArray, MapArray, NullBufferBuilder, OffsetSizeTrait, PrimitiveArray,
-    RunArray, StringViewArray, StructArray, UnionArray, make_array,
+    Array, ArrayRef, ArrowPrimitiveType, DictionaryArray, FixedSizeListArray,
+    GenericStringArray, Int32Array, Int64Array, ListArray, ListViewArray, MapArray,
+    NullArray, NullBufferBuilder, OffsetSizeTrait, PrimitiveArray, RunArray,
+    StringViewArray, StructArray, UnionArray, make_array,
 };
 use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer};
 use arrow::datatypes::{
@@ -127,6 +128,21 @@ fn criterion_benchmark(c: &mut Criterion) {
             array: create_run_array::<Int32Type>(BATCH_SIZE),
             supports_nulls: true,
         },
+        BenchData {
+            name: "null",
+            array: Arc::new(NullArray::new(BATCH_SIZE)),
+            supports_nulls: false,
+        },
+        BenchData {
+            name: "list_view_array",
+            array: list_view_array(BATCH_SIZE),
+            supports_nulls: true,
+        },
+        BenchData {
+            name: "fixed_size_list_array",
+            array: fixed_size_list_array(BATCH_SIZE),
+            supports_nulls: true,
+        },
     ];
 
     for BenchData {
@@ -603,5 +619,49 @@ where
     )
 }
 
-criterion_group!(benches, criterion_benchmark, sliced_array_benchmark);
+fn list_view_array(num_rows: usize) -> ArrayRef {
+    let elements_per_row = 5;
+    let values = primitive_array::<Int64Type>(num_rows * elements_per_row);
+    let offsets: ScalarBuffer<i32> = (0..num_rows)
+        .map(|i| (i * elements_per_row) as i32)
+        .collect();
+    let sizes: ScalarBuffer<i32> =
+        (0..num_rows).map(|_| elements_per_row as i32).collect();
+    Arc::new(ListViewArray::new(
+        Arc::new(Field::new("item", DataType::Int64, true)),
+        offsets,
+        sizes,
+        values,
+        None,
+    ))
+}
+
+fn fixed_size_list_array(num_rows: usize) -> ArrayRef {
+    let list_size = 4;
+    Arc::new(FixedSizeListArray::new(
+        Arc::new(Field::new("item", DataType::Int64, true)),
+        list_size as i32,
+        primitive_array::<Int64Type>(num_rows * list_size),
+        None,
+    ))
+}
+
+/// Heterogeneous key columns
+fn mixed_columns_benchmark(c: &mut Criterion) {
+    let pool = StringPool::new(100, 64);
+    let int64 = primitive_array::<Int64Type>(BATCH_SIZE);
+    let utf8 = pool.string_array::<i32>(BATCH_SIZE);
+    let utf8_view = pool.string_view_array(BATCH_SIZE);
+    let arrays = vec![int64, utf8, utf8_view];
+    c.bench_function("mixed: 3 columns (int64, utf8, utf8_view)", |b| {
+        do_hash_test(b, &arrays)
+    });
+}
+
+criterion_group!(
+    benches,
+    criterion_benchmark,
+    sliced_array_benchmark,
+    mixed_columns_benchmark
+);
 criterion_main!(benches);

datafusion/common/src/config.rs

@@ -1809,7 +1809,9 @@ config_namespace! {
         /// rule. When set to false, any rules that produce errors will cause the query to fail
         pub skip_failed_rules: bool, default = false
 
-        /// Number of times that the optimizer will attempt to optimize the plan
+        /// The logical optimizer applies rules in order for up to `max_passes` passes,
+        /// stopping early if a pass leaves the plan unchanged or repeats an earlier plan.
+        /// The physical optimizer always runs in one pass.
         pub max_passes: usize, default = 3
 
         /// When set to true, the physical plan optimizer will run a top down

datafusion/core/tests/fuzz_cases/join_fuzz.rs

@@ -1604,14 +1604,14 @@ fn pwmj_plan(
     join_type: JoinType,
 ) -> Arc<dyn ExecutionPlan> {
     // Matches `PiecewiseMergeJoinExec::required_input_ordering`: descending for `<`/`<=`,
-    // ascending for `>`/`>=`, NULLs first either way. Right existence joins require no
+    // ascending for `>`/`>=`, reversing NULL placement too. Right existence joins require no
     // ordering at all -- they only read the buffered side's min/max -- so they are fed the
     // left side unsorted, which is the input shape they will see in a real plan.
     let buffered = match join_type {
         JoinType::RightSemi | JoinType::RightAnti | JoinType::RightMark => left,
         _ => {
             let sort_options = match op {
-                Operator::Lt | Operator::LtEq => SortOptions::new(true, true),
+                Operator::Lt | Operator::LtEq => SortOptions::new(true, false),
                 Operator::Gt | Operator::GtEq => SortOptions::new(false, true),
                 other => panic!("not a range operator: {other:?}"),
             };