|
22 | 22 | use std::sync::Arc; |
23 | 23 |
|
24 | 24 | use arrow::compute::BatchCoalescer; |
25 | | -use arrow::datatypes::SchemaRef; |
| 25 | +use arrow::datatypes::{DataType, SchemaRef}; |
26 | 26 | use arrow::record_batch::RecordBatch; |
27 | 27 | use datafusion_common::{DataFusionError, Result}; |
28 | 28 | use datafusion_execution::memory_pool::MemoryReservation; |
@@ -107,20 +107,16 @@ impl BucketedAggregation { |
107 | 107 |
|
108 | 108 | /// True if a schema of partial state rows can be bucketed. |
109 | 109 | /// |
110 | | - /// Bucketing turns the state of a table into rows and merges those rows |
111 | | - /// again. That is cheap for fixed-width and string state, but nested |
112 | | - /// state (the lists kept by `count(distinct)`, `array_agg` or `median`) |
113 | | - /// is costly to rebuild and takes more memory as rows than inside the |
114 | | - /// accumulator, so such aggregations keep their single table. |
115 | | - pub(super) fn supports_state( |
116 | | - state_schema: &SchemaRef, |
117 | | - num_group_columns: usize, |
118 | | - ) -> bool { |
119 | | - !state_schema |
120 | | - .fields() |
121 | | - .iter() |
122 | | - .skip(num_group_columns) |
123 | | - .any(|field| field.data_type().is_nested()) |
| 110 | + /// Bucketing moves every row once more before it is aggregated, which only |
| 111 | + /// pays when moving a row is cheap compared to the cache miss it saves. |
| 112 | + /// That holds for fixed-width columns. Variable-length keys or state |
| 113 | + /// (strings, lists, ...) cost as much to move as the smaller tables save, |
| 114 | + /// so such aggregations keep their single table. |
| 115 | + pub(super) fn supports_state(state_schema: &SchemaRef) -> bool { |
| 116 | + state_schema.fields().iter().all(|field| { |
| 117 | + let data_type = field.data_type(); |
| 118 | + data_type.is_primitive() || *data_type == DataType::Boolean |
| 119 | + }) |
124 | 120 | } |
125 | 121 |
|
126 | 122 | pub(super) fn threshold(&self) -> usize { |
@@ -365,3 +361,47 @@ impl BucketedAggregation { |
365 | 361 | }) |
366 | 362 | } |
367 | 363 | } |
| 364 | + |
| 365 | +#[cfg(test)] |
| 366 | +mod tests { |
| 367 | + use super::*; |
| 368 | + |
| 369 | + use arrow::datatypes::{Field, Fields, Schema}; |
| 370 | + |
| 371 | + fn schema(types: Vec<DataType>) -> SchemaRef { |
| 372 | + let fields: Fields = types |
| 373 | + .into_iter() |
| 374 | + .enumerate() |
| 375 | + .map(|(i, data_type)| Field::new(format!("c{i}"), data_type, true)) |
| 376 | + .collect(); |
| 377 | + Arc::new(Schema::new(fields)) |
| 378 | + } |
| 379 | + |
| 380 | + #[test] |
| 381 | + fn only_fixed_width_state_is_bucketed() { |
| 382 | + assert!(BucketedAggregation::supports_state(&schema(vec![ |
| 383 | + DataType::Int64, |
| 384 | + DataType::Boolean, |
| 385 | + DataType::Decimal128(38, 10), |
| 386 | + DataType::Float64, |
| 387 | + DataType::Date32, |
| 388 | + ]))); |
| 389 | + |
| 390 | + // A variable-length group key or state column turns bucketing off |
| 391 | + for variable_length in [ |
| 392 | + DataType::Utf8, |
| 393 | + DataType::Utf8View, |
| 394 | + DataType::LargeBinary, |
| 395 | + DataType::List(Arc::new(Field::new_list_field(DataType::Int64, true))), |
| 396 | + ] { |
| 397 | + assert!(!BucketedAggregation::supports_state(&schema(vec![ |
| 398 | + DataType::Int64, |
| 399 | + variable_length.clone(), |
| 400 | + ]))); |
| 401 | + assert!(!BucketedAggregation::supports_state(&schema(vec![ |
| 402 | + variable_length, |
| 403 | + DataType::Int64, |
| 404 | + ]))); |
| 405 | + } |
| 406 | + } |
| 407 | +} |
0 commit comments