Skip to content

Commit 934ed0f

Browse files
mohitgurav20mohitgurav20
authored
Merge branch 'main' into fix/estimate-equality-scalar-subquery
2 parents a2218dc + 264ee3d commit 934ed0f

40 files changed

Lines changed: 1632 additions & 174 deletions

File tree

‎.github/workflows/audit.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ jobs:
4545
steps:
4646
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
4747
- name: Install cargo-audit
48-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
48+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
4949
with:
5050
tool: cargo-audit
5151
- name: Run audit check

‎.github/workflows/breaking_changes_detector.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ jobs:
8989
9090
- name: Install cargo-semver-checks
9191
if: steps.changed_crates.outputs.packages != ''
92-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
92+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
9393
with:
9494
tool: cargo-semver-checks
9595

‎.github/workflows/dependencies.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ jobs:
6969
. ci/scripts/utils/tool_versions.sh
7070
echo "CARGO_MACHETE_VERSION=${CARGO_MACHETE_VERSION}" >> "$GITHUB_ENV"
7171
- name: Install cargo-machete
72-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
72+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
7373
with:
7474
tool: cargo-machete@${{ env.CARGO_MACHETE_VERSION }}
7575
- name: Detect unused dependencies

‎.github/workflows/dev.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ jobs:
4242
steps:
4343
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
4444
- name: Install HawkEye
45-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
45+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
4646
with:
4747
tool: hawkeye@7.0.0
4848
- name: Run license header check
@@ -70,7 +70,7 @@ jobs:
7070
source ci/scripts/utils/tool_versions.sh
7171
echo "LYCHEE_VERSION=${LYCHEE_VERSION}" >> "$GITHUB_ENV"
7272
- name: Install lychee
73-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
73+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
7474
with:
7575
tool: lychee@${{ env.LYCHEE_VERSION }}
7676
- name: Run markdown link check
@@ -95,7 +95,7 @@ jobs:
9595
# it may cause checks to fail more often.
9696
# We can upgrade it manually once a while.
9797
- name: Install typos
98-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
98+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
9999
with:
100100
tool: typos@1.37.0
101101
- name: Run typos check

‎.github/workflows/docs.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ jobs:
5353
sudo apt-get update
5454
sudo apt-get install -y graphviz
5555
- name: Install cargo-depgraph
56-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
56+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
5757
with:
5858
tool: cargo-depgraph@1.6
5959

‎.github/workflows/docs_pr.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ jobs:
6161
sudo apt-get update
6262
sudo apt-get install -y graphviz
6363
- name: Install cargo-depgraph
64-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
64+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
6565
with:
6666
tool: cargo-depgraph@1.6
6767
- name: Build docs html and check for warnings

‎.github/workflows/extended.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ jobs:
6060
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
6161
# note: do not use amd/rust container to preserve disk space
6262
steps:
63-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
63+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
6464
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
6565
with:
6666
ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
@@ -99,7 +99,7 @@ jobs:
9999
container:
100100
image: amd64/rust
101101
steps:
102-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
102+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
103103
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
104104
with:
105105
ref: ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
@@ -123,7 +123,7 @@ jobs:
123123
container:
124124
image: amd64/rust
125125
steps:
126-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
126+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
127127
- parallel:
128128
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
129129
with:

‎.github/workflows/rust.yml‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ jobs:
7575
container:
7676
image: amd64/rust
7777
steps:
78-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
78+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
7979
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
8080
- name: Setup Rust toolchain
8181
uses: ./.github/actions/setup-builder
@@ -204,7 +204,7 @@ jobs:
204204
container:
205205
image: amd64/rust
206206
steps:
207-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
207+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
208208
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
209209
- name: Setup Rust toolchain
210210
uses: ./.github/actions/setup-builder
@@ -240,7 +240,7 @@ jobs:
240240
container:
241241
image: amd64/rust
242242
steps:
243-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
243+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
244244
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
245245
- name: Setup Rust toolchain
246246
uses: ./.github/actions/setup-builder
@@ -269,7 +269,7 @@ jobs:
269269
container:
270270
image: amd64/rust
271271
steps:
272-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
272+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
273273
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
274274
- name: Setup Rust toolchain
275275
uses: ./.github/actions/setup-builder
@@ -418,7 +418,7 @@ jobs:
418418
volumes:
419419
- /usr/local:/host/usr/local
420420
steps:
421-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
421+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
422422
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
423423
with:
424424
submodules: true
@@ -430,7 +430,7 @@ jobs:
430430
- name: Install llvm-tools-preview
431431
run: rustup component add llvm-tools-preview
432432
- name: Install cargo-llvm-cov
433-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
433+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
434434
with:
435435
tool: cargo-llvm-cov
436436
- name: Rust Dependency Cache
@@ -468,7 +468,7 @@ jobs:
468468
if: ${{ !(github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'apache/datafusion') }}
469469
runs-on: ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
470470
steps:
471-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
471+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
472472
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
473473
with:
474474
submodules: true
@@ -515,7 +515,7 @@ jobs:
515515
container:
516516
image: amd64/rust
517517
steps:
518-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
518+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
519519
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
520520
with:
521521
submodules: true
@@ -547,7 +547,7 @@ jobs:
547547
container:
548548
image: amd64/rust
549549
steps:
550-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
550+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
551551
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
552552
with:
553553
submodules: true
@@ -570,7 +570,7 @@ jobs:
570570
container:
571571
image: amd64/rust
572572
steps:
573-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
573+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
574574
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
575575
- name: Setup Rust toolchain
576576
uses: ./.github/actions/setup-builder
@@ -598,7 +598,7 @@ jobs:
598598
clang-18 --version
599599
clang++-18 --version
600600
- name: Setup wasm-pack
601-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
601+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
602602
with:
603603
tool: wasm-pack
604604
- name: Run tests with headless mode
@@ -618,7 +618,7 @@ jobs:
618618
container:
619619
image: amd64/rust
620620
steps:
621-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
621+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
622622
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
623623
with:
624624
submodules: true
@@ -665,7 +665,7 @@ jobs:
665665
--health-timeout 5s
666666
--health-retries 5
667667
steps:
668-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
668+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
669669
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
670670
with:
671671
submodules: true
@@ -689,7 +689,7 @@ jobs:
689689
container:
690690
image: amd64/rust
691691
steps:
692-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
692+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
693693
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
694694
with:
695695
submodules: true
@@ -794,7 +794,7 @@ jobs:
794794
container:
795795
image: amd64/rust
796796
steps:
797-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
797+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
798798
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
799799
with:
800800
submodules: true
@@ -831,7 +831,7 @@ jobs:
831831
with:
832832
rust-version: stable
833833
- name: Install taplo
834-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
834+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
835835
with:
836836
tool: taplo-cli@0.9
837837
# if you encounter an error, try running 'taplo format' to fix the formatting automatically.
@@ -846,7 +846,7 @@ jobs:
846846
container:
847847
image: amd64/rust
848848
steps:
849-
- uses: runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
849+
- uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0
850850
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
851851
with:
852852
submodules: true
@@ -923,7 +923,7 @@ jobs:
923923
- name: Setup Rust toolchain
924924
uses: ./.github/actions/setup-builder
925925
- name: Install cargo-msrv
926-
uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
926+
uses: taiki-e/install-action@861a07ce7084f55488e375df125cdc99bba60eb7 # v2.87.23
927927
with:
928928
tool: cargo-msrv
929929

‎benchmarks/src/statistics.rs‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,10 @@ impl RunOpt {
184184
let logical_plan = state.optimize(&logical_plan)?;
185185
let physical_plan = state.create_physical_plan(&logical_plan).await?;
186186

187-
let statistics = capture_statistics(physical_plan.as_ref())?;
187+
let statistics = capture_statistics(
188+
physical_plan.as_ref(),
189+
&state.statistics_registry().cloned().unwrap_or_default(),
190+
)?;
188191
collect(Arc::clone(&physical_plan), state.task_ctx()).await?;
189192

190193
let mut report = Vec::with_capacity(statistics.len());
@@ -248,10 +251,12 @@ enum QError {
248251
ExactZero,
249252
}
250253

251-
fn capture_statistics(plan: &dyn ExecutionPlan) -> Result<Vec<CapturedStatistics>> {
252-
let statistics_context = StatisticsRegistry::default_with_builtin_providers();
254+
fn capture_statistics(
255+
plan: &dyn ExecutionPlan,
256+
statistics_context: &StatisticsRegistry,
257+
) -> Result<Vec<CapturedStatistics>> {
253258
let mut result = vec![];
254-
capture_statistics_inner(plan, &statistics_context, "0", &mut result)?;
259+
capture_statistics_inner(plan, statistics_context, "0", &mut result)?;
255260
Ok(result)
256261
}
257262

@@ -723,6 +728,11 @@ fn collect_parquet_files(path: &Path, files: &mut Vec<PathBuf>) -> Result<()> {
723728
#[cfg(test)]
724729
mod tests {
725730
use super::*;
731+
use datafusion::execution::session_state::SessionStateBuilder;
732+
use datafusion::physical_plan::Statistics;
733+
use datafusion::physical_plan::operator_statistics::{
734+
ClosureStatisticsProvider, StatisticsResult,
735+
};
726736
use tempfile::tempdir;
727737

728738
#[test]
@@ -786,6 +796,43 @@ mod tests {
786796
assert!(!reports.is_empty());
787797
}
788798

799+
#[tokio::test]
800+
async fn reports_estimates_from_session_statistics_registry() {
801+
let directory = tempdir().unwrap();
802+
let options = RunOpt {
803+
query: None,
804+
compare: None,
805+
path: directory.path().to_path_buf(),
806+
query_path: directory.path().to_path_buf(),
807+
};
808+
let mut registry = StatisticsRegistry::new();
809+
registry.register(Arc::new(ClosureStatisticsProvider::new(
810+
|plan, _child_stats| {
811+
Ok(StatisticsResult::Computed(
812+
Statistics {
813+
num_rows: Precision::Inexact(42),
814+
..Statistics::new_unknown(plan.schema().as_ref())
815+
}
816+
.into(),
817+
))
818+
},
819+
)));
820+
let ctx = SessionContext::from(
821+
SessionStateBuilder::new()
822+
.with_default_features()
823+
.with_statistics_registry(registry)
824+
.build(),
825+
);
826+
let statement =
827+
sql_statements("SELECT 1", &ctx.state().config_options().sql_parser)
828+
.unwrap()
829+
.pop_front()
830+
.unwrap();
831+
832+
let reports = options.report_statement(&ctx, statement).await.unwrap();
833+
assert_eq!(reports[0].estimated_rows, StatisticValue::Inexact(42));
834+
}
835+
789836
async fn report_query_files(
790837
options: &RunOpt,
791838
ctx: &SessionContext,

‎datafusion-examples/README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -220,9 +220,9 @@ cargo run --example dataframe -- dataframe
220220

221221
#### Category: Single Process
222222

223-
| Subcommand | File Path | Description |
224-
| ------------ | ------------------------------------------------------------------- | ----------------------------------------------------------------------- |
225-
| join_reorder | [`statistics/join_reorder.rs`](examples/statistics/join_reorder.rs) | Supply and refine column statistics via a provider to flip a join order |
223+
| Subcommand | File Path | Description |
224+
| ------------ | ------------------------------------------------------------------- | -------------------------------------------------------------------- |
225+
| join_reorder | [`statistics/join_reorder.rs`](examples/statistics/join_reorder.rs) | Supply catalog column statistics via a provider to flip a join order |
226226

227227
## UDF Examples
228228

0 commit comments

Comments
 (0)