1515# specific language governing permissions and limitations
1616# under the License.
1717
18+ # Each test command below is implemented with `cargo xtask` so developers can
19+ # run the same suite locally. Append `--explain` to print the underlying
20+ # `cargo test` invocation without running it. Job setup, the clean working tree
21+ # check, and `cargo clean` remain in this workflow.
22+ #
23+ # See `xtask/README.md` for details about DataFusion's `xtask` tooling.
24+ # Track progress at https://github.com/apache/datafusion/issues/24487.
25+
1826name : Datafusion extended tests
1927
2028concurrency :
2129 group : ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
2230 cancel-in-progress : true
2331
24- # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#running-your-pull_request-workflow-when-a-pull-request-merges
25- #
26- # These jobs are not run as part of PR checks as they are time-consuming
27- # and should not fail often.
28- #
29- # We still run them as they provide important coverage to ensure correctness
30- # in the (very rare) event of a hash failure or sqlite library query failure.
32+ # Run these required checks in the merge queue before merging into main.
33+ # On PR updates, skip the jobs so their required checks allow queue entry
34+ # without running the expensive suites. Do not add pull_request path filters:
35+ # a skipped workflow would leave required checks pending.
3136on :
37+ pull_request :
38+ merge_group :
3239 push :
3340 branches :
34- - main
35- # support extended test suite for release candidate branches,
36- # it is not expected to have many changes in these branches,
37- # so running extended tests is not a burden
41+ # Release branches retain post-push coverage. main is covered by the
42+ # merge queue, so do not repeat the tests after merging.
3843 - ' branch-*'
39- # Also run for changes to some critical areas that are most likely
40- # to trigger errors in extended tests
41- pull_request :
42- branches : [ '**' ]
43- paths :
44- - ' datafusion/physical*/**/*.rs'
45- - ' datafusion/expr*/**/*.rs'
46- - ' datafusion/optimizer/**/*.rs'
47- - ' datafusion/sql/**/*.rs'
48- - ' datafusion-testing'
4944 workflow_dispatch :
5045 inputs :
5146 pr_head_sha :
@@ -60,13 +55,15 @@ jobs:
6055 # Run extended tests (with feature 'extended_tests')
6156 linux-test-extended :
6257 name : cargo test 'extended_tests' (amd64)
58+ # Long-running job: run in the merge queue, on release pushes, or manually.
59+ if : github.event_name != 'pull_request'
6360 runs-on : ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
6461 # note: do not use amd/rust container to preserve disk space
6562 steps :
66- - uses : runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
63+ - uses : runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
6764 - uses : actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
6865 with :
69- ref : ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
66+ ref : ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
7067 submodules : true
7168 fetch-depth : 1
7269 - name : Free Disk Space (Ubuntu)
8683 run : |
8784 df -h
8885name : Run tests (excluding doctests)
89- env :
90- RUST_BACKTRACE : 1
91- # Run more random scenarios of the spill pool fuzzer than the
92- # default suite does (about a minute).
93- DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS : 1000
94- run : |
95- cargo test \
96- --profile ci \
97- --exclude datafusion-examples \
98- --exclude datafusion-benchmarks \
99- --exclude datafusion-cli \
100- --workspace \
101- --lib \
102- --tests \
103- --bins \
104- --features avro,json,backtrace,extended_tests,recursive_protection,parquet_encryption
86+ run : cargo xtask ci step test extended
10587 - parallel :
10688 - name : Verify Working Directory Clean
10789 run : git diff --exit-code
@@ -111,14 +93,16 @@ jobs:
11193 # Check answers are correct when hash values collide
11294 hash-collisions :
11395 name : cargo test hash collisions (amd64)
96+ # Long-running job: run in the merge queue, on release pushes, or manually.
97+ if : github.event_name != 'pull_request'
11498 runs-on : ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
11599 container :
116100 image : amd64/rust
117101 steps :
118- - uses : runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
102+ - uses : runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
119103 - uses : actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
120104 with :
121- ref : ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
105+ ref : ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
122106 submodules : true
123107 fetch-depth : 1
124108 - name : Setup Rust toolchain
@@ -127,27 +111,28 @@ jobs:
127111 rust-version : stable
128112 - name : Run tests
129113 run : |
114+ cargo xtask ci step test hash-collisions
130115 cd datafusion
131- cargo test --profile ci --exclude datafusion-examples --exclude datafusion-benchmarks --exclude datafusion-sqllogictest --exclude datafusion-cli --workspace --lib --tests --features=force_hash_collisions,avro
132116 cargo clean
133117
134118sqllogictest-sqlite :
135119 name : " Run sqllogictests with the sqlite test suite"
120+ # Long-running job: run in the merge queue, on release pushes, or manually.
121+ if : github.event_name != 'pull_request'
136122 runs-on : ${{ vars.USE_RUNS_ON == 'true' && format('runs-on={0},family=m8a+m7a+c8a,cpu=32,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
137123 container :
138124 image : amd64/rust
139125 steps :
140- - uses : runs-on/action@46910bf61b41721b0579f237e186afb35477007a # v2.3.0
126+ - uses : runs-on/action@efac073ea2507ec18797de3a81704201ade11d9d # v2.3.1
141127 - parallel :
142128 - uses : actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
143129 with :
144- ref : ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
130+ ref : ${{ github.event.inputs.pr_head_sha }} # otherwise use the event's commit, including the merge group
145131 submodules : true
146132 fetch-depth : 1
147133 # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive
148134 - name : Install protobuf compiler
149135 run : |
150136 apt-get update && apt-get install -y protobuf-compiler
151137name : Run sqllogictest
152- run : |
153- cargo test --features backtrace,parquet_encryption --profile ci-optimized --test sqllogictests -- --include-sqlite
138+ run : cargo xtask ci step test sqlite
0 commit comments