Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
#!/usr/bin/env bash

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# This script runs all the Rust lints locally the same way the
# DataFusion CI does
#
# Note: The installed checking tools (e.g., taplo) are not guaranteed to match
# the CI versions for simplicity, there might be some minor differences. Check
# `.github/workflows` for the CI versions.
#
#
#
# For each lint scripts:
#
# By default, they run in check mode:
# ./ci/scripts/rust_fmt.sh
#
# With `--write`, scripts perform best-effort auto fixes:
# ./ci/scripts/rust_fmt.sh --write
#
# The `--write` flag assumes a clean git repository (no uncommitted changes); to force
# auto fixes even if there are unstaged changes, use `--allow-dirty`:
# ./ci/scripts/rust_fmt.sh --write --allow-dirty
#
# New scripts can use `rust_fmt.sh` as a reference.

set -euo pipefail

usage() {
cat >&2 <<EOF
Usage: $0 [--write] [--allow-dirty]

Runs the local Rust lint suite similar to CI.
--write Run formatters, clippy and other non-functional checks in best-effort write/fix mode (requires a clean git worktree, no uncommitted changes; some checks are test-only and ignore this flag).
--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
EOF
exit 1
}

ensure_tool() {
local cmd="$1"
local install_cmd="$2"
if ! command -v "$cmd" &> /dev/null; then
echo "Installing $cmd using: $install_cmd"
eval "$install_cmd"
fi
}

MODE="check"
ALLOW_DIRTY=0

while [[ $# -gt 0 ]]; do
case "$1" in
--write)
MODE="write"
;;
--allow-dirty)
ALLOW_DIRTY=1
;;
-h|--help)
usage
;;
*)
usage
;;
esac
shift
done

SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"

ensure_tool "taplo" "cargo install taplo-cli --locked"
ensure_tool "hawkeye" "cargo install hawkeye --locked"
ensure_tool "typos" "cargo install typos-cli --locked"

run_step() {
local name="$1"
shift
echo "[${SCRIPT_NAME}] Running ${name}"
"$@"
}

declare -a WRITE_STEPS=(
"ci/scripts/rust_fmt.sh|true"
"ci/scripts/rust_clippy.sh|true"
"ci/scripts/rust_toml_fmt.sh|true"
"ci/scripts/license_header.sh|true"
"ci/scripts/typos_check.sh|true"
"ci/scripts/doc_prettier_check.sh|true"
)

declare -a READONLY_STEPS=(
"ci/scripts/check_no_cargo_install_in_workflows.sh|false"
"ci/scripts/rust_docs.sh|false"
)

for entry in "${WRITE_STEPS[@]}" "${READONLY_STEPS[@]}"; do
IFS='|' read -r script_path supports_write <<<"$entry"
script_name="$(basename "$script_path")"
args=()
if [[ "$supports_write" == "true" && "$MODE" == "write" ]]; then
args+=(--write)
[[ $ALLOW_DIRTY -eq 1 ]] && args+=(--allow-dirty)
fi
if [[ ${#args[@]} -gt 0 ]]; then
run_step "$script_name" "$script_path" "${args[@]}"
else
run_step "$script_name" "$script_path"
fi
done
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
A,b,c
1,2,3
1,10,5
2,5,6
2,1,4
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
source "${SCRIPT_DIR}/utils/git.sh"

MODE="check"
ALLOW_DIRTY=0

usage() {
cat >&2 <<EOF
Usage: $0 [--write] [--allow-dirty]

Runs \`cargo fmt --all -- --check\` by default to verify Rust formatting.
--write Run \`cargo fmt --all\` to auto-fix formatting (requires a clean git worktree, no uncommitted changes).
--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
EOF
exit 1
}

while [[ $# -gt 0 ]]; do
case "$1" in
--write)
MODE="write"
;;
--allow-dirty)
ALLOW_DIRTY=1
;;
-h|--help)
usage
;;
*)
usage
;;
esac
shift
done

if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
require_clean_work_tree "$SCRIPT_NAME" || exit 1
fi

if [[ "$MODE" == "write" ]]; then
echo "[${SCRIPT_NAME}] \`cargo fmt --all\`"
cargo fmt --all
else
echo "[${SCRIPT_NAME}] \`cargo fmt --all -- --check\`"
cargo fmt --all -- --check
fi
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"

source "${SCRIPT_DIR}/utils/git.sh"

MODE="check"
ALLOW_DIRTY=0

usage() {
cat >&2 <<EOF
Usage: $0 [--write] [--allow-dirty]

Runs \`taplo format --check\` by default to verify TOML formatting.
--write Run \`taplo format\` to auto-fix formatting (best-effort; requires a clean git worktree, no uncommitted changes).
--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted changes.
EOF
exit 1
}

while [[ $# -gt 0 ]]; do
case "$1" in
--write)
MODE="write"
;;
--allow-dirty)
ALLOW_DIRTY=1
;;
-h|--help)
usage
;;
*)
usage
;;
esac
shift
done

if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
require_clean_work_tree "$SCRIPT_NAME" || exit 1
fi

if [[ "$MODE" == "write" ]]; then
echo "[${SCRIPT_NAME}] \`taplo format\`"
taplo format
else
echo "[${SCRIPT_NAME}] \`taplo format --check\`"
taplo format --check
fi
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPT_NAME="$(basename "${BASH_SOURCE[0]}")"
CLIPPY_FEATURES="avro,integration-tests,extended_tests"
CLIPPY_ARGS=(--all-targets --workspace --features "$CLIPPY_FEATURES")
CLIPPY_LINT_ARGS=(-- -D warnings)

source "${SCRIPT_DIR}/utils/git.sh"

MODE="check"
ALLOW_DIRTY=0

usage() {
cat >&2 <<EOF
Usage: $SCRIPT_NAME [--write] [--allow-dirty]

Runs \`cargo clippy\` to lint.
--write Run \`cargo clippy --fix\` to apply fixes for clippy lints (requires a clean git worktree, no uncommitted changes).
--allow-dirty Allow \`--write\` to run even when the git worktree has uncommitted or staged changes.
EOF
exit 1
}

while [[ $# -gt 0 ]]; do
case "$1" in
--write)
MODE="write"
;;
--allow-dirty)
ALLOW_DIRTY=1
;;
-h|--help)
usage
;;
*)
usage
;;
esac
shift
done

if [[ "$MODE" == "write" && $ALLOW_DIRTY -eq 0 ]]; then
require_clean_work_tree "$SCRIPT_NAME" || exit 1
fi

CLIPPY_CMD=(cargo clippy)
if [[ "$MODE" == "write" ]]; then
CLIPPY_CMD+=(--fix)
if [[ $ALLOW_DIRTY -eq 1 ]]; then
CLIPPY_CMD+=(--allow-dirty --allow-staged)
fi
fi
CLIPPY_CMD+=("${CLIPPY_ARGS[@]}" "${CLIPPY_LINT_ARGS[@]}")

echo "[${SCRIPT_NAME}] \`${CLIPPY_CMD[*]}\`"
"${CLIPPY_CMD[@]}"
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
-- Test scanning out dictionary columns
select dict_10_optional from t;
select dict_100_optional from t;
select dict_1000_optional from t;

-- Test filtering dictionary columns
select count(*) from t where dict_10_required = 'prefix#0';
select count(*) from t where dict_100_required = 'prefix#0';
select count(*) from t where dict_1000_required = 'prefix#0';

-- Test select integers
select i64_optional from t where dict_10_required = 'prefix#2' and dict_1000_required = 'prefix#10';
select i64_required from t where dict_10_required = 'prefix#2' and dict_1000_required = 'prefix#10';

-- Test select integers
select string_optional from t where dict_10_required = 'prefix#1' and dict_1000_required = 'prefix#1';
select string_required from t where dict_10_required = 'prefix#1' and dict_1000_required = 'prefix#1';

-- Test select distinct
select distinct dict_10_required from t where dict_1000_optional is not NULL and i64_optional > 0;
select distinct dict_10_required from t where dict_1000_optional is not NULL and i64_optional > 0;
select distinct dict_10_required from t where dict_1000_optional is not NULL and i64_required > 0;
select distinct dict_10_required from t where dict_1000_optional is not NULL and i64_required > 0;

-- Test basic aggregations
select dict_10_optional, count(*) from t group by dict_10_optional;
select dict_10_optional, dict_100_optional, count(*) from t group by dict_10_optional, dict_100_optional;

-- Test float aggregations
select dict_10_optional, dict_100_optional, MIN(f64_required), MAX(f64_required), AVG(f64_required) from t group by dict_10_optional, dict_100_optional;
select dict_10_optional, dict_100_optional, MIN(f64_optional), MAX(f64_optional), AVG(f64_optional) from t group by dict_10_optional, dict_100_optional;
select dict_10_required, dict_100_required, MIN(f64_optional), MAX(f64_optional), AVG(f64_optional) from t group by dict_10_required, dict_100_required;
Loading