From 6483fb845110616524f725db4a8005d8b802b025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 14:36:46 -0400 Subject: [PATCH 01/20] Vendor the Turso v0.8.1 sqltest corpus and record its new differences Refresh conformance/sqlite-sqltests from the pinned turso-src v0.8.1 (8549c1659): 33 new files and 41 refreshed files since v0.8.0-pre.7. The 113 newly exposed case-level differences are recorded in the expected- failures ledger pending closure; the inventory's live count follows. Co-Authored-By: Claude Opus 5.5 --- conformance/sqlite-sqltests/affinity.sqltest | 74 ++ .../json-group-object-null-label.sqltest | 27 + .../agg-functions/memory.sqltest | 51 ++ .../agg-functions/sum-large-float.sqltest | 65 ++ .../alter-add-column-quoted-pk-column.sqltest | 41 + .../alter-column-keeps-pk-desc.sqltest | 34 + ...rename-table-to-digit-leading-name.sqltest | 62 ++ ...er-table-fk-without-parent-columns.sqltest | 40 + ...ter-table-keeps-on-conflict-clause.sqltest | 132 +++ .../sqlite-sqltests/alter_table.sqltest | 156 +++- .../btree-tiny-cell-divider.sqltest | 58 ++ conformance/sqlite-sqltests/collate.sqltest | 73 ++ conformance/sqlite-sqltests/compare.sqltest | 10 + conformance/sqlite-sqltests/concat.sqltest | 41 + .../create_table_as_select.sqltest | 18 + .../sqlite-sqltests/foreign_keys.sqltest | 67 ++ .../sqlite-sqltests/function_rowid.sqltest | 26 + conformance/sqlite-sqltests/gencol.sqltest | 727 ++++++++++++++- .../sqlite-sqltests/generate_series.sqltest | 11 + .../sqlite-sqltests/groupby/memory.sqltest | 103 +++ .../hash-join-unmatched-row-filters.sqltest | 87 ++ .../index-seek-null-key.sqltest | 76 ++ .../index_seek_comparison_affinity.sqltest | 15 + .../int64-overflow-seek.sqltest | 32 + .../integrity_check/memory.sqltest | 105 +++ ...arity_strict_gencol_type_violation.sqltest | 20 + .../parity_strict_not_null_violation.sqltest | 20 + .../parity_strict_real_integer_serial.sqltest | 21 + .../parity_strict_type_violation.sqltest | 19 + conformance/sqlite-sqltests/join/hash.sqltest | 120 +++ .../join/hash_join_order_by.sqltest | 177 ++++ .../sqlite-sqltests/join/memory.sqltest | 580 ++++++++++++ .../sqlite-sqltests/json/default.sqltest | 27 + .../json/json-nested-array-path.sqltest | 50 ++ .../json/json_tree_columns.sqltest | 72 ++ ...join-in-subquery-on-indexed-column.sqltest | 78 ++ .../materialized_subquery.sqltest | 20 + .../sqlite-sqltests/math/memory.sqltest | 32 + .../multi_index_intersection.sqltest | 36 + .../multi_index_or_compound.sqltest | 261 +++++- ...cc-analyze-refreshes-planner-stats.sqltest | 55 ++ .../sqlite-sqltests/offset/memory.sqltest | 47 +- ...erby_plan__orderby_a_desc_uses_sorter.snap | 2 +- .../orderby_plan__orderby_a_uses_sorter.snap | 2 +- ...by_plan__orderby_rowid_desc_no_sorter.snap | 2 +- ...orderby_plan__orderby_rowid_no_sorter.snap | 2 +- conformance/sqlite-sqltests/page1.sqltest | 16 + .../sqlite-sqltests/partial_idx.sqltest | 161 +++- .../pragma-name-case-insensitive.sqltest | 56 ++ .../sqlite-sqltests/pragma/memory.sqltest | 32 +- ..._rejects_metadata_and_drop_trigger.sqltest | 52 ++ .../sqlite-sqltests/row-value-in.sqltest | 127 +++ .../scalar-functions-datetime.sqltest | 63 ++ .../sqlite-sqltests/scalar-functions.sqltest | 75 ++ .../simple-count-optimization.sqltest | 33 + .../aggregation/aggregation.sqltest | 11 + .../aggregation__aggregation-with-join.snap | 93 +- ...tion__column-free-scan-covering-index.snap | 2 +- .../snapshots/aggregation__count-column.snap | 2 +- .../aggregation__count-distinct.snap | 2 +- .../aggregation__count-notnull-column.snap | 26 + .../aggregation__count-star-plus-expr.snap | 2 +- ...aggregation__distinct-count-with-join.snap | 93 +- .../aggregation__group-by-with-having.snap | 2 +- .../aggregation__min-max-per-group.snap | 2 +- .../aggregation__min-max-with-index.snap | 2 +- .../aggregation__multi-column-group-by.snap | 78 +- .../aggregation__multiple-aggregates.snap | 114 ++- .../aggregation__nested-aggregation.snap | 2 +- ...aggregation__partial-index-count-star.snap | 32 +- ...gation__simple-group-by-single-column.snap | 2 +- .../aggregation__whole-table-aggregates.snap | 2 +- .../analyze__aggregate-after-analyze.snap | 91 +- .../analyze__join-after-analyze.snap | 43 +- ...analyze__join-aggregate-after-analyze.snap | 107 ++- .../snapshots/analyze__max-after-analyze.snap | 2 +- .../snapshots/analyze__min-after-analyze.snap | 2 +- ...analyze__order-by-index-after-analyze.snap | 23 +- ...analyze__query-after-analyze-category.snap | 2 +- .../analyze__query-after-analyze-range.snap | 2 +- ...e__query-composite-full-after-analyze.snap | 21 +- ...query-composite-partial-after-analyze.snap | 2 +- ...nalyze__query-for-nulls-after-analyze.snap | 30 +- ...ery-identical-composite-after-analyze.snap | 2 +- ..._query-identical-values-after-analyze.snap | 29 +- ...three-column-first-only-after-analyze.snap | 30 +- ...ry-three-column-partial-after-analyze.snap | 35 +- ...alyze__query-with-nulls-after-analyze.snap | 29 +- ...onstant-parameter-hoisted-out-of-scan.snap | 2 +- .../snapshot_tests/indexes/indexes.sqltest | 8 + .../indexes__covering-index-all-columns.snap | 2 +- .../indexes__covering-index-subset.snap | 2 +- ...exes__covering-index-with-aggregation.snap | 2 +- .../indexes__in-list-composite-prefix.snap | 2 +- ...related-max-materialized-cte-bytecode.snap | 81 +- .../indexes__index-for-max-bytecode.snap | 2 +- ...dex-for-max-materialized-cte-bytecode.snap | 53 +- ...ax-materialized-cte-no-where-bytecode.snap | 47 +- .../indexes__index-for-min-bytecode.snap | 2 +- .../indexes__indexed-by-delete-category.snap | 2 +- .../indexes__indexed-by-delete-sku.snap | 2 +- .../indexes__indexed-by-join-both.snap | 2 +- .../indexes__indexed-by-join-one-side.snap | 2 +- .../indexes__indexed-by-update-price.snap | 2 +- .../indexes__indexed-by-update-sku.snap | 2 +- ...es__join-with-covering-index-bytecode.snap | 82 ++ .../snapshots/indexes__join-with-index.snap | 5 +- ...es__max-with-equality-prefix-bytecode.snap | 2 +- .../snapshots/indexes__or-same-column.snap | 2 +- .../indexes__or-with-and-combinations.snap | 2 +- .../indexes__rowid-range-no-alias.snap | 21 +- .../indexes__subquery-with-index.snap | 2 +- ...related-max-materialized-cte-bytecode.snap | 105 +-- ...ordered-materialized-cte-max-bytecode.snap | 45 +- ...idx-predicate-for-covering-index-scan.snap | 13 + ...-where-clause-for-covering-index-scan.snap | 13 + ...s__bigass-three-table-join-rowid-seek.snap | 4 +- .../snapshots/joins__cross-join-explicit.snap | 4 +- .../snapshots/joins__cross-join-limited.snap | 4 +- .../joins__cross-join-with-filter.snap | 58 +- .../joins__hash-join-inner-basic.snap | 51 +- .../joins__hash-join-three-table.snap | 167 ++-- .../joins__hash-join-with-order-by.snap | 17 +- ...joins__index-assisted-join-range-scan.snap | 2 +- .../snapshots/joins__join-with-subquery.snap | 4 +- .../joins__left-outer-join-basic.snap | 2 +- .../joins__left-outer-join-chained.snap | 4 +- .../joins__left-outer-join-find-nulls.snap | 49 +- ...ins__left-outer-join-with-aggregation.snap | 103 +-- .../joins__self-join-category-hierarchy.snap | 2 +- .../joins__self-join-employee-manager.snap | 2 +- .../modifications__delete-with-subquery.snap | 4 +- .../modifications__delete-with-where.snap | 2 +- .../modifications__insert-select.snap | 81 +- ...fications__update-where-self-subquery.snap | 41 +- ...ions__update-where-unrelated-subquery.snap | 37 +- .../returning__delete-returning-expr.snap | 60 +- .../returning__delete-returning-star.snap | 59 +- .../returning__insert-returning-expr.snap | 18 +- .../returning__insert-returning-star.snap | 18 +- .../returning__update-returning-expr.snap | 22 +- .../returning__update-returning-star.snap | 20 +- .../returning__upsert-returning.snap | 61 +- ...p-temp-trigger-opens-temp-transaction.snap | 37 +- .../setops__compound-mixed-union.snap | 54 +- .../setops__compound-union-all-three-way.snap | 43 +- .../setops__compound-union-except.snap | 6 +- .../setops__compound-union-intersect.snap | 6 +- .../snapshots/setops__except-basic.snap | 4 +- .../setops__except-multiple-columns.snap | 41 +- .../snapshots/setops__intersect-basic.snap | 4 +- .../setops__intersect-multiple-columns.snap | 49 +- .../setops__subquery-with-union.snap | 65 +- .../snapshots/setops__union-all-basic.snap | 32 +- .../snapshots/setops__union-distinct.snap | 43 +- .../setops__union-explicit-distinct.snap | 4 +- .../setops__union-order-by-literals.snap | 35 +- .../setops__union-subquery-aggregation.snap | 143 ++- .../snapshots/setops__union-with-limit.snap | 47 +- .../sorting__large-offset-small-limit.snap | 2 +- .../snapshots/sorting__limit-with-offset.snap | 25 +- .../sorting__limit-without-order.snap | 23 +- .../sorting__order-by-expression.snap | 40 +- .../sorting__order-by-limit-one.snap | 2 +- .../sorting__order-by-mixed-direction.snap | 22 +- .../sorting__order-by-single-column.snap | 2 +- ...ort-elim-left-join-outer-rowid-suffix.snap | 2 +- ...subqueries__correlated-subquery-count.snap | 101 ++- ...ubqueries__correlated-subquery-salary.snap | 111 ++- ...subqueries__correlated-subquery-where.snap | 107 ++- .../subqueries__cte-aggregation.snap | 6 +- .../subqueries__cte-in-subquery-and-main.snap | 255 +++--- .../subqueries__cte-multiple-chained.snap | 8 +- .../subqueries__cte-multiple-independent.snap | 155 ++-- ...ueries__cte-referenced-multiple-times.snap | 132 +-- .../subqueries__cte-referencing-previous.snap | 167 ++-- ...queries__cte-with-correlated-subquery.snap | 139 ++- ...ies__delete-returning-target-selfread.snap | 80 +- .../subqueries__derived-table-join.snap | 128 +-- .../subqueries__derived-table-nested.snap | 149 ++-- .../subqueries__exists-correlated.snap | 2 +- ...ubqueries__exists-multiple-conditions.snap | 2 +- ...s__grouped-correlated-subquery-having.snap | 160 ++-- ..._grouped-correlated-subquery-order-by.snap | 177 ++-- ...queries__grouped-subquery-builds-once.snap | 125 +++ .../subqueries__in-subquery-aggregation.snap | 2 +- .../subqueries__not-exists-never-ordered.snap | 2 +- .../subqueries__not-exists-no-orders.snap | 2 +- ...bqueries__scalar-subquery-cross-table.snap | 92 +- ...subqueries__scalar-subquery-max-price.snap | 4 +- .../subqueries__subquery-within-cte.snap | 2 +- ...ies__update-returning-target-selfread.snap | 25 +- .../snapshots/tpch__q1-pricing-summary.snap | 169 ++-- .../snapshots/tpch__q11-important-stock.snap | 359 ++++++-- .../snapshots/tpch__q12-shipping-modes.snap | 180 ++-- .../snapshots/tpch__q14-promotion-effect.snap | 2 +- .../snapshots/tpch__q16-parts-supplier.snap | 2 +- .../tpch__q17-small-quantity-order.snap | 170 ++-- .../tpch__q19-discounted-revenue.snap | 123 +-- ...n-bound-stays-outside-autoindex-build.snap | 113 +++ .../tpch__q20-potential-part-promotion.snap | 211 ++++- .../tpch__q21-suppliers-kept-waiting.snap | 12 +- .../tpch__q22-global-sales-opportunity.snap | 4 +- .../snapshots/tpch__q4-order-priority.snap | 103 ++- .../tpch__q5-local-supplier-volume.snap | 4 +- .../tpch__q6-forecasting-revenue.snap | 69 +- .../snapshots/tpch__q7-volume-shipping.snap | 12 +- .../tpch__q9-product-type-profit.snap | 11 +- .../snapshot_tests/tpch/tpch.sqltest | 23 +- .../snapshots/windows__aggregate-windows.snap | 256 +++--- .../snapshots/windows__multiple-windows.snap | 591 ++++++------ .../snapshots/windows__percent-of-total.snap | 238 +++-- .../snapshots/windows__row-number-basic.snap | 123 ++- .../snapshots/windows__running-total.snap | 394 ++++---- .../snapshots/windows__window-order-by.snap | 155 ++-- .../windows__window-over-grouped.snap | 208 +++-- .../windows__window-with-subquery.snap | 255 +++--- ..._small-cte-materialized-big-cte-outer.snap | 4 +- ...ee-ctes-big-outer-others-materialized.snap | 4 +- ...__fk-update-column-no-child-points-at.snap | 34 + ...-update-parent-key-with-cascade-child.snap | 88 ++ ...update-parent-key-with-mixed-children.snap | 110 +++ ...pdate-parent-key-with-no-action-child.snap | 70 ++ ...__fk-upsert-column-no-child-points-at.snap | 59 ++ ...-upsert-parent-key-with-cascade-child.snap | 101 +++ ...scan-covering-index-scan-tags-next-p5.snap | 25 + ...lscan-reverse-table-scan-tags-prev-p5.snap | 25 + ...eps__fullscan-table-scan-tags-next-p5.snap | 25 + ...exed-range-search-not-tagged-fullscan.snap | 29 + ...ol_drop_virtual_column_no_row_rewrite.snap | 49 +- ...e_reuses_generated_columns_for_checks.snap | 35 + ...xt_pk__cte-neighbor-traversal-text-pk.snap | 2 +- ...index-preferred-over-intersection-eqp.snap | 2 +- ...i_index_intersection__multi_index_and.snap | 44 + ...dex_intersection__multi_index_and_eqp.snap | 13 + ...__compound-or-correlated-subquery-eqp.snap | 2 +- ...ing-where-can-prove-after-rewrite-eqp.snap | 2 +- ...join-on-predicate-can-prove-index-eqp.snap | 2 +- ...here-predicate-cannot-prove-index-eqp.snap | 2 +- .../sql-error-message-prefix.sqltest | 15 + .../subquery-from-name-vs-outer-alias.sqltest | 116 +++ .../sqlite-sqltests/subquery/memory.sqltest | 2 +- .../turso/alter_column.sqltest | 170 ++++ .../turso/explain-query-plan-json.sqltest | 22 +- conformance/sqlite-sqltests/turso/fts.sqltest | 88 ++ .../mvcc_checkpoint_delete_order.sqltest | 51 ++ .../turso/mvcc_sequence.sqltest | 182 ++++ .../sqlite-sqltests/turso/sequence.sqltest | 32 + .../turso/upsert-target-alias.sqltest | 43 + .../sqlite-sqltests/turso/vacuum.sqltest | 33 + .../turso/without_rowid.sqltest | 11 + .../sqlite-sqltests/type_affinity.sqltest | 54 ++ .../sqlite-sqltests/unnest-correlated.sqltest | 844 +++++++++++++++++- conformance/sqlite-sqltests/upsert.sqltest | 43 + conformance/sqlite-sqltests/vacuum.sqltest | 69 ++ docs/turso-gap-inventory.json | 4 +- .../managed-sqltest-expected-failures.txt | 117 +++ 257 files changed, 11800 insertions(+), 4183 deletions(-) create mode 100644 conformance/sqlite-sqltests/agg-functions/json-group-object-null-label.sqltest create mode 100644 conformance/sqlite-sqltests/alter-add-column-quoted-pk-column.sqltest create mode 100644 conformance/sqlite-sqltests/alter-column-keeps-pk-desc.sqltest create mode 100644 conformance/sqlite-sqltests/alter-rename-table-to-digit-leading-name.sqltest create mode 100644 conformance/sqlite-sqltests/alter-table-fk-without-parent-columns.sqltest create mode 100644 conformance/sqlite-sqltests/alter-table-keeps-on-conflict-clause.sqltest create mode 100644 conformance/sqlite-sqltests/btree-tiny-cell-divider.sqltest create mode 100644 conformance/sqlite-sqltests/function_rowid.sqltest create mode 100644 conformance/sqlite-sqltests/hash-join-unmatched-row-filters.sqltest create mode 100644 conformance/sqlite-sqltests/index-seek-null-key.sqltest create mode 100644 conformance/sqlite-sqltests/integrity_check/parity_strict_gencol_type_violation.sqltest create mode 100644 conformance/sqlite-sqltests/integrity_check/parity_strict_not_null_violation.sqltest create mode 100644 conformance/sqlite-sqltests/integrity_check/parity_strict_real_integer_serial.sqltest create mode 100644 conformance/sqlite-sqltests/integrity_check/parity_strict_type_violation.sqltest create mode 100644 conformance/sqlite-sqltests/join/hash_join_order_by.sqltest create mode 100644 conformance/sqlite-sqltests/json/json-nested-array-path.sqltest create mode 100644 conformance/sqlite-sqltests/json/json_tree_columns.sqltest create mode 100644 conformance/sqlite-sqltests/left-join-in-subquery-on-indexed-column.sqltest create mode 100644 conformance/sqlite-sqltests/materialized_subquery.sqltest create mode 100644 conformance/sqlite-sqltests/mvcc-analyze-refreshes-planner-stats.sqltest create mode 100644 conformance/sqlite-sqltests/page1.sqltest create mode 100644 conformance/sqlite-sqltests/pragma-name-case-insensitive.sqltest create mode 100644 conformance/sqlite-sqltests/pragma_query_only_rejects_metadata_and_drop_trigger.sqltest create mode 100644 conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-notnull-column.snap create mode 100644 conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__join-with-covering-index-bytecode.snap create mode 100644 conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__use-partial-idx-predicate-for-covering-index-scan.snap create mode 100644 conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__use-partial-idx-predicate-where-predicate-is-disjunction-term-from-index-where-clause-for-covering-index-scan.snap create mode 100644 conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-subquery-builds-once.snap create mode 100644 conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q20-function-bound-stays-outside-autoindex-build.snap create mode 100644 conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-column-no-child-points-at.snap create mode 100644 conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-cascade-child.snap create mode 100644 conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-mixed-children.snap create mode 100644 conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-no-action-child.snap create mode 100644 conformance/sqlite-sqltests/snapshots/foreign_keys__fk-upsert-column-no-child-points-at.snap create mode 100644 conformance/sqlite-sqltests/snapshots/foreign_keys__fk-upsert-parent-key-with-cascade-child.snap create mode 100644 conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-covering-index-scan-tags-next-p5.snap create mode 100644 conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-reverse-table-scan-tags-prev-p5.snap create mode 100644 conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-table-scan-tags-next-p5.snap create mode 100644 conformance/sqlite-sqltests/snapshots/fullscan_steps__indexed-range-search-not-tagged-fullscan.snap create mode 100644 conformance/sqlite-sqltests/snapshots/gencol__strict_update_reuses_generated_columns_for_checks.snap create mode 100644 conformance/sqlite-sqltests/snapshots/multi_index_intersection__multi_index_and.snap create mode 100644 conformance/sqlite-sqltests/snapshots/multi_index_intersection__multi_index_and_eqp.snap create mode 100644 conformance/sqlite-sqltests/subquery-from-name-vs-outer-alias.sqltest create mode 100644 conformance/sqlite-sqltests/turso/mvcc_checkpoint_delete_order.sqltest create mode 100644 conformance/sqlite-sqltests/turso/upsert-target-alias.sqltest create mode 100644 conformance/sqlite-sqltests/turso/vacuum.sqltest create mode 100644 conformance/sqlite-sqltests/type_affinity.sqltest create mode 100644 conformance/sqlite-sqltests/vacuum.sqltest diff --git a/conformance/sqlite-sqltests/affinity.sqltest b/conformance/sqlite-sqltests/affinity.sqltest index 2824549c..e5205c60 100644 --- a/conformance/sqlite-sqltests/affinity.sqltest +++ b/conformance/sqlite-sqltests/affinity.sqltest @@ -467,6 +467,41 @@ expect { text|3 } +# ============================================ +# Vertical tab (0x0B) is whitespace to SQLite's ctype table (0x09-0x0D) +# even though it is not "ASCII whitespace" per Rust's is_ascii_whitespace(). +# https://github.com/tursodatabase/turso/issues/8454 +# ============================================ +@cross-check-integrity +test affinity-vertical-tab-leading { + CREATE TABLE t1(i INTEGER); + INSERT INTO t1 VALUES (CHAR(11) || '12'); + SELECT TYPEOF(i), i = 12 FROM t1; +} +expect { + integer|1 +} + +@cross-check-integrity +test affinity-vertical-tab-trailing { + CREATE TABLE t1(i INTEGER); + INSERT INTO t1 VALUES ('12' || CHAR(11)); + SELECT TYPEOF(i), i = 12 FROM t1; +} +expect { + integer|1 +} + +@cross-check-integrity +test affinity-vertical-tab-integer-primary-key { + CREATE TABLE t1(i INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES (CHAR(11) || '12'); + SELECT COUNT(*), TYPEOF(i), i FROM t1; +} +expect { + 1|integer|12 +} + # ============================================ # REAL affinity with leading + sign @@ -540,6 +575,33 @@ expect { 1a|text } +@cross-check-integrity +test affinity-any-strict-index-comparisons { + CREATE TABLE t(id INTEGER PRIMARY KEY, c ANY) STRICT; + INSERT INTO t VALUES (1, '000123'), (2, 123), (3, '123'); + SELECT id FROM t WHERE c = 123 ORDER BY id; + SELECT id FROM t WHERE c = '123' ORDER BY id; + CREATE UNIQUE INDEX t_c ON t(c); + SELECT id FROM t INDEXED BY t_c WHERE c = 123 ORDER BY id; + SELECT id FROM t INDEXED BY t_c WHERE c = '123' ORDER BY id; + SELECT id FROM t INDEXED BY t_c WHERE c = '000123' ORDER BY id; + UPDATE t SET c = '1.5' WHERE id = 1; + INSERT INTO t VALUES (2, '000456') ON CONFLICT(id) DO UPDATE SET c = excluded.c; + SELECT id, c, typeof(c) FROM t ORDER BY id; + PRAGMA integrity_check; +} +expect { + 2 + 3 + 2 + 3 + 1 + 1|1.5|text + 2|000456|text + 3|123|text + ok +} + @cross-check-integrity test cast-text-to-numeric-integral-float { SELECT CAST('3.0' AS NUMERIC), typeof(CAST('3.0' AS NUMERIC)); @@ -572,6 +634,18 @@ expect @js { 2251799813685248|real } +@cross-check-integrity +test cast-text-to-numeric-vertical-tab { + SELECT CAST(CHAR(11) || '12' AS NUMERIC); + SELECT CAST('12' || CHAR(11) AS NUMERIC); + SELECT CAST(CHAR(9) || '12' AS NUMERIC); +} +expect { + 12 + 12 + 12 +} + # Regression tests for https://github.com/tursodatabase/turso/issues/4927 # A compound (UNION/UNION ALL) subquery column's affinity must be combined # across all arms, not taken from the leftmost arm alone. Scanning left to diff --git a/conformance/sqlite-sqltests/agg-functions/json-group-object-null-label.sqltest b/conformance/sqlite-sqltests/agg-functions/json-group-object-null-label.sqltest new file mode 100644 index 00000000..84a27d5f --- /dev/null +++ b/conformance/sqlite-sqltests/agg-functions/json-group-object-null-label.sqltest @@ -0,0 +1,27 @@ +@database :memory: + +test json_group_object_skips_null_label { + CREATE TABLE t(k TEXT, n INT); + INSERT INTO t VALUES('a', 1), (NULL, 2), ('b', 3); + SELECT json_group_object(k, n) FROM t; +} +expect { + {"a":1,"b":3} +} + +test jsonb_group_object_skips_null_label { + CREATE TABLE t(k TEXT, n INT); + INSERT INTO t VALUES('a', 1), (NULL, 2), ('b', 3); + SELECT json(jsonb_group_object(k, n)) FROM t; +} +expect { + {"a":1,"b":3} +} + +test json_group_object_null_label_only { + SELECT json_group_object(NULL, 1), json(jsonb_group_object(NULL, 1)), + json_group_object('a', NULL), json_group_array(NULL); +} +expect { + {}|{}|{"a":null}|[null] +} diff --git a/conformance/sqlite-sqltests/agg-functions/memory.sqltest b/conformance/sqlite-sqltests/agg-functions/memory.sqltest index 33749157..b8ed87a4 100644 --- a/conformance/sqlite-sqltests/agg-functions/memory.sqltest +++ b/conformance/sqlite-sqltests/agg-functions/memory.sqltest @@ -1,5 +1,56 @@ @database :memory: +test finalize-mixed-aggregates-across-groups { + CREATE TABLE t(g INTEGER, v INTEGER, s TEXT); + INSERT INTO t VALUES (1, 2, 'a'), (1, 4, 'a'), + (2, NULL, NULL), (2, NULL, NULL), + (3, 7, 'c'), (4, 9, 'd'); + SELECT g, count(*), count(v), count(DISTINCT v), + coalesce(sum(v), -1), total(v), coalesce(avg(v), -1), + coalesce(min(s), '-'), coalesce(max(s), '-'), + coalesce(group_concat(s), '-'), count(*) FILTER (WHERE 0) + FROM t GROUP BY g ORDER BY g; +} +expect { + 1|2|2|2|6|6.0|3.0|a|a|a,a|0 + 2|2|0|0|-1|0.0|-1|-|-|-|0 + 3|1|1|1|7|7.0|7.0|c|c|c|0 + 4|1|1|1|9|9.0|9.0|d|d|d|0 +} +expect @js { + 1|2|2|2|6|6|3|a|a|a,a|0 + 2|2|0|0|-1|0|-1|-|-|-|0 + 3|1|1|1|7|7|7|c|c|c|0 + 4|1|1|1|9|9|9|d|d|d|0 +} + +test aggregate-values-preserve-state-across-window-rows { + CREATE TABLE t(id INTEGER, g INTEGER, v INTEGER); + INSERT INTO t VALUES (1, 1, 2), (2, 1, NULL), (3, 1, 4), + (4, 2, NULL), (5, 2, 7); + SELECT id, count(*) OVER w, count(v) OVER w, + coalesce(sum(v) OVER w, -1), total(v) OVER w, + count(*) FILTER (WHERE 0) OVER w + FROM t + WINDOW w AS (PARTITION BY g ORDER BY id + ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) + ORDER BY id; +} +expect { + 1|1|1|2|2.0|0 + 2|2|1|2|2.0|0 + 3|2|1|4|4.0|0 + 4|1|0|-1|0.0|0 + 5|2|1|7|7.0|0 +} +expect @js { + 1|1|1|2|2|0 + 2|2|1|2|2|0 + 3|2|1|4|4|0 + 4|1|0|-1|0|0 + 5|2|1|7|7|0 +} + @cross-check-integrity test min-null-regression-test { CREATE TABLE t (a); diff --git a/conformance/sqlite-sqltests/agg-functions/sum-large-float.sqltest b/conformance/sqlite-sqltests/agg-functions/sum-large-float.sqltest index e6318afa..b8b97da9 100644 --- a/conformance/sqlite-sqltests/agg-functions/sum-large-float.sqltest +++ b/conformance/sqlite-sqltests/agg-functions/sum-large-float.sqltest @@ -20,3 +20,68 @@ test avg-large-float-infinity { expect { Inf } + +test float-sum-and-avg-compensate-rounding-error { + WITH t(x) AS (VALUES(0.1),(0.1),(0.1),(0.1),(0.1),(0.1),(0.1),(0.1),(0.1),(0.1)) + SELECT sum(x) = 1.0, total(x) = 1.0, avg(x) = 0.1 FROM t; +} +expect { + 1|1|1 +} + +test float-sum-after-integer-overflow { + WITH t(x) AS (VALUES(9223372036854775807),(1),(0.5),(0.25)) + SELECT typeof(sum(x)), sum(x), total(x) FROM t; +} +expect { + real|9.22337203685478e+18|9.22337203685478e+18 +} +expect @js { + real|9223372036854776000|9223372036854776000 +} + +test float-avg-then-large-integer { + WITH t(x) AS (VALUES(0.5),(9007199254740993),(1.5)) + SELECT avg(x), sum(x) FROM t; +} +expect { + 3.00239975158033e+15|9.007199254741e+15 +} +expect @js { + 3002399751580332|9007199254740996 +} + +test float-sum-reaches-infinity-then-finite-value { + WITH t(x) AS (VALUES(1e308),(1e308),(-1e308),(0.5)) + SELECT sum(x), avg(x) FROM t; +} +expect { + Inf|Inf +} + +test count-mixed-value-types { + WITH t(x) AS (VALUES(1),('a'),(NULL),(2.5),(x'00'),(3)) + SELECT count(x), count(*) FROM t; +} +expect { + 5|6 +} + +test float-sum-then-integers { + WITH t(x) AS (VALUES(0.5),(1),(-3),(4503599627370495),(4503599627370496),(9007199254740993),(-9223372036854775808),(2)) + SELECT typeof(sum(x)), sum(x), total(x), avg(x) FROM t; +} +expect { + real|-9.20535763834529e+18|-9.20535763834529e+18|-1.15066970479316e+18 +} +expect @js { + real|-9205357638345294000|-9205357638345294000|-1150669704793161700 +} + +test total-over-integers { + WITH t(x) AS (VALUES(9007199254740991),(1),(1),(-2),(4503599627370495)) + SELECT printf('%.17g', total(x)), printf('%.17g', avg(x)) FROM t; +} +expect { + 13510798882111490|2702159776422297 +} diff --git a/conformance/sqlite-sqltests/alter-add-column-quoted-pk-column.sqltest b/conformance/sqlite-sqltests/alter-add-column-quoted-pk-column.sqltest new file mode 100644 index 00000000..1b8f1356 --- /dev/null +++ b/conformance/sqlite-sqltests/alter-add-column-quoted-pk-column.sqltest @@ -0,0 +1,41 @@ +@database :memory: + +# Regression tests for https://github.com/tursodatabase/turso/issues/9068 +# +# ADD COLUMN rewrites the stored CREATE TABLE text. A column named with a +# keyword (`"order"`) that appears inside a table-level PRIMARY KEY must keep +# its quotes in the rewritten text. Otherwise the schema text becomes +# `PRIMARY KEY (id, order)`, which no longer parses, so every later ALTER +# fails and the database file cannot be opened again. + +setup keyword-pk-column { + CREATE TABLE g(id INTEGER, "order" NUMERIC, val INTEGER, PRIMARY KEY(id, "order")); + INSERT INTO g VALUES (1, 2, 3); + ALTER TABLE g ADD COLUMN added REAL; +} + +@setup keyword-pk-column +test add-column-keeps-quotes-in-table-level-primary-key { + SELECT instr(sql, '"order")') > 0 FROM sqlite_schema WHERE name = 'g'; +} +expect { + 1 +} + +@setup keyword-pk-column +test rename-column-after-add-column-with-quoted-pk-column { + ALTER TABLE g RENAME COLUMN val TO v2; + SELECT id, "order", v2, added FROM g; +} +expect { + 1|2|3| +} + +@setup keyword-pk-column +test add-second-column-after-add-column-with-quoted-pk-column { + ALTER TABLE g ADD COLUMN more TEXT; + SELECT id, "order", val, added, more FROM g; +} +expect { + 1|2|3|| +} diff --git a/conformance/sqlite-sqltests/alter-column-keeps-pk-desc.sqltest b/conformance/sqlite-sqltests/alter-column-keeps-pk-desc.sqltest new file mode 100644 index 00000000..a9fd2a1f --- /dev/null +++ b/conformance/sqlite-sqltests/alter-column-keeps-pk-desc.sqltest @@ -0,0 +1,34 @@ +@database :memory: + +setup desc-pk-tables { + CREATE TABLE t(a INTEGER PRIMARY KEY DESC, q TEXT); + CREATE TABLE v(a TEXT PRIMARY KEY DESC, q TEXT); + CREATE TABLE w(a INTEGER PRIMARY KEY DESC, q TEXT, r TEXT); +} + +@setup desc-pk-tables +test add-column-keeps-desc-on-integer-primary-key { + ALTER TABLE t ADD COLUMN z TEXT; + SELECT instr(sql, 'PRIMARY KEY DESC') > 0 FROM sqlite_schema WHERE name = 't'; +} +expect { + 1 +} + +@setup desc-pk-tables +test add-column-keeps-desc-on-text-primary-key { + ALTER TABLE v ADD COLUMN z TEXT; + SELECT instr(sql, 'PRIMARY KEY DESC') > 0 FROM sqlite_schema WHERE name = 'v'; +} +expect { + 1 +} + +@setup desc-pk-tables +test drop-column-keeps-desc-on-integer-primary-key { + ALTER TABLE w DROP COLUMN r; + SELECT instr(sql, 'PRIMARY KEY DESC') > 0 FROM sqlite_schema WHERE name = 'w'; +} +expect { + 1 +} diff --git a/conformance/sqlite-sqltests/alter-rename-table-to-digit-leading-name.sqltest b/conformance/sqlite-sqltests/alter-rename-table-to-digit-leading-name.sqltest new file mode 100644 index 00000000..709c09dd --- /dev/null +++ b/conformance/sqlite-sqltests/alter-rename-table-to-digit-leading-name.sqltest @@ -0,0 +1,62 @@ +@database :memory: + +# Regression tests for https://github.com/tursodatabase/turso/issues/8700 +# +# ALTER TABLE ... RENAME TO "1a" must keep the quotes around the new name when +# it rewrites the stored CREATE TABLE and CREATE INDEX text. A bare `1a` is not +# a valid identifier, so writing it unquoted leaves schema rows that no longer +# parse: the next ALTER in the same session fails with `bad number '1'`, and +# the database file can no longer be opened by Turso or SQLite. + +setup renamed-to-digit-leading-name { + CREATE TABLE t(a); + CREATE TABLE u(a); + CREATE INDEX i ON t(a); + INSERT INTO t VALUES (7); + ALTER TABLE t RENAME TO "1a"; +} + +@setup renamed-to-digit-leading-name +test rename-to-digit-leading-name-keeps-quotes-in-table-sql { + SELECT sql LIKE '%"1a"%' FROM sqlite_schema WHERE name = '1a'; +} +expect { + 1 +} + +@setup renamed-to-digit-leading-name +test rename-to-digit-leading-name-keeps-quotes-in-index-sql { + SELECT sql LIKE '%"1a"%' FROM sqlite_schema WHERE name = 'i'; +} +expect { + 1 +} + +@setup renamed-to-digit-leading-name +test next-alter-after-rename-to-digit-leading-name-still-works { + ALTER TABLE u RENAME TO "b c"; + SELECT count(*) FROM sqlite_schema WHERE name = 'b c'; +} +expect { + 1 +} + +@setup renamed-to-digit-leading-name +test renamed-digit-leading-table-can-be-renamed-again { + ALTER TABLE "1a" RENAME TO "select"; + SELECT sql LIKE '%"select"%' FROM sqlite_schema WHERE name = 'select'; + SELECT a FROM "select"; +} +expect { + 1 + 7 +} + +test rename-to-name-with-space-keeps-quotes { + CREATE TABLE w(a); + ALTER TABLE w RENAME TO "b c"; + SELECT sql LIKE '%"b c"%' FROM sqlite_schema WHERE name = 'b c'; +} +expect { + 1 +} diff --git a/conformance/sqlite-sqltests/alter-table-fk-without-parent-columns.sqltest b/conformance/sqlite-sqltests/alter-table-fk-without-parent-columns.sqltest new file mode 100644 index 00000000..d34d9240 --- /dev/null +++ b/conformance/sqlite-sqltests/alter-table-fk-without-parent-columns.sqltest @@ -0,0 +1,40 @@ +# Regression test for https://github.com/tursodatabase/turso/issues/8701 +# ALTER TABLE ADD COLUMN and DROP COLUMN rebuild the stored CREATE TABLE text. +# A foreign key that names no parent columns must be written back as +# REFERENCES p, not REFERENCES p(), which no engine can parse on reopen. + +@database :memory: + +setup parent { + CREATE TABLE p(id INTEGER PRIMARY KEY); +} + +@setup parent +test alter-add-column-keeps-table-fk-without-parent-columns { + CREATE TABLE t(a INTEGER, FOREIGN KEY(a) REFERENCES p); + ALTER TABLE t ADD COLUMN c INTEGER; + SELECT sql LIKE '%REFERENCES p()%', sql LIKE '%REFERENCES p%' FROM sqlite_schema WHERE name = 't'; +} +expect { + 0|1 +} + +@setup parent +test alter-drop-column-keeps-column-fk-without-parent-columns { + CREATE TABLE u(a INTEGER REFERENCES p, b INTEGER, d INTEGER); + ALTER TABLE u DROP COLUMN d; + SELECT sql LIKE '%REFERENCES p()%', sql LIKE '%REFERENCES p%' FROM sqlite_schema WHERE name = 'u'; +} +expect { + 0|1 +} + +@setup parent +test alter-add-column-keeps-fk-without-parent-columns-with-actions { + CREATE TABLE t(a INTEGER, FOREIGN KEY(a) REFERENCES p ON DELETE CASCADE); + ALTER TABLE t ADD COLUMN c INTEGER; + SELECT sql LIKE '%REFERENCES p()%', sql LIKE '%REFERENCES p ON DELETE CASCADE%' FROM sqlite_schema WHERE name = 't'; +} +expect { + 0|1 +} diff --git a/conformance/sqlite-sqltests/alter-table-keeps-on-conflict-clause.sqltest b/conformance/sqlite-sqltests/alter-table-keeps-on-conflict-clause.sqltest new file mode 100644 index 00000000..d0d1a5c8 --- /dev/null +++ b/conformance/sqlite-sqltests/alter-table-keeps-on-conflict-clause.sqltest @@ -0,0 +1,132 @@ +# Regression test for https://github.com/tursodatabase/turso/issues/8703 +# ALTER TABLE ADD COLUMN and DROP COLUMN rebuild the stored CREATE TABLE text. +# The rebuilt text must keep each constraint's ON CONFLICT clause, otherwise a +# connection that reopens the file enforces ABORT where the schema said ROLLBACK. + +@database :memory: + +test alter-add-column-keeps-unique-on-conflict-rollback { + CREATE TABLE t(id INTEGER, k TEXT UNIQUE ON CONFLICT ROLLBACK, x INTEGER, j TEXT); + ALTER TABLE t ADD COLUMN note TEXT; + SELECT sql LIKE '%ON CONFLICT ROLLBACK%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-drop-column-keeps-unique-on-conflict-rollback { + CREATE TABLE u(id INTEGER, k TEXT UNIQUE ON CONFLICT ROLLBACK, x INTEGER, j TEXT); + ALTER TABLE u DROP COLUMN j; + SELECT sql LIKE '%ON CONFLICT ROLLBACK%' FROM sqlite_master WHERE name='u'; +} +expect { + 1 +} + +test alter-add-column-keeps-not-null-on-conflict-ignore { + CREATE TABLE t(id INTEGER, k TEXT NOT NULL ON CONFLICT IGNORE, j TEXT); + ALTER TABLE t ADD COLUMN note TEXT; + SELECT sql LIKE '%NOT NULL ON CONFLICT IGNORE%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-add-column-keeps-primary-key-on-conflict-replace { + CREATE TABLE t(id INTEGER PRIMARY KEY ON CONFLICT REPLACE, j TEXT); + ALTER TABLE t ADD COLUMN note TEXT; + SELECT sql LIKE '%PRIMARY KEY ON CONFLICT REPLACE%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-rename-column-keeps-unique-on-conflict-rollback { + CREATE TABLE v(id INTEGER, k TEXT UNIQUE ON CONFLICT ROLLBACK, x INTEGER, j TEXT); + ALTER TABLE v RENAME COLUMN j TO j2; + SELECT sql LIKE '%ON CONFLICT ROLLBACK%' FROM sqlite_master WHERE name='v'; +} +expect { + 1 +} + +test alter-add-column-keeps-each-clause-on-its-own-column-constraint { + CREATE TABLE t(id INTEGER, a TEXT NOT NULL ON CONFLICT IGNORE UNIQUE ON CONFLICT REPLACE, j TEXT); + ALTER TABLE t ADD COLUMN note TEXT; + SELECT sql LIKE '%NOT NULL ON CONFLICT IGNORE UNIQUE ON CONFLICT REPLACE%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-drop-column-keeps-each-clause-on-its-own-column-constraint-with-primary-key { + CREATE TABLE t(a TEXT NOT NULL ON CONFLICT FAIL PRIMARY KEY ON CONFLICT ROLLBACK, b TEXT NOT NULL ON CONFLICT IGNORE UNIQUE ON CONFLICT REPLACE, j TEXT); + ALTER TABLE t DROP COLUMN j; + SELECT sql LIKE '%a TEXT NOT NULL ON CONFLICT FAIL PRIMARY KEY ON CONFLICT ROLLBACK, b TEXT NOT NULL ON CONFLICT IGNORE UNIQUE ON CONFLICT REPLACE%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-add-column-keeps-table-primary-key-on-conflict-rollback { + CREATE TABLE t(c INTEGER, d TEXT, j TEXT, PRIMARY KEY (c, d) ON CONFLICT ROLLBACK); + ALTER TABLE t ADD COLUMN note TEXT; + SELECT sql LIKE '%PRIMARY KEY (c, d) ON CONFLICT ROLLBACK%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-drop-column-keeps-table-primary-key-on-conflict-rollback { + CREATE TABLE t(c INTEGER, d TEXT, j TEXT, PRIMARY KEY (c, d) ON CONFLICT ROLLBACK); + ALTER TABLE t DROP COLUMN j; + SELECT sql LIKE '%PRIMARY KEY (c, d) ON CONFLICT ROLLBACK%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-add-column-keeps-table-unique-on-conflict-ignore { + CREATE TABLE t(c INTEGER, d TEXT, j TEXT, UNIQUE (c, d) ON CONFLICT IGNORE); + ALTER TABLE t ADD COLUMN note TEXT; + SELECT sql LIKE '%UNIQUE (c, d) ON CONFLICT IGNORE%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-drop-column-keeps-each-table-unique-clause { + CREATE TABLE t(c INTEGER, d TEXT, e TEXT, j TEXT, UNIQUE (c, d) ON CONFLICT IGNORE, UNIQUE (d, e) ON CONFLICT REPLACE); + ALTER TABLE t DROP COLUMN j; + SELECT sql LIKE '%UNIQUE (c, d) ON CONFLICT IGNORE%UNIQUE (d, e) ON CONFLICT REPLACE%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-add-column-keeps-text-primary-key-on-conflict-replace { + CREATE TABLE t(k TEXT PRIMARY KEY ON CONFLICT REPLACE, j TEXT); + ALTER TABLE t ADD COLUMN note TEXT; + SELECT sql LIKE '%PRIMARY KEY ON CONFLICT REPLACE%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-add-column-keeps-without-rowid-primary-key-on-conflict-ignore { + CREATE TABLE t(k TEXT PRIMARY KEY ON CONFLICT IGNORE, j TEXT) WITHOUT ROWID; + ALTER TABLE t ADD COLUMN note TEXT; + SELECT sql LIKE '%PRIMARY KEY ON CONFLICT IGNORE%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} + +test alter-add-column-keeps-without-rowid-table-primary-key-on-conflict-fail { + CREATE TABLE t(c INTEGER, d TEXT, j TEXT, PRIMARY KEY (c, d) ON CONFLICT FAIL) WITHOUT ROWID; + ALTER TABLE t ADD COLUMN note TEXT; + SELECT sql LIKE '%PRIMARY KEY (c, d) ON CONFLICT FAIL%' FROM sqlite_master WHERE name='t'; +} +expect { + 1 +} diff --git a/conformance/sqlite-sqltests/alter_table.sqltest b/conformance/sqlite-sqltests/alter_table.sqltest index 82e9babe..9c2a80b8 100644 --- a/conformance/sqlite-sqltests/alter_table.sqltest +++ b/conformance/sqlite-sqltests/alter_table.sqltest @@ -45,6 +45,30 @@ expect { CREATE INDEX i ON t (b) } +@cross-check-integrity +test alter-table-rename-column-table-unique-collate { + CREATE TABLE t (a TEXT, b TEXT, UNIQUE (a COLLATE NOCASE)); + INSERT INTO t VALUES ('a', 'x'); + ALTER TABLE t RENAME COLUMN b TO c; + ALTER TABLE t RENAME COLUMN a TO d; + SELECT d, c FROM t; +} +expect { + a|x +} + +@cross-check-integrity +test alter-table-rename-column-table-primary-key-collate { + CREATE TABLE t (a TEXT, b TEXT, PRIMARY KEY (a COLLATE NOCASE)); + INSERT INTO t VALUES ('a', 'x'); + ALTER TABLE t RENAME COLUMN b TO c; + ALTER TABLE t RENAME COLUMN a TO d; + SELECT d, c FROM t; +} +expect { + a|x +} + @skip-if mvcc "views not supported in MVCC mode" test alter-table-rename-column-view { CREATE TABLE t (a, b); @@ -272,6 +296,19 @@ expect { 1|0 } +@cross-check-integrity +test alter-table-add-drop-column-preserves-type-size { + CREATE TABLE t (a VARCHAR(10)); + ALTER TABLE t ADD COLUMN b DECIMAL(10, 2); + SELECT sql FROM sqlite_schema; + ALTER TABLE t DROP COLUMN a; + SELECT sql FROM sqlite_schema; +} +expect { + CREATE TABLE t (a VARCHAR(10), b DECIMAL(10, 2)) + CREATE TABLE t (b DECIMAL(10, 2)) +} + @cross-check-integrity test alter-table-add-column-nullability { CREATE TABLE t (a INTEGER); @@ -582,6 +619,28 @@ test fail-alter-table-rename-to-existing-table { expect error { } +@skip-if mvcc "views not supported in MVCC mode" +@cross-check-integrity +test fail-alter-table-rename-to-existing-view { + CREATE TABLE x (a); + CREATE VIEW y AS SELECT 1; + ALTER TABLE x RENAME TO y; +} +expect error { + there is already another table or index with this name: y +} + +@skip-if mvcc "views not supported in MVCC mode" +@cross-check-integrity +test fail-alter-table-rename-to-existing-view-different-case { + CREATE TABLE x (a); + CREATE VIEW y AS SELECT 1; + ALTER TABLE x RENAME TO Y; +} +expect error { + there is already another table or index with this name: Y +} + @cross-check-integrity test alter-table-rename-to-quoted-identifier { CREATE TABLE t (a); @@ -802,11 +861,13 @@ expect { test strict-alter-add-column-any { CREATE TABLE t(a INTEGER) STRICT; ALTER TABLE t ADD COLUMN b ANY; - INSERT INTO t VALUES(1, 3.14); - SELECT a, b FROM t; + INSERT INTO t VALUES(1, 3.14), (2, '000123'), (3, '1.5'); + SELECT a, b, typeof(b) FROM t ORDER BY a; } expect { - 1|3.14 + 1|3.14|real + 2|000123|text + 3|1.5|text } @cross-check-integrity @@ -902,6 +963,51 @@ expect error { cannot store REAL value in INTEGER column t.b } +# https://github.com/tursodatabase/turso/issues/8763 +@cross-check-integrity +test nonstrict-alter-add-column-default-blob-from-text { + CREATE TABLE t(a INTEGER); + INSERT INTO t VALUES (1); + ALTER TABLE t ADD COLUMN b BLOB DEFAULT ''; + SELECT quote(b), typeof(b) FROM t; +} +expect { + ''|text +} + +@cross-check-integrity +test nonstrict-alter-add-column-default-integer-fractional { + CREATE TABLE t(a INTEGER); + INSERT INTO t VALUES (1); + ALTER TABLE t ADD COLUMN b INTEGER DEFAULT 3.7; + SELECT b, typeof(b) FROM t; +} +expect { + 3.7|real +} + +@cross-check-integrity +test nonstrict-alter-add-column-default-text-from-blob { + CREATE TABLE t(a INTEGER); + INSERT INTO t VALUES (1); + ALTER TABLE t ADD COLUMN b TEXT DEFAULT X'CAFE'; + SELECT quote(b), typeof(b) FROM t; +} +expect { + X'CAFE'|blob +} + +@cross-check-integrity +test nonstrict-alter-add-column-default-real-nonnumeric { + CREATE TABLE t(a INTEGER); + INSERT INTO t VALUES (1); + ALTER TABLE t ADD COLUMN b REAL DEFAULT 'abc'; + SELECT quote(b), typeof(b) FROM t; +} +expect { + 'abc'|text +} + @cross-check-integrity test alter-add-column-invalid-collation { CREATE TABLE t(c1 INTEGER); @@ -3016,3 +3122,47 @@ test alter-table-add-column-nonconstant-default-nonempty-table-errors { expect error { Cannot add a column with non-constant default } + +# The following preserve-schema-collate tests ensure that alter table +# preserve the original collation defined on table creation. +# https://github.com/tursodatabase/turso/issues/8254 +@cross-check-integrity +test alter-table-add-column-preserve-schema-collate-nocase { + CREATE TABLE tbl (c TEXT COLLATE NOCASE); + ALTER TABLE tbl ADD COLUMN d; + SELECT sql FROM sqlite_schema WHERE name = 'tbl'; +} +expect { + CREATE TABLE tbl (c TEXT COLLATE NOCASE, d) +} + +@cross-check-integrity +test alter-table-add-column-preserve-schema-collate-rtrim { + CREATE TABLE tbl (c TEXT COLLATE RTRIM); + ALTER TABLE tbl ADD COLUMN d; + SELECT sql FROM sqlite_schema WHERE name = 'tbl'; +} +expect { + CREATE TABLE tbl (c TEXT COLLATE RTRIM, d) +} + +@cross-check-integrity +test alter-table-add-column-preserve-schema-collate-binary { + CREATE TABLE tbl (c TEXT COLLATE BINARY); + ALTER TABLE tbl ADD COLUMN d; + SELECT sql FROM sqlite_schema WHERE name = 'tbl'; +} +expect { + CREATE TABLE tbl (c TEXT COLLATE BINARY, d) +} + +@cross-check-integrity +@skip-if sqlite "SQLite does not support locale-based collation" +test alter-table-add-column-preserve-schema-collate-locale { + CREATE TABLE tbl (c TEXT COLLATE 'fr-FR'); + ALTER TABLE tbl ADD COLUMN d; + SELECT sql FROM sqlite_schema WHERE name = 'tbl'; +} +expect { + CREATE TABLE tbl (c TEXT COLLATE "fr-FR", d) +} diff --git a/conformance/sqlite-sqltests/btree-tiny-cell-divider.sqltest b/conformance/sqlite-sqltests/btree-tiny-cell-divider.sqltest new file mode 100644 index 00000000..94c19ee8 --- /dev/null +++ b/conformance/sqlite-sqltests/btree-tiny-cell-divider.sqltest @@ -0,0 +1,58 @@ +@database :memory: + +# A zero-length string is a 2-byte record, so its index cell is 3 bytes: one +# byte below the 4-byte minimum cell size. When balancing promotes that cell +# from a leaf to a divider cell in the parent, balance_non_root aborts with +# "corrupted database, cells were not balanced properly". This is the bug +# class upstream in2.test (2007) was written for. +# +# The sorted b-tree contents are N integers, one empty string, then 2000 +# short strings. N = 311 is the first count that puts the empty string on a +# split boundary; N = 310 is a control that does not. + +setup tiny-cell-table { + CREATE TABLE a(i INTEGER PRIMARY KEY, a); + WITH RECURSIVE n(v) AS (SELECT 0 UNION ALL SELECT v + 1 FROM n WHERE v < 1999) + INSERT INTO a SELECT v, v FROM n; + INSERT INTO a VALUES(4000, ''); + WITH RECURSIVE n(v) AS (SELECT 0 UNION ALL SELECT v + 1 FROM n WHERE v < 1999) + INSERT INTO a SELECT NULL, printf('x%04d', v) FROM n; +} + +# The IN (...) subquery builds an ephemeral index b-tree from the selected +# values, so the empty string is a 3-byte cell in it. + +@setup tiny-cell-table +test in-ephemeral-btree-empty-string-off-split-boundary { + SELECT 1 IN (SELECT a FROM a WHERE (i < 310) OR (i >= 2000)); +} +expect { + 1 +} + +@setup tiny-cell-table +test in-ephemeral-btree-empty-string-on-split-boundary { + SELECT 1 IN (SELECT a FROM a WHERE (i < 311) OR (i >= 2000)); +} +expect { + 1 +} + +# A WITHOUT ROWID table with a single-column key is a persistent index b-tree, +# so an empty-string key is a 3-byte cell on disk and hits the same path on +# INSERT. + +@skip-if mvcc "WITHOUT ROWID tables are not supported in MVCC mode" +@cross-check-integrity +test without-rowid-empty-string-key-on-split-boundary { + CREATE TABLE t(a PRIMARY KEY) WITHOUT ROWID; + WITH RECURSIVE n(v) AS (SELECT 0 UNION ALL SELECT v + 1 FROM n WHERE v < 310) + INSERT INTO t SELECT v FROM n; + INSERT INTO t VALUES(''); + WITH RECURSIVE n(v) AS (SELECT 0 UNION ALL SELECT v + 1 FROM n WHERE v < 1999) + INSERT INTO t SELECT printf('x%04d', v) FROM n; + SELECT count(*), min(a), max(a) FROM t; +} +expect { + 2312|0|x1999 +} diff --git a/conformance/sqlite-sqltests/collate.sqltest b/conformance/sqlite-sqltests/collate.sqltest index 0682da06..f74a5ea2 100644 --- a/conformance/sqlite-sqltests/collate.sqltest +++ b/conformance/sqlite-sqltests/collate.sqltest @@ -274,6 +274,26 @@ expect { Apple|0 } +@cross-check-integrity +test collate_does_not_inherit_previous_column { + CREATE TABLE t(a TEXT COLLATE NOCASE, b TEXT, id INTEGER); + INSERT INTO t VALUES('A', 'B', 1); + SELECT 'b' = b FROM t WHERE a IS NOT NULL; + SELECT id, 'a' = a FROM t; +} +expect { + 0 + 1|1 +} + +@cross-check-integrity +test collate_in_list_does_not_leak_to_following_result { + SELECT 'A' COLLATE NOCASE IN ('a'), 'A' IN ('a'), 'B' IN ('b'); +} +expect { + 1|0|0 +} + # ===================================================================== # --- Hash join with NOCASE on build-side column, comparison in SELECT --- # The NOCASE column ends up in the hash payload; the cached register must @@ -1495,3 +1515,56 @@ test agg_collate_no_leak_with_count { expect { a|a|2 } + +setup mixed_collation_join { + CREATE TABLE nocase_side(id INTEGER PRIMARY KEY, k TEXT COLLATE NOCASE); + CREATE TABLE binary_side(k TEXT, v INTEGER); + INSERT INTO nocase_side VALUES (1,'abc'),(2,'DEF'),(3,'ghi'); + INSERT INTO binary_side VALUES ('ABC',1),('def',1); +} + +# The left operand's column supplies the comparison collation, even when that +# column has no declared collation (BINARY). binary_side.k is the left operand, +# so the comparison is BINARY and no rows match. +@setup mixed_collation_join +test join-comparison-uses-left-column-collation { + SELECT n.id FROM nocase_side n JOIN binary_side b ON b.k = n.k ORDER BY n.id; +} +expect { +} + +# Flipped: nocase_side.k is the left operand, so the comparison is NOCASE. +@setup mixed_collation_join +test join-comparison-uses-left-column-collation-flipped { + SELECT n.id FROM nocase_side n JOIN binary_side b ON n.k = b.k ORDER BY n.id; +} +expect { + 1 + 2 +} + +# Same rule through the NOT EXISTS hash anti join: BINARY comparison, so every +# nocase_side row survives. +@setup mixed_collation_join +test not-exists-comparison-uses-left-column-collation { + SELECT n.id FROM nocase_side n + WHERE NOT EXISTS (SELECT 1 FROM binary_side b WHERE b.k = n.k) + ORDER BY n.id; +} +expect { + 1 + 2 + 3 +} + +# A NOCASE index cannot serve a BINARY comparison. INDEXED BY only forces the +# index to be considered; the seek must still be rejected here. +@setup mixed_collation_join +test nocase-index-does-not-serve-binary-comparison { + CREATE INDEX nocase_side_k ON nocase_side(k); + SELECT n.id FROM binary_side b JOIN nocase_side n INDEXED BY nocase_side_k + ON b.k = n.k + ORDER BY n.id; +} +expect { +} diff --git a/conformance/sqlite-sqltests/compare.sqltest b/conformance/sqlite-sqltests/compare.sqltest index bfbe2895..93db52aa 100644 --- a/conformance/sqlite-sqltests/compare.sqltest +++ b/conformance/sqlite-sqltests/compare.sqltest @@ -728,3 +728,13 @@ expect { expect @js { -1870756973399217200|0|0|1 } + +test compare-text-operands-affinity-and-collation { + CREATE TABLE t(d DATE, b, n NUMERIC, c TEXT COLLATE NOCASE); + INSERT INTO t VALUES ('1997-06-01', '9', '10x', 'abc'), ('1997-09-01', '10', '9x', 'ABD'); + SELECT d, d >= '1997-06-01', d < '1997-09-01', b < n, b > n, b = '9', c = 'ABC', c < 'abd', b < '10' FROM t; +} +expect { + 1997-06-01|1|1|1|0|1|1|1|0 + 1997-09-01|1|0|1|0|0|0|0|0 +} diff --git a/conformance/sqlite-sqltests/concat.sqltest b/conformance/sqlite-sqltests/concat.sqltest index 814254e7..72f8f419 100644 --- a/conformance/sqlite-sqltests/concat.sqltest +++ b/conformance/sqlite-sqltests/concat.sqltest @@ -61,3 +61,44 @@ test concat-ws-one-arg-error { expect error { wrong number of arguments to function concat_ws() } + +# `||` converts each side to text. Only a TEXT value is already in that form; +# every other type is rendered first, so each one needs its own case. +test concat-value-types { + SELECT quote(1 || 2); + SELECT quote(1.5 || 'x'); + SELECT quote(-0.0 || ''); + SELECT quote(1e300 || ''); + SELECT quote('' || 'b'); + SELECT quote(NULL || 'a'); + SELECT quote('a' || NULL); +} +expect { + '12' + '1.5x' + '0.0' + '1.0e+300' + 'b' + NULL + NULL +} + +test concat-mixed-blob-and-text { + SELECT quote(x'41' || 'b'); + SELECT quote('b' || x'41'); + SELECT quote(x'e282ac' || 'x'); +} +expect { + 'Ab' + 'bA' + '€x' +} + +test concat-multibyte-text { + SELECT quote('héllo' || '—'); + SELECT length('héllo' || '—'); +} +expect { + 'héllo—' + 6 +} diff --git a/conformance/sqlite-sqltests/create_table_as_select.sqltest b/conformance/sqlite-sqltests/create_table_as_select.sqltest index a007a9f3..f20ab300 100644 --- a/conformance/sqlite-sqltests/create_table_as_select.sqltest +++ b/conformance/sqlite-sqltests/create_table_as_select.sqltest @@ -629,3 +629,21 @@ test ctas_compound_select_column_names { expect { CREATE TABLE dst(a INT,b TEXT) } + +@cross-check-integrity +test ctas_compound_select_keeps_mixed_value_types { + CREATE TABLE source(x REAL); + INSERT INTO source VALUES (2.5); + CREATE TABLE dst AS + SELECT x FROM source + UNION ALL + SELECT '3.5'; + + SELECT name, type FROM pragma_table_info('dst'); + SELECT quote(x), typeof(x) FROM dst; +} +expect { + x| + 2.5|real + '3.5'|text +} diff --git a/conformance/sqlite-sqltests/foreign_keys.sqltest b/conformance/sqlite-sqltests/foreign_keys.sqltest index b3bab3ec..a309a7bd 100644 --- a/conformance/sqlite-sqltests/foreign_keys.sqltest +++ b/conformance/sqlite-sqltests/foreign_keys.sqltest @@ -4334,3 +4334,70 @@ test fk-deferred-null-parent-key-does-not-repay { } expect error { } + +# UPDATE and UPSERT only emit parent-side foreign-key bytecode when the +# statement can change the key that the children point at. These snapshots record +# which statements get the OLD/NEW key probes and which get the CASCADE, +# SET NULL or SET DEFAULT action subprograms. + +setup fk_bytecode_no_action_child { + PRAGMA foreign_keys=ON; + CREATE TABLE parent(id INTEGER PRIMARY KEY, k INTEGER UNIQUE, v INTEGER); + CREATE TABLE child(id INTEGER PRIMARY KEY, pk INTEGER REFERENCES parent(k) ON UPDATE NO ACTION); +} + +setup fk_bytecode_cascade_child { + PRAGMA foreign_keys=ON; + CREATE TABLE parent(id INTEGER PRIMARY KEY, k INTEGER UNIQUE, v INTEGER); + CREATE TABLE child(id INTEGER PRIMARY KEY, pk INTEGER REFERENCES parent(k) ON UPDATE CASCADE); +} + +setup fk_bytecode_mixed_children { + PRAGMA foreign_keys=ON; + CREATE TABLE parent(id INTEGER PRIMARY KEY, k INTEGER UNIQUE); + CREATE TABLE child_no_action(id INTEGER PRIMARY KEY, pk INTEGER REFERENCES parent(k) ON UPDATE NO ACTION); + CREATE TABLE child_cascade(id INTEGER PRIMARY KEY, pk INTEGER REFERENCES parent(k) ON UPDATE CASCADE); +} + +# No child points at parent.v, so this UPDATE gets no key probe and no action. +@setup fk_bytecode_no_action_child +@skip-if mvcc "MVCC uses negative root page numbers, so the bytecode snapshot differs" +snapshot fk-update-column-no-child-points-at { + UPDATE parent SET v = v + 1 WHERE id = 1; +} + +# The UPDATE changes parent.k, so the NO ACTION child gets its key probe. +@setup fk_bytecode_no_action_child +@skip-if mvcc "MVCC uses negative root page numbers, so the bytecode snapshot differs" +snapshot fk-update-parent-key-with-no-action-child { + UPDATE parent SET k = k + 1 WHERE id = 1; +} + +# The UPDATE changes parent.k, so the CASCADE child gets its action subprogram. +@setup fk_bytecode_cascade_child +@skip-if mvcc "MVCC uses negative root page numbers, so the bytecode snapshot differs" +snapshot fk-update-parent-key-with-cascade-child { + UPDATE parent SET k = k + 1 WHERE id = 1; +} + +# Both children point at parent.k, but only the CASCADE child gets an action +# subprogram after the row is written. +@setup fk_bytecode_mixed_children +@skip-if mvcc "MVCC uses negative root page numbers, so the bytecode snapshot differs" +snapshot fk-update-parent-key-with-mixed-children { + UPDATE parent SET k = k + 1 WHERE id = 1; +} + +# The upsert writes parent.v only, so the CASCADE child gets no action. +@setup fk_bytecode_cascade_child +@skip-if mvcc "MVCC uses negative root page numbers, so the bytecode snapshot differs" +snapshot fk-upsert-column-no-child-points-at { + INSERT INTO parent(id, k, v) VALUES (1, 10, 2) ON CONFLICT(id) DO UPDATE SET v = excluded.v; +} + +# The upsert writes parent.k, so the CASCADE child gets its action subprogram. +@setup fk_bytecode_cascade_child +@skip-if mvcc "MVCC uses negative root page numbers, so the bytecode snapshot differs" +snapshot fk-upsert-parent-key-with-cascade-child { + INSERT INTO parent(id, k, v) VALUES (1, 10, 2) ON CONFLICT(id) DO UPDATE SET k = excluded.k; +} diff --git a/conformance/sqlite-sqltests/function_rowid.sqltest b/conformance/sqlite-sqltests/function_rowid.sqltest new file mode 100644 index 00000000..de669ef3 --- /dev/null +++ b/conformance/sqlite-sqltests/function_rowid.sqltest @@ -0,0 +1,26 @@ +@database :memory: + +@cross-check-integrity +test function-rowid-last-insert-rowid-null-then-explicit { + CREATE TABLE test_rowid (id INTEGER PRIMARY KEY, val TEXT); + INSERT INTO test_rowid (id, val) VALUES (NULL, 'test1'); + SELECT last_insert_rowid(); + INSERT INTO test_rowid (id, val) VALUES (5, 'test2'); + SELECT last_insert_rowid(); +} +expect { + 1 + 5 +} + +@cross-check-integrity +test function-rowid-integer-primary-key-auto-assignment-after-negative { + CREATE TABLE test_rowid (id INTEGER PRIMARY KEY); + INSERT INTO test_rowid VALUES (-1); + INSERT INTO test_rowid VALUES (NULL); + SELECT * FROM test_rowid; +} +expect { + -1 + 0 +} diff --git a/conformance/sqlite-sqltests/gencol.sqltest b/conformance/sqlite-sqltests/gencol.sqltest index d588f90d..d3dc907a 100644 --- a/conformance/sqlite-sqltests/gencol.sqltest +++ b/conformance/sqlite-sqltests/gencol.sqltest @@ -1410,7 +1410,6 @@ expect { 1|5|105 } -@skip "TODO: INSERT OR REPLACE with virtual column index triggers SELF_TABLE parse error" test gencol_insert_or_replace_with_index { CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER, c AS (b * 2)); CREATE INDEX idx1 ON t1(c); @@ -3308,13 +3307,24 @@ expect error { (Cannot add a PRIMARY KEY column|generated columns cannot be part of the PRIMARY KEY) } -@skip "TODO: parser accepts DEFAULT + GENERATED without error" +# Regression test for https://github.com/tursodatabase/turso/issues/7058 +# Generated columns may not use the DEFAULT clause (SQLite gencol.html +# §2.3 limitation 1). The order of DEFAULT vs. GENERATED/AS does not +# matter; both combinations are rejected. test gencol_alter_add_column_generated_default_rejected { CREATE TABLE t1(a INTEGER); ALTER TABLE t1 ADD COLUMN b DEFAULT 5 AS (a+1); } expect error { - (Cannot add a column with non-constant default|error in generated column|cannot have a DEFAULT) + (Cannot add a column with non-constant default|error in generated column|cannot have a DEFAULT|cannot use DEFAULT on a generated column) +} + +test gencol_alter_add_column_default_after_generated_rejected { + CREATE TABLE t1(a INTEGER); + ALTER TABLE t1 ADD COLUMN b AS (a+1) DEFAULT 5; +} +expect error { + (Cannot add a column with non-constant default|error in generated column|cannot have a DEFAULT|cannot use DEFAULT on a generated column) } # NOT NULL on virtual column validated against existing data @@ -4261,6 +4271,21 @@ expect { 3|30|3|30 } +# Regression test for https://github.com/tursodatabase/turso/issues/8472 +test gencol_autoindex_dependency { + CREATE TABLE measurements ( + computed_value AS (base_value), + base_value + ); + INSERT INTO measurements (base_value) VALUES (1); + SELECT measurements.base_value + FROM measurements, (SELECT 1) AS one_row + WHERE measurements.computed_value > 0; +} +expect { + 1 +} + # ============================================================================= # Three-table INNER JOIN with virtual generated columns (#6151) # ============================================================================= @@ -5064,3 +5089,699 @@ test update-or-virtual-deferred-seek-no-panic-explicit-index { expect error { UNIQUE constraint failed } + +# ============================================================================= +# Coverage from GitHub issues. +# ============================================================================= + +# Issue #7065: indexed generated-column seeks must use the column's +# declared affinity (INT for "g INT AS (a)"), not the generating expression's. + +test gencol_issue7065_indexed_seek_select { + CREATE TABLE t(a TEXT, g INT AS (a)); + CREATE INDEX i ON t(g); + INSERT INTO t(a) VALUES('1'),('2'); + SELECT a, g FROM t WHERE g = 1; +} +expect { + 1|1 +} + +test gencol_issue7065_indexed_seek_delete { + CREATE TABLE t(a TEXT, g INT AS (a)); + CREATE INDEX i ON t(g); + INSERT INTO t(a) VALUES('1'),('2'); + DELETE FROM t WHERE g = 1; + SELECT a, g FROM t; +} +expect { + 2|2 +} + +test gencol_issue7065_indexed_seek_update { + CREATE TABLE t(a TEXT, g INT AS (a)); + CREATE INDEX i ON t(g); + INSERT INTO t(a) VALUES('1'),('2'); + UPDATE t SET a = '9' WHERE g = 2; + SELECT a, g FROM t ORDER BY a; +} +expect { + 1|1 + 9|9 +} + +test gencol_issue7065_indexed_seek_between { + CREATE TABLE t(a TEXT, g INT AS (a)); + CREATE INDEX i ON t(g); + INSERT INTO t(a) VALUES('1'),('2'),('3'); + SELECT a, g FROM t WHERE g BETWEEN 1 AND 2; +} +expect { + 1|1 + 2|2 +} + +test gencol_issue7065_indexed_seek_in { + CREATE TABLE t(a TEXT, g INT AS (a)); + CREATE INDEX i ON t(g); + INSERT INTO t(a) VALUES('1'),('2'),('3'); + SELECT a, g FROM t WHERE g IN (1, 3); +} +expect { + 1|1 + 3|3 +} + +test gencol_issue7065_indexed_seek_join { + CREATE TABLE t(a TEXT, g INT AS (a)); + CREATE INDEX i ON t(g); + INSERT INTO t(a) VALUES('1'),('2'),('3'); + CREATE TABLE u(k INT); + INSERT INTO u VALUES(1),(3); + SELECT t.a, u.k FROM t JOIN u ON t.g = u.k ORDER BY u.k; +} +expect { + 1|1 + 3|3 +} + +test gencol_issue7065_indexed_seek_text_gencol_from_int { + CREATE TABLE t(a INT, g TEXT AS (a)); + CREATE INDEX i ON t(g); + INSERT INTO t(a) VALUES(1),(2); + SELECT a, g FROM t WHERE g = '1'; + SELECT a, g FROM t WHERE g = 1; +} +expect { + 1|1 + 1|1 +} + +# Issue #8252: self-join on generated columns built the automatic +# index by evaluating the generated expression against the ephemeral index +# cursor instead of the table cursor, so the join returned no rows. + +test gencol_issue8252_self_join_two_gencols { + CREATE TABLE g(a INTEGER, v1 AS (a + 1) VIRTUAL, s1 AS (a * 2) VIRTUAL); + INSERT INTO g(a) VALUES (1), (2); + SELECT g1.a, g2.a FROM g g1 JOIN g g2 ON g1.v1 = g2.s1; +} +expect { + 1|1 +} + +test gencol_issue8252_self_join_same_gencol { + CREATE TABLE g(a INTEGER, v1 AS (a + 1) VIRTUAL, s1 AS (a * 2) VIRTUAL); + INSERT INTO g(a) VALUES (1), (2); + SELECT g1.a, g2.a FROM g g1 JOIN g g2 ON g1.v1 = g2.v1 ORDER BY g1.a, g2.a; +} +expect { + 1|1 + 2|2 +} + +test gencol_issue8252_cross_join_where_gencols { + CREATE TABLE g(a INTEGER, v1 AS (a + 1) VIRTUAL, s1 AS (a * 2) VIRTUAL); + INSERT INTO g(a) VALUES (1), (2); + SELECT g1.a, g2.a FROM g g1, g g2 WHERE g1.v1 = g2.s1; +} +expect { + 1|1 +} + +# Issue #7069: a unique index on a generated column must use the +# column's declared affinity, not the expression's. With g INT AS (a), +# '1' and '01' both generate integer 1 and must collide. + +test gencol_issue7069_unique_index_uses_column_affinity { + CREATE TABLE t(a TEXT, g INT AS (a)); + CREATE UNIQUE INDEX i ON t(g); + INSERT INTO t(a) VALUES('1'); + INSERT INTO t(a) VALUES('01'); +} +expect error { + UNIQUE constraint failed: t.g +} + +test gencol_issue7069_create_index_detects_existing_duplicates { + CREATE TABLE t(a TEXT, g INT AS (a)); + INSERT INTO t(a) VALUES('1'); + INSERT INTO t(a) VALUES('01'); + CREATE UNIQUE INDEX i ON t(g); +} +expect error { + UNIQUE constraint failed: t.g +} + +test gencol_issue7069_index_lookup_integer_affinity { + CREATE TABLE t(a TEXT, g INT AS (a)); + CREATE INDEX i ON t(g); + INSERT INTO t(a) VALUES('1'),('2'),('10'); + SELECT a, g, typeof(g) FROM t WHERE g = 2; + SELECT a FROM t WHERE g > 1 ORDER BY g; +} +expect { + 2|2|integer + 2 + 10 +} + +# Issue #7070: FK through an explicit UNIQUE index on a generated +# column used to fail falsely when the gen-col affinity (INT) differed from +# its source column (TEXT). + +test gencol_issue7070_fk_gencol_affinity_match { + PRAGMA foreign_keys=ON; + CREATE TABLE p(a TEXT, g INT AS(a)); + CREATE UNIQUE INDEX pg ON p(g); + INSERT INTO p(a) VALUES('1'); + CREATE TABLE c(x REFERENCES p(g)); + INSERT INTO c VALUES(1); + SELECT x FROM c; +} +expect { + 1 +} + +test gencol_issue7070_fk_gencol_text_child_value { + PRAGMA foreign_keys=ON; + CREATE TABLE p(a TEXT, g INT AS(a)); + CREATE UNIQUE INDEX pg ON p(g); + INSERT INTO p(a) VALUES('1'); + CREATE TABLE c(x REFERENCES p(g)); + INSERT INTO c VALUES('1'); + SELECT typeof(x), x FROM c; + SELECT typeof(g), g FROM p; +} +expect { + text|1 + integer|1 +} + +test gencol_issue7070_fk_gencol_missing_parent_still_fails { + PRAGMA foreign_keys=ON; + CREATE TABLE p(a TEXT, g INT AS(a)); + CREATE UNIQUE INDEX pg ON p(g); + INSERT INTO p(a) VALUES('1'); + CREATE TABLE c(x REFERENCES p(g)); + INSERT INTO c VALUES(2); +} +expect error { + FOREIGN KEY constraint failed +} + +# Issue #6414: NOT NULL on a virtual generated column must be +# enforced during UPDATE, including when only a base column of the +# generating expression changes, and under OR REPLACE / OR IGNORE. + +test gencol_issue6414_update_base_col_null { + CREATE TABLE t(a INT, b INT, c INT GENERATED ALWAYS AS (a+b) VIRTUAL NOT NULL); + INSERT INTO t(a,b) VALUES(1,2); + UPDATE t SET a = NULL; +} +expect error { + NOT NULL constraint failed: t.c +} + +test gencol_issue6414_update_other_base_col_null { + CREATE TABLE t(a INT, b INT, c INT GENERATED ALWAYS AS (a+b) VIRTUAL NOT NULL); + INSERT INTO t(a,b) VALUES(1,2); + UPDATE t SET b = NULL; +} +expect error { + NOT NULL constraint failed: t.c +} + +test gencol_issue6414_update_or_replace_null { + CREATE TABLE t(a INT, b INT, c INT GENERATED ALWAYS AS (a+b) VIRTUAL NOT NULL); + INSERT INTO t(a,b) VALUES(1,2); + UPDATE OR REPLACE t SET a = NULL; +} +expect error { + NOT NULL constraint failed: t.c +} + +test gencol_issue6414_update_or_ignore_keeps_row { + CREATE TABLE t(a INT, b INT, c INT GENERATED ALWAYS AS (a+b) VIRTUAL NOT NULL); + INSERT INTO t(a,b) VALUES(1,2); + UPDATE OR IGNORE t SET a = NULL; + SELECT a,b,c FROM t; +} +expect { + 1|2|3 +} + +test gencol_issue6414_update_multi_col_null { + CREATE TABLE t(a INT, b INT, c INT GENERATED ALWAYS AS (a+b) VIRTUAL NOT NULL); + INSERT INTO t(a,b) VALUES(1,2); + UPDATE t SET a = NULL, b = 5; +} +expect error { + NOT NULL constraint failed: t.c +} + +# Issue #6155: the CREATE TABLE text stored in sqlite_master must keep +# the GENERATED ALWAYS keywords. Tested via LIKE so the test does not also +# depend on Turso's whitespace normalization of stored SQL. + +test gencol_issue6155_sqlite_master_keeps_generated_always { + CREATE TABLE t1(a INTEGER, b GENERATED ALWAYS AS (a*2) VIRTUAL); + SELECT sql LIKE '%GENERATED ALWAYS%' FROM sqlite_master WHERE name = 't1'; +} +expect { + 1 +} + +test gencol_issue6155_generated_always_alter_round_trip { + CREATE TABLE t1(a INTEGER, b GENERATED ALWAYS AS (a*2) VIRTUAL, c INTEGER); + SELECT sql LIKE '%GENERATED ALWAYS%' FROM sqlite_master WHERE name = 't1'; + ALTER TABLE t1 RENAME COLUMN a TO renamed; + SELECT sql LIKE '%GENERATED ALWAYS%' FROM sqlite_master WHERE name = 't1'; + ALTER TABLE t1 DROP COLUMN c; + SELECT sql LIKE '%GENERATED ALWAYS%' FROM sqlite_master WHERE name = 't1'; + ALTER TABLE t1 ADD COLUMN d INTEGER; + SELECT sql LIKE '%GENERATED ALWAYS%' FROM sqlite_master WHERE name = 't1'; +} +expect { + 1 + 1 + 1 + 1 +} + +test gencol_issue6155_add_generated_column_keeps_spelling { + CREATE TABLE t1(a INTEGER); + ALTER TABLE t1 ADD COLUMN b GENERATED ALWAYS AS (a*2) VIRTUAL; + SELECT sql LIKE '%b GENERATED ALWAYS AS%' FROM sqlite_master WHERE name = 't1'; + ALTER TABLE t1 ADD COLUMN c AS (a+1) VIRTUAL; + SELECT sql LIKE '%b GENERATED ALWAYS AS%', sql LIKE '%c AS%' + FROM sqlite_master WHERE name = 't1'; +} +expect { + 1 + 1|1 +} + +test gencol_issue6155_bare_as_alter_round_trip { + CREATE TABLE t1(a INTEGER, b AS (a*2) VIRTUAL, c INTEGER); + SELECT sql LIKE '%GENERATED ALWAYS%' FROM sqlite_master WHERE name = 't1'; + ALTER TABLE t1 RENAME COLUMN a TO renamed; + SELECT sql LIKE '%GENERATED ALWAYS%' FROM sqlite_master WHERE name = 't1'; + ALTER TABLE t1 DROP COLUMN c; + SELECT sql LIKE '%GENERATED ALWAYS%' FROM sqlite_master WHERE name = 't1'; + ALTER TABLE t1 ADD COLUMN d INTEGER; + SELECT sql LIKE '%GENERATED ALWAYS%' FROM sqlite_master WHERE name = 't1'; +} +expect { + 0 + 0 + 0 + 0 +} + +# Issue #7066: row-value comparisons in generated-column expressions. + +test gencol_issue7066_rowvalue_lt_column_ref { + CREATE TABLE t1(a, b AS ((a,1)<(a,2))); + INSERT INTO t1(a) VALUES(5); + SELECT * FROM t1; +} +expect { + 5|1 +} + +test gencol_issue7066_rowvalue_eq_column_ref { + CREATE TABLE t1(a, b AS ((a,1)=(5,1))); + INSERT INTO t1(a) VALUES(5); + INSERT INTO t1(a) VALUES(6); + SELECT * FROM t1; +} +expect { + 5|1 + 6|0 +} + +test gencol_issue7066_rowvalue_nocase_column_ref { + CREATE TABLE t(a TEXT COLLATE NOCASE, b AS ((a,1)=('X',1))); + INSERT INTO t(a) VALUES('x'); + SELECT * FROM t; +} +expect { + x|1 +} + +# An explicit COLLATE anywhere in the comparison beats the column's implicit +# collation, so BINARY wins over the NOCASE column here. +test gencol_issue7066_rowvalue_explicit_collate_beats_column { + CREATE TABLE t(a TEXT COLLATE NOCASE, b AS ((a,1)=('X' COLLATE BINARY,1))); + INSERT INTO t(a) VALUES('x'); + SELECT a, b FROM t; + SELECT count(*) FROM t WHERE (a,1)=('X' COLLATE BINARY,1); + SELECT count(*) FROM t WHERE (a,1)=('X',1); +} +expect { + x|0 + 0 + 1 +} + +test gencol_issue7066_rowvalue_no_column_ref { + CREATE TABLE t1(a, b AS ((1,2)<(1,3))); + INSERT INTO t1(a) VALUES(9); + SELECT * FROM t1; +} +expect { + 9|1 +} + +# Issue #7068: on REPLACE, a NULL written to a NOT NULL column with a +# DEFAULT is swapped for the default, but generated columns are computed from +# the original NULL and never recomputed before CHECK, UNIQUE indexing, and +# RETURNING. + +test gencol_issue7068_returning_recomputed_after_default { + CREATE TABLE t(a NOT NULL DEFAULT 5, b AS (a+1) UNIQUE); + REPLACE INTO t(a) VALUES(NULL) RETURNING a, b; +} +expect { + 5|6 +} + +test gencol_issue7068_unique_index_uses_recomputed_value { + CREATE TABLE t(a NOT NULL DEFAULT 5, b AS (a+1) UNIQUE); + REPLACE INTO t(a) VALUES(NULL); + SELECT rowid, a, b FROM t; + PRAGMA integrity_check; +} +expect { + 1|5|6 + ok +} + +test gencol_issue7068_check_sees_recomputed_value { + CREATE TABLE t(a NOT NULL DEFAULT 5, b AS (a+1) CHECK(b IS 6)); + REPLACE INTO t(a) VALUES(NULL); + SELECT a, b FROM t; +} +expect { + 5|6 +} + +test gencol_issue7068_multiple_defaults_before_generated_notnull { + CREATE TABLE t(b AS(a+c) NOT NULL UNIQUE CHECK(b=12), a NOT NULL DEFAULT 5, c NOT NULL DEFAULT 7); + INSERT OR REPLACE INTO t(a,c) VALUES(NULL,NULL) RETURNING a,c,b; + PRAGMA integrity_check; +} +expect { + 5|7|12 + ok +} + +# Issue #7072: an upsert conflict target must match an explicit +# UNIQUE index on a generated column. + +test gencol_issue7072_upsert_do_update_on_gencol_index { + CREATE TABLE t(a, b AS (a*2)); + CREATE UNIQUE INDEX idx_b ON t(b); + INSERT INTO t(a) VALUES(1); + INSERT INTO t(a) VALUES(1) ON CONFLICT(b) DO UPDATE SET a=2; + SELECT rowid, a, b FROM t; +} +expect { + 1|2|4 +} + +test gencol_issue7072_upsert_do_nothing_on_gencol_index { + CREATE TABLE t(a, b AS (a*2)); + CREATE UNIQUE INDEX idx_b ON t(b); + INSERT INTO t(a) VALUES(1); + INSERT INTO t(a) VALUES(1) ON CONFLICT(b) DO NOTHING; + SELECT rowid, a, b FROM t; +} +expect { + 1|1|2 +} + +test gencol_issue7072_upsert_expression_does_not_match_gencol_index { + CREATE TABLE t(a, b AS (a*2)); + CREATE UNIQUE INDEX idx_b ON t(b); + INSERT INTO t(a) VALUES(1); + INSERT INTO t(a) VALUES(1) ON CONFLICT(a*2) DO NOTHING; +} +expect error { + ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint +} + +# Issue #6161: STRICT tables must reject a computed virtual generated-column +# value that cannot be losslessly stored under the column's declared type. +# SQLite added this enforcement in 3.51 (verified: 3.51.1 rejects, 3.50.4 +# does not). We believe enforcing is the correct behavior and follow the +# newer SQLite here. The error-expecting tests below were confirmed to pass +# against sqlite3 3.51.1 locally; they carry @skip-if sqlite only because CI +# pins its sqlite oracle to 3.50.4, the version turso itself reports. + +@skip-if sqlite "SQLite only enforces STRICT declared types of virtual generated columns from 3.51; the CI runner binary is older" +test gencol_issue6161_strict_insert_text_into_int_gencol { + CREATE TABLE t1(a TEXT, b INTEGER GENERATED ALWAYS AS (a) VIRTUAL) STRICT; + INSERT INTO t1(a) VALUES('abc'); +} +expect error { + cannot store TEXT value in INTEGER column t1.b +} + +@skip-if sqlite "SQLite only enforces STRICT declared types of virtual generated columns from 3.51; the CI runner binary is older" +test gencol_issue6161_strict_update_text_into_int_gencol { + CREATE TABLE t2(a TEXT, b INTEGER GENERATED ALWAYS AS (a) VIRTUAL) STRICT; + INSERT INTO t2(a) VALUES('42'); + UPDATE t2 SET a='abc'; +} +expect error { + cannot store TEXT value in INTEGER column t2.b +} + +@skip-if sqlite "SQLite only enforces STRICT declared types of virtual generated columns from 3.51; the CI runner binary is older" +test gencol_issue6161_strict_insert_blob_into_text_gencol { + CREATE TABLE t3(a BLOB, b TEXT GENERATED ALWAYS AS (a) VIRTUAL) STRICT; + INSERT INTO t3(a) VALUES(x'6162'); +} +expect error { + cannot store BLOB value in TEXT column t3.b +} + +@skip-if sqlite "SQLite only enforces STRICT declared types of virtual generated columns from 3.51; the CI runner binary is older" +test gencol_issue6161_strict_upsert_text_into_int_gencol { + CREATE TABLE t(id INTEGER PRIMARY KEY, a TEXT, b INT AS(a)) STRICT; + INSERT INTO t VALUES(1, '42'); + INSERT INTO t VALUES(1, '43') ON CONFLICT(id) DO UPDATE SET a='abc'; +} +expect error { + cannot store TEXT value in INT column t.b +} + +test gencol_issue6161_strict_coercion_all_write_paths { + CREATE TABLE t(id INTEGER PRIMARY KEY, b INTEGER AS(a), a TEXT, c ANY AS(a), d TEXT AS(b)) STRICT; + INSERT INTO t(id,a) VALUES(1,'42'),(2,NULL); + SELECT id,b,typeof(b),c,typeof(c),d,typeof(d) FROM t ORDER BY id; + UPDATE t SET a='43' WHERE id=1; + SELECT b,typeof(b),c,typeof(c),d,typeof(d) FROM t WHERE id=1; + INSERT INTO t(id,a) VALUES(1,'44') ON CONFLICT(id) DO UPDATE SET a=excluded.a; + SELECT b,typeof(b),c,typeof(c),d,typeof(d) FROM t WHERE id=1; +} +expect { + 1|42|integer|42|text|42|text + 2||null||null||null + 43|integer|43|text|43|text + 44|integer|44|text|44|text +} + +test gencol_issue6161_strict_compute_after_base_affinity { + CREATE TABLE t(id INTEGER PRIMARY KEY, a INTEGER, + b INT AS(CASE typeof(a) WHEN 'integer' THEN 1 ELSE 'bad' END)) STRICT; + INSERT INTO t VALUES(1,'42'); + UPDATE t SET a='43'; + INSERT INTO t VALUES(1,'44') ON CONFLICT(id) DO UPDATE SET a='45'; + UPDATE t SET id=2; + SELECT * FROM t; +} +expect { + 2|45|1 +} + +setup strict_update_generated_columns { + CREATE TABLE t(a INTEGER, b INTEGER AS(a + 1), c INTEGER, CHECK(b > 0)) STRICT; + INSERT INTO t VALUES(1, 0); +} + +@skip-if mvcc "mvcc has different root pages" +@setup strict_update_generated_columns +snapshot strict_update_reuses_generated_columns_for_checks { + UPDATE t SET c = '42'; +} + +@setup strict_update_generated_columns +test strict_update_generated_columns_values { + UPDATE t SET c = '42'; + SELECT a, b, c FROM t; +} +expect { + 1|2|42 +} + +test gencol_strict_update_checks_input_before_primary_key_conflict { + CREATE TABLE t(id INTEGER PRIMARY KEY, a INT, b INT AS(a + 1)) STRICT; + INSERT INTO t VALUES(1, 1), (2, 2); + UPDATE OR IGNORE t SET id=2, a='bad' WHERE id=1; +} +expect error { + cannot store TEXT value in INT column t.a +} + +test gencol_issue6161_strict_compute_after_replace_default { + CREATE TABLE t(a TEXT NOT NULL ON CONFLICT REPLACE DEFAULT '42', b INT AS(a)) STRICT; + INSERT INTO t VALUES(NULL); + SELECT a,b,typeof(b) FROM t; +} +expect { + 42|42|integer +} + +test gencol_issue6161_strict_any_exempt { + CREATE TABLE t(id INTEGER PRIMARY KEY, a ANY, b ANY AS(a)) STRICT; + INSERT INTO t VALUES(1,'abc'); + SELECT b,typeof(b) FROM t; + UPDATE t SET a=x'6162'; + SELECT hex(b),typeof(b) FROM t; + INSERT INTO t VALUES(1,NULL) ON CONFLICT(id) DO UPDATE SET a=excluded.a; + SELECT typeof(b) FROM t; +} +expect { + abc|text + 6162|blob + null +} + +test gencol_issue6161_non_strict_all_write_paths { + CREATE TABLE t(id INTEGER PRIMARY KEY, a TEXT, b INTEGER AS(a)); + INSERT INTO t VALUES(1,'abc'); + SELECT b,typeof(b) FROM t; + UPDATE t SET a='def'; + SELECT b,typeof(b) FROM t; + INSERT INTO t VALUES(1,'ghi') ON CONFLICT(id) DO UPDATE SET a=excluded.a; + SELECT b,typeof(b) FROM t; +} +expect { + abc|text + def|text + ghi|text +} + +@cross-check-integrity +test gencol_issue6161_strict_any_gencol_via_alter { + CREATE TABLE al(id INTEGER PRIMARY KEY, a TEXT) STRICT; + INSERT INTO al VALUES(1,'42'); + ALTER TABLE al ADD COLUMN c ANY AS (a); + SELECT id, c, typeof(c) FROM al; + CREATE INDEX al_c ON al(c); + PRAGMA integrity_check; +} +expect { + 1|42|text + ok +} + +# Issues #6161/#7071: the INSERT-side rejection below matches SQLite >= 3.51 +# (confirmed passing against 3.51.1 locally; skipped against CI's pinned +# 3.50.4, which accepts the row). The integrity_check facet is covered by +# crafted-database-file tests in tests/integration/integrity_check.rs. + +@skip-if sqlite "SQLite only enforces STRICT declared types of virtual generated columns from 3.51; the CI runner binary is older" +test gencol_issue7071_strict_gencol_violation_rejected { + CREATE TABLE t(a ANY, b INT AS(a) NOT NULL) STRICT; + INSERT INTO t(a) VALUES('abc'); +} +expect error { + cannot store TEXT value in INT column t.b +} + +# Issue #7058: ALTER TABLE ADD COLUMN must reject a column that has +# both DEFAULT and a GENERATED clause. + +test gencol_issue7058_alter_add_default_on_gencol { + CREATE TABLE t(a); + ALTER TABLE t ADD COLUMN b DEFAULT 5 AS (a+1); +} +expect error { + error in generated column "b" +} + +# Issue #7059: an unknown type token after a generated-column clause +# must be a parse error, not silently dropped from the schema. + +test gencol_issue7059_unknown_type_token_rejected { + CREATE TABLE t(a, b AS (a) WAT); +} +expect error { + error in generated column "b" +} + +# Issue #7060: a column with more than one generated clause must be +# rejected. + +test gencol_issue7060_multiple_generated_clauses_rejected { + CREATE TABLE t(a, b, c AS (a+1) AS (a+2)); +} +expect error { + error in generated column "c" +} + +test gencol_issue6161_any_affinity_strict_and_non_strict { + CREATE TABLE s(id INTEGER PRIMARY KEY, a ANY, g aNy AS(a), kind TEXT AS(typeof(g))) STRICT; + INSERT INTO s(id,a) VALUES(1,'00042'),(2,42),(3,42.5),(4,x'3432'),(5,NULL),(6,'abc'); + SELECT id,quote(g),typeof(g),kind FROM s ORDER BY id; + CREATE TABLE t(id INTEGER PRIMARY KEY, a BLOB, g aNy AS(a), kind TEXT AS(typeof(g))); + INSERT INTO t(id,a) SELECT id,a FROM s; + SELECT id,quote(g),typeof(g),kind FROM t ORDER BY id; +} +expect { + 1|'00042'|text|text + 2|42|integer|integer + 3|42.5|real|real + 4|X'3432'|blob|blob + 5|NULL|null|null + 6|'abc'|text|text + 1|42|integer|integer + 2|42|integer|integer + 3|42.5|real|real + 4|X'3432'|blob|blob + 5|NULL|null|null + 6|'abc'|text|text +} + +@cross-check-integrity +test gencol_issue6161_strict_any_index_all_write_paths { + CREATE TABLE t(id INTEGER PRIMARY KEY, a ANY, g ANY AS(a)) STRICT; + INSERT INTO t(id,a) VALUES(1,'00042'),(2,42),(3,42.5),(4,x'3432'),(5,NULL); + CREATE UNIQUE INDEX t_g ON t(g); + INSERT INTO t(id,a) VALUES(6,'42'); + SELECT id,quote(g),typeof(g) FROM t INDEXED BY t_g WHERE g='00042'; + SELECT id,quote(g),typeof(g) FROM t INDEXED BY t_g WHERE g=42; + SELECT id,quote(g),typeof(g) FROM t INDEXED BY t_g WHERE g='42'; + UPDATE t SET a='00043' WHERE g='00042'; + INSERT INTO t(id,a) VALUES(2,'00044') ON CONFLICT(id) DO UPDATE SET a=excluded.a; + DELETE FROM t WHERE g=x'3432'; + SELECT id,quote(g),typeof(g) FROM t INDEXED BY t_g ORDER BY id; + SELECT count(*) FROM t INDEXED BY t_g WHERE g IN ('00042',42,x'3432'); + PRAGMA integrity_check; +} +expect { + 1|'00042'|text + 2|42|integer + 6|'42'|text + 1|'00043'|text + 2|'00044'|text + 3|42.5|real + 5|NULL|null + 6|'42'|text + 0 + ok +} diff --git a/conformance/sqlite-sqltests/generate_series.sqltest b/conformance/sqlite-sqltests/generate_series.sqltest index 160c5253..0f14ee1d 100644 --- a/conformance/sqlite-sqltests/generate_series.sqltest +++ b/conformance/sqlite-sqltests/generate_series.sqltest @@ -35,6 +35,17 @@ expect { 10 } +test generate-series-null-arguments-return-no-rows { + SELECT count(*) FROM generate_series(NULL, 3); + SELECT count(*) FROM generate_series(1, NULL); + SELECT count(*) FROM generate_series(1, 3, NULL); +} +expect { + 0 + 0 + 0 +} + test generate-series-explicit-negative-stop { SELECT count(*) FROM generate_series(1, -1); } diff --git a/conformance/sqlite-sqltests/groupby/memory.sqltest b/conformance/sqlite-sqltests/groupby/memory.sqltest index 0a088265..c4650047 100644 --- a/conformance/sqlite-sqltests/groupby/memory.sqltest +++ b/conformance/sqlite-sqltests/groupby/memory.sqltest @@ -1,5 +1,108 @@ @database :memory: +setup group_key_expression { + CREATE TABLE t(x TEXT, z INT); + INSERT INTO t VALUES ('C', 1), ('A', 2), ('B', 3); +} + +@setup group_key_expression +@cross-check-integrity +test group-by-key-order-asc { + SELECT min(x) FROM t GROUP BY lower(x) ORDER BY lower(x), min(z); +} +expect { + A + B + C +} + +@setup group_key_expression +@cross-check-integrity +test group-by-key-order-desc { + SELECT min(x) FROM t GROUP BY lower(x) ORDER BY lower(x) DESC, min(z); +} +expect { + C + B + A +} + +@setup group_key_expression +@cross-check-integrity +test group-by-key-collate-limit { + SELECT min(x) FROM t + GROUP BY x COLLATE NOCASE ORDER BY x COLLATE NOCASE, min(z) LIMIT 1; +} +expect { + A +} + +@setup group_key_expression +@cross-check-integrity +test group-by-key-nested-order { + SELECT min(x) FROM t GROUP BY lower(x) ORDER BY lower(x) || '', min(z); +} +expect { + A + B + C +} + +@setup group_key_expression +@cross-check-integrity +test group-by-key-nested-result { + SELECT lower(x) || '' FROM t GROUP BY lower(x) ORDER BY min(z); +} +expect { + c + a + b +} + +@setup group_key_expression +@cross-check-integrity +test group-by-key-having { + SELECT min(x) FROM t GROUP BY lower(x) HAVING lower(x) = 'a' ORDER BY min(z); +} +expect { + A +} + +@setup group_key_expression +@cross-check-integrity +test group-by-key-expression-index { + CREATE INDEX t_lower ON t(lower(x)); + SELECT min(x) FROM t INDEXED BY t_lower + GROUP BY lower(x) ORDER BY lower(x), min(z); +} +expect { + A + B + C +} + +@setup group_key_expression +@cross-check-integrity +test group-by-key-bare-column { + SELECT min(x) FROM t GROUP BY x ORDER BY x, min(z); +} +expect { + A + B + C +} + +@setup group_key_expression +@cross-check-integrity +test group-by-key-single-order-term { + SELECT min(x) FROM t GROUP BY lower(x) ORDER BY lower(x) DESC; +} +expect { + C + B + A +} + @cross-check-integrity test group_by_no_sorting_required_reordered_columns { create table t0 (a INT, b INT, c INT); diff --git a/conformance/sqlite-sqltests/hash-join-unmatched-row-filters.sqltest b/conformance/sqlite-sqltests/hash-join-unmatched-row-filters.sqltest new file mode 100644 index 00000000..49797550 --- /dev/null +++ b/conformance/sqlite-sqltests/hash-join-unmatched-row-filters.sqltest @@ -0,0 +1,87 @@ +@database :memory: + +# When a LEFT JOIN runs as a hash join, the unmatched build rows (the ones +# that get NULLs for the probe table) are emitted by a separate scan after the +# probe loop. That scan must still apply WHERE filters that reference the +# probe table, including filters that came from a later join's ON clause and +# became plain WHERE terms when that join was rewritten to an inner join. +# The bug: the unmatched scan skipped a filter that also referenced the build +# table of an earlier hash join, because that table's values live in payload +# registers rather than a positioned cursor. The null-extended rows then +# bypassed the filter entirely. +# +# All queries here need: no indexes (so the planner picks hash joins), a LEFT +# JOIN whose ON clause uses IS (so it cannot itself be rewritten to inner), +# and a WHERE term that lets the next LEFT JOIN be rewritten to inner. + +test hash-join-null-extended-row-must-not-skip-is-filter { + CREATE TABLE t1(id INTEGER PRIMARY KEY, a INT, c INT); + CREATE TABLE t2(id INTEGER PRIMARY KEY, a INT, c INT); + CREATE TABLE t3(id INTEGER PRIMARY KEY, c INT, d INT); + CREATE TABLE t4(id INTEGER PRIMARY KEY, c INT, d INT); + INSERT INTO t1 VALUES (1, 1, 1); + INSERT INTO t3 VALUES (1, 5, 5); + INSERT INTO t4 VALUES (1, 5, 5); + -- t1's row has no t2 match, so t2 is null-extended and `t2.c IS t3.c` + -- becomes `NULL IS 5`, which is false: no row may come out. + SELECT count(*) FROM t1 + LEFT JOIN t2 ON t1.a = t2.a + LEFT JOIN t3 ON t2.c IS t3.c + JOIN t4 ON t3.c = t4.c AND t3.d = t4.d + WHERE t3.d IS NOT NULL; +} +expect { + 0 +} + +test hash-join-filters-null-extended-rows-but-keeps-matched-rows { + CREATE TABLE t1(id INTEGER PRIMARY KEY, a INT, c INT); + CREATE TABLE t2(id INTEGER PRIMARY KEY, a INT, c INT); + CREATE TABLE t3(id INTEGER PRIMARY KEY, c INT, d INT); + CREATE TABLE t4(id INTEGER PRIMARY KEY, c INT, d INT); + INSERT INTO t1 VALUES (1, 1, 1); + INSERT INTO t1 VALUES (2, 2, 2); + INSERT INTO t2 VALUES (10, 2, 7); + INSERT INTO t3 VALUES (1, 5, 5); + INSERT INTO t3 VALUES (2, NULL, 6); + INSERT INTO t3 VALUES (3, 7, 8); + INSERT INTO t4 VALUES (1, 5, 5); + INSERT INTO t4 VALUES (2, 7, 8); + -- t1.id=2 matches t2.id=10 and its t2.c=7 pairs with t3.id=3/t4.id=2. + -- t1.id=1 has no t2 match and must be filtered out by `t2.c IS t3.c`. + SELECT t1.id, t2.id, t3.id, t4.id FROM t1 + LEFT JOIN t2 ON t1.a = t2.a + LEFT JOIN t3 ON t2.c IS t3.c + JOIN t4 ON t3.c = t4.c AND t3.d = t4.d + WHERE t3.d IS NOT NULL ORDER BY 1, 2, 3, 4; +} +expect { + 2|10|3|2 +} + +test hash-join-null-extended-row-still-matches-is-null-partner { + CREATE TABLE t1(id INTEGER PRIMARY KEY, a INT, c INT); + CREATE TABLE t2(id INTEGER PRIMARY KEY, a INT, c INT); + CREATE TABLE t3(id INTEGER PRIMARY KEY, c INT, d INT); + CREATE TABLE t4(id INTEGER PRIMARY KEY, c INT, d INT); + INSERT INTO t1 VALUES (1, 1, 1); + INSERT INTO t1 VALUES (2, 2, 2); + INSERT INTO t2 VALUES (10, 2, 7); + INSERT INTO t3 VALUES (1, 5, 5); + INSERT INTO t3 VALUES (2, NULL, 6); + INSERT INTO t3 VALUES (3, 7, 8); + INSERT INTO t4 VALUES (1, 5, 5); + INSERT INTO t4 VALUES (2, 7, 8); + INSERT INTO t4 VALUES (3, 9, 6); + -- The other polarity: the null-extended t2 row must still pair with the + -- t3.c IS NULL row, because `NULL IS NULL` is true. + SELECT t1.id, t2.id, t3.id, t4.id FROM t1 + LEFT JOIN t2 ON t1.a = t2.a + LEFT JOIN t3 ON t2.c IS t3.c + JOIN t4 ON t3.d = t4.d + WHERE t3.d IS NOT NULL ORDER BY 1, 2, 3, 4; +} +expect { + 1||2|3 + 2|10|3|2 +} diff --git a/conformance/sqlite-sqltests/index-seek-null-key.sqltest b/conformance/sqlite-sqltests/index-seek-null-key.sqltest new file mode 100644 index 00000000..f59bbe07 --- /dev/null +++ b/conformance/sqlite-sqltests/index-seek-null-key.sqltest @@ -0,0 +1,76 @@ +@database :temp: + +# Regression tests for NULL-key index seek (issue #8690). +# Expr::is_nonnull() must return false for JSON arrow operators (-> / ->>) +# and for LIKE with a nullable ESCAPE clause, because these expressions can +# evaluate to NULL even when their operands are non-NULL. If the optimizer +# believes the seek key is non-NULL, the IsNull guard is skipped and the +# seek starts at a NULL key, reading the full index: SELECT returns extra +# rows and DELETE erases all rows. + +setup schema { + CREATE TABLE t3(c); + CREATE INDEX i3 ON t3(c); + INSERT INTO t3 VALUES (1),(2),(3); +} + +@setup schema +test json-arrow-shift-missing-path-select { + SELECT count(*) FROM t3 WHERE c > ('{"k":1}' ->> '$.missing'); +} +expect { + 0 +} + +@setup schema +test json-arrow-shift-missing-path-delete { + DELETE FROM t3 WHERE c > ('{"k":1}' ->> '$.missing'); + SELECT count(*) FROM t3; +} +expect { + 3 +} + +@setup schema +test json-arrow-missing-path-select { + SELECT count(*) FROM t3 WHERE c > ('{"k":1}' -> '$.missing'); +} +expect { + 0 +} + +@setup schema +test like-escape-null-select { + SELECT count(*) FROM t3 WHERE c > ('a' LIKE 'a' ESCAPE NULL); +} +expect { + 0 +} + +@setup schema +test json-arrow-existing-path-still-uses-index { + SELECT count(*) FROM t3 WHERE c > ('{"k":2}' ->> '$.k'); +} +expect { + 1 +} + +@skip-if sqlite "@> isn't supported by sqlite" +@requires custom_types "array operators require --experimental-custom-types" +@setup schema +test array-contains-non-array-operand-select { + SELECT count(*) FROM t3 WHERE c > (5 @> 5); +} +expect { + 0 +} + +@skip-if sqlite "&& isn't supported by sqlite" +@requires custom_types "array operators require --experimental-custom-types" +@setup schema +test array-overlap-non-array-operand-select { + SELECT count(*) FROM t3 WHERE c > (5 && 5); +} +expect { + 0 +} diff --git a/conformance/sqlite-sqltests/index_seek_comparison_affinity.sqltest b/conformance/sqlite-sqltests/index_seek_comparison_affinity.sqltest index 25a9e48c..79b91f1b 100644 --- a/conformance/sqlite-sqltests/index_seek_comparison_affinity.sqltest +++ b/conformance/sqlite-sqltests/index_seek_comparison_affinity.sqltest @@ -163,3 +163,18 @@ test issue-6715-real-affinity-seek-computed-expr { expect { 0|1 } + +# Indexed REAL col's IN seek mustn't round rhs integers past 2^53. +setup real-affinity-in-list-schema { + CREATE TABLE t(x REAL); + INSERT INTO t VALUES(9007199254740992); + CREATE INDEX t_x ON t(x); +} + +@setup real-affinity-in-list-schema +test real-affinity-in-list-preserves-integer { + SELECT count(*) FROM t WHERE x IN (9007199254740993); +} +expect { +0 +} diff --git a/conformance/sqlite-sqltests/int64-overflow-seek.sqltest b/conformance/sqlite-sqltests/int64-overflow-seek.sqltest index 2b6b989a..f57cfe6b 100644 --- a/conformance/sqlite-sqltests/int64-overflow-seek.sqltest +++ b/conformance/sqlite-sqltests/int64-overflow-seek.sqltest @@ -66,3 +66,35 @@ test int64-min-lt { } expect { } + +@setup schema +test rowid-seek-directions-at-matches-gaps-and-int64-limits { + INSERT INTO v0 VALUES (-10), (10); + WITH probes(k) AS ( + VALUES (-9223372036854775808), (-9223372036854775807), + (-11), (-10), (-9), (-1), (0), (1), (9), (10), (11), + (9223372036854775806), (9223372036854775807) + ) + SELECT k, + (SELECT v1 FROM v0 WHERE v1 = k), + (SELECT v1 FROM v0 WHERE v1 > k ORDER BY v1 LIMIT 1), + (SELECT v1 FROM v0 WHERE v1 >= k ORDER BY v1 LIMIT 1), + (SELECT v1 FROM v0 WHERE v1 < k ORDER BY v1 DESC LIMIT 1), + (SELECT v1 FROM v0 WHERE v1 <= k ORDER BY v1 DESC LIMIT 1) + FROM probes ORDER BY k; +} +expect { + -9223372036854775808|-9223372036854775808|-10|-9223372036854775808||-9223372036854775808 + -9223372036854775807||-10|-10|-9223372036854775808|-9223372036854775808 + -11||-10|-10|-9223372036854775808|-9223372036854775808 + -10|-10|0|-10|-9223372036854775808|-10 + -9||0|0|-10|-10 + -1||0|0|-10|-10 + 0|0|10|0|-10|0 + 1||10|10|0|0 + 9||10|10|0|0 + 10|10|9223372036854775807|10|0|10 + 11||9223372036854775807|9223372036854775807|10|10 + 9223372036854775806||9223372036854775807|9223372036854775807|10|10 + 9223372036854775807|9223372036854775807||9223372036854775807|10|9223372036854775807 +} diff --git a/conformance/sqlite-sqltests/integrity_check/memory.sqltest b/conformance/sqlite-sqltests/integrity_check/memory.sqltest index 366d5312..4b25a69f 100644 --- a/conformance/sqlite-sqltests/integrity_check/memory.sqltest +++ b/conformance/sqlite-sqltests/integrity_check/memory.sqltest @@ -10,6 +10,32 @@ expect { ok } + +test integrity-check-qualified-empty-attached-database { + CREATE TABLE main.t(a UNIQUE); + ATTACH ':memory:' AS aux; + PRAGMA aux.user_version = 1; + PRAGMA main.page_count; + PRAGMA aux.page_count; + PRAGMA aux.integrity_check; + PRAGMA aux.quick_check; +} +expect { + 3 + 1 + ok + ok +} + +test integrity-check-qualified-corrupt-attached-database { + ATTACH 'database/integrity_missing_index_entry.db' AS corrupt_aux; + PRAGMA corrupt_aux.integrity_check; +} +expect unordered { + wrong # of entries in index idx_b + row 3 missing from index idx_b +} + # Test basic quick_check on empty database test quick-check-empty { PRAGMA quick_check; @@ -214,3 +240,82 @@ expect { ok } +# integrity_check must validate the declared types of STRICT table columns. +# A healthy STRICT table reports "ok" for every storage class. +@cross-check-integrity +test integrity-check-strict-stored-types { + CREATE TABLE t11(i INT, n INTEGER, r REAL, tx TEXT, b BLOB, a ANY) STRICT; + INSERT INTO t11 VALUES(1, 2, 1.5, 'x', x'0102', 'any'); + INSERT INTO t11 VALUES(NULL, NULL, NULL, NULL, NULL, NULL); + SELECT typeof(i),typeof(n),typeof(r),typeof(tx),typeof(b),typeof(a) FROM t11; + PRAGMA integrity_check; + PRAGMA quick_check; +} +expect { + integer|integer|real|text|blob|text + null|null|null|null|null|null + ok + ok +} + +# A REAL column can hold a value with an integer storage class, because the +# file format stores integral reals as integers. This must stay "ok". +@cross-check-integrity +test integrity-check-strict-real-holds-integer { + CREATE TABLE t12(r REAL NOT NULL) STRICT; + INSERT INTO t12 VALUES(42); + SELECT typeof(r), r FROM t12; + PRAGMA integrity_check; +} +expect { + real|42.0 + ok +} +# JS Number drops the .0 suffix on whole floats +expect @js { + real|42 + ok +} + +# Virtual generated columns get the affinity of their declared type before the +# type check, so a TEXT base column that holds '42' is valid for an INT column. +@cross-check-integrity +test integrity-check-strict-virtual-generated-columns { + CREATE TABLE t13(a TEXT, b INT AS(a), c REAL AS(a), d ANY AS(a), e TEXT AS(a||'!')) STRICT; + INSERT INTO t13(a) VALUES('42'); + SELECT typeof(b),typeof(c),typeof(d),typeof(e) FROM t13; + PRAGMA integrity_check; + PRAGMA quick_check; +} +expect { + integer|real|text|text + ok + ok +} + +# NOT NULL and the STRICT type check apply to the same column together. +@cross-check-integrity +test integrity-check-strict-not-null-columns { + CREATE TABLE t14(id INTEGER PRIMARY KEY, b TEXT NOT NULL, c INT NOT NULL AS(length(b))) STRICT; + INSERT INTO t14(id,b) VALUES(1,'abc'); + SELECT c FROM t14; + PRAGMA integrity_check; +} +expect { + 3 + ok +} + +# Tables that are not STRICT keep no declared-type guarantee, so a TEXT value +# in an INT column is not an integrity error. +@cross-check-integrity +test integrity-check-non-strict-type-mismatch-is-ok { + CREATE TABLE t15(a INT, b INTEGER NOT NULL); + INSERT INTO t15 VALUES('abc', 1); + SELECT typeof(a) FROM t15; + PRAGMA integrity_check; +} +expect { + text + ok +} diff --git a/conformance/sqlite-sqltests/integrity_check/parity_strict_gencol_type_violation.sqltest b/conformance/sqlite-sqltests/integrity_check/parity_strict_gencol_type_violation.sqltest new file mode 100644 index 00000000..67422e20 --- /dev/null +++ b/conformance/sqlite-sqltests/integrity_check/parity_strict_gencol_type_violation.sqltest @@ -0,0 +1,20 @@ +# The fixture holds a STRICT table whose virtual generated INT column computes +# a TEXT value, because the base column stores an empty string. + +@database database/integrity_strict_gencol_type_violation.db readonly + +@skip-file-if mvcc "not supported in MVCC mode" + +test integrity-check-strict-gencol-type-violation { + PRAGMA integrity_check; +} +expect { + non-INT value in t.b +} + +test quick-check-strict-gencol-type-violation { + PRAGMA quick_check; +} +expect { + non-INT value in t.b +} diff --git a/conformance/sqlite-sqltests/integrity_check/parity_strict_not_null_violation.sqltest b/conformance/sqlite-sqltests/integrity_check/parity_strict_not_null_violation.sqltest new file mode 100644 index 00000000..df6f77dd --- /dev/null +++ b/conformance/sqlite-sqltests/integrity_check/parity_strict_not_null_violation.sqltest @@ -0,0 +1,20 @@ +# The fixture holds a STRICT table whose NOT NULL TEXT column stores NULL. +# Only the NULL error is reported, because NULL is valid for every type. + +@database database/integrity_strict_not_null_violation.db readonly + +@skip-file-if mvcc "not supported in MVCC mode" + +test integrity-check-strict-not-null-violation { + PRAGMA integrity_check; +} +expect { + NULL value in t.b +} + +test quick-check-strict-not-null-violation { + PRAGMA quick_check; +} +expect { + NULL value in t.b +} diff --git a/conformance/sqlite-sqltests/integrity_check/parity_strict_real_integer_serial.sqltest b/conformance/sqlite-sqltests/integrity_check/parity_strict_real_integer_serial.sqltest new file mode 100644 index 00000000..eaff8d74 --- /dev/null +++ b/conformance/sqlite-sqltests/integrity_check/parity_strict_real_integer_serial.sqltest @@ -0,0 +1,21 @@ +# The fixture holds a STRICT table whose REAL column stores a value with an +# integer storage class. SQLite writes integral reals this way, so the database +# is valid and both engines must report "ok". + +@database database/integrity_strict_real_integer_serial.db readonly + +@skip-file-if mvcc "not supported in MVCC mode" + +test integrity-check-strict-real-integer-serial { + PRAGMA integrity_check; +} +expect { + ok +} + +test quick-check-strict-real-integer-serial { + PRAGMA quick_check; +} +expect { + ok +} diff --git a/conformance/sqlite-sqltests/integrity_check/parity_strict_type_violation.sqltest b/conformance/sqlite-sqltests/integrity_check/parity_strict_type_violation.sqltest new file mode 100644 index 00000000..83d44317 --- /dev/null +++ b/conformance/sqlite-sqltests/integrity_check/parity_strict_type_violation.sqltest @@ -0,0 +1,19 @@ +# The fixture holds a STRICT table whose TEXT column stores an integer. + +@database database/integrity_strict_type_violation.db readonly + +@skip-file-if mvcc "not supported in MVCC mode" + +test integrity-check-strict-type-violation { + PRAGMA integrity_check; +} +expect { + non-TEXT value in t.b +} + +test quick-check-strict-type-violation { + PRAGMA quick_check; +} +expect { + non-TEXT value in t.b +} diff --git a/conformance/sqlite-sqltests/join/hash.sqltest b/conformance/sqlite-sqltests/join/hash.sqltest index 1abd9574..c030b842 100644 --- a/conformance/sqlite-sqltests/join/hash.sqltest +++ b/conformance/sqlite-sqltests/join/hash.sqltest @@ -275,6 +275,41 @@ expect { 900|600|600 } +@cross-check-integrity +test grace-full-outer-does-not-emit-deferred-probe-rows { + PRAGMA temp_store=FILE; + CREATE TABLE t1(a INTEGER, payload TEXT); + CREATE TABLE t2(b INTEGER, payload TEXT); + INSERT INTO t1 + SELECT value, printf('%080d', value) FROM generate_series(1, 1500); + INSERT INTO t2 + SELECT value, 'right-' || value FROM generate_series(1, 1500); + SELECT count(*), count(t1.a), count(t2.b), + sum(t1.a IS NULL), sum(t2.b IS NULL) + FROM t1 FULL OUTER JOIN t2 ON t1.a = t2.b; +} +expect { + 1500|1500|1500|0|0 +} + +@cross-check-integrity +test grace-full-outer-materialized-prefix-does-not-emit-deferred-probe-rows { + PRAGMA temp_store=FILE; + CREATE TABLE a(id INTEGER PRIMARY KEY, k INTEGER, k2 INTEGER); + CREATE TABLE b(id INTEGER PRIMARY KEY, k INTEGER, k2 INTEGER); + CREATE TABLE c(id INTEGER PRIMARY KEY, k INTEGER, k2 INTEGER); + INSERT INTO a SELECT value, value % 10, value % 7 FROM generate_series(1, 60); + INSERT INTO b SELECT value, value % 10, value % 7 FROM generate_series(1, 70); + INSERT INTO c SELECT value, value % 10, value % 7 FROM generate_series(1, 80); + SELECT count(*), count(a.id), count(b.id), count(c.id), + sum(a.id IS NULL), sum(b.id IS NULL), sum(c.id IS NULL) + FROM a JOIN b ON a.k = b.k + FULL JOIN c ON b.k2 = c.k2; +} +expect { + 4800|4800|4800|4800|0|0|0 +} + # ============================================================================= # 8. Aggregation correctness over grace hash joins # ============================================================================= @@ -838,3 +873,88 @@ test grace-complex-on-and-where { expect { 400 } + +# Compare Turso with SQLite across join types, join keys, filters, and limits. +setup hash-join-matrix-rows { + CREATE TABLE matrix_left(id INTEGER PRIMARY KEY, k, k2, flag, payload); + CREATE TABLE matrix_right(id INTEGER PRIMARY KEY, k, k2, flag, payload); + INSERT INTO matrix_left VALUES + (1, 1, 1, 1, 'l1'), + (2, 1.0, 2, 0, NULL), + (3, '1', 1, 1, 'l3'), + (4, NULL, 1, 0, 'l4'), + (5, 2, NULL, 1, 'l5'), + (6, 2, 2, NULL, 'l6'), + (7, 'x', 1, 1, NULL), + (8, X'31', 1, 0, 'l8'); + INSERT INTO matrix_right VALUES + (10, 1, 1, 0, 'r10'), + (11, 1.0, 2, 1, NULL), + (12, '1', 2, 0, 'r12'), + (13, NULL, 1, 1, 'r13'), + (14, 2, NULL, 0, NULL), + (15, 2, 2, 1, 'r15'), + (16, 2, 2, 1, 'r16'), + (17, 'y', 1, NULL, 'r17'), + (18, X'31', 1, 1, 'r18'); + -- These estimates make Turso choose a hash join while the test data stays small. + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('matrix_left', NULL, '1000'), + ('matrix_right', NULL, '100000'); + ANALYZE sqlite_schema; +} + +@setup hash-join-matrix-rows +@skip-if sqlite "SQLite does not have a hash join plan" +test hash-join-result-matrix-uses-a-hash-join { + EXPLAIN QUERY PLAN + SELECT count(*) + FROM matrix_left AS l + JOIN matrix_right AS r ON l.k = r.k + WHERE l.id <= 8 OR l.id IS NULL; +} +expect pattern { + (?s)HASH JOIN matrix_right AS r.*SCAN matrix_left AS l +} + +@setup hash-join-matrix-rows +@skip-if sqlite "SQLite does not have a hash join plan" +test hash-join-result-matrix-left-join-uses-a-hash-join { + EXPLAIN QUERY PLAN + SELECT count(*) + FROM matrix_left AS l + LEFT JOIN matrix_right AS r ON l.k = r.k AND l.k2 = r.k2 + WHERE l.id <= 8 OR l.id IS NULL; +} +expect pattern { + HASH JOIN matrix_right AS r +} + +@setup hash-join-matrix-rows +@skip-if sqlite "SQLite does not have a hash join plan" +test hash-join-result-matrix-full-join-uses-a-hash-join { + EXPLAIN QUERY PLAN + SELECT count(*) + FROM matrix_left AS l + FULL OUTER JOIN matrix_right AS r ON l.k = r.k + WHERE l.id <= 8 OR l.id IS NULL; +} +expect pattern { + HASH JOIN matrix_right AS r +} + +@setup hash-join-matrix-rows +@var join_type { JOIN | LEFT JOIN | FULL OUTER JOIN } +@var join_condition { l.k = r.k | r.k = l.k | l.k = r.k AND l.k2 = r.k2 | r.k2 = l.k2 AND r.k = l.k } +@var filter { 1 | l.flag = 1 | r.flag = 1 | l.payload IS NULL | r.payload IS NULL | coalesce(l.flag, -1) <> coalesce(r.flag, -1) } +@var limit { | LIMIT 7 | LIMIT 5 OFFSET 3 } +matrix hash-join-result-matrix { + SELECT l.id, r.id, l.k, r.k, l.k2, r.k2, l.payload, r.payload + FROM matrix_left AS l + $join_type matrix_right AS r ON $join_condition + WHERE (l.id <= 8 OR l.id IS NULL) AND ($filter) + ORDER BY l.id, r.id + $limit; +} diff --git a/conformance/sqlite-sqltests/join/hash_join_order_by.sqltest b/conformance/sqlite-sqltests/join/hash_join_order_by.sqltest new file mode 100644 index 00000000..d48423b0 --- /dev/null +++ b/conformance/sqlite-sqltests/join/hash_join_order_by.sqltest @@ -0,0 +1,177 @@ +@database :memory: + +setup hash_join_order_by { + CREATE TABLE a(id INTEGER PRIMARY KEY, k INTEGER); + INSERT INTO a VALUES (1, 10), (2, 20), (3, 30); + CREATE TABLE b(k INTEGER); + INSERT INTO b VALUES (30), (10), (20); + ANALYZE; +} + +@setup hash_join_order_by +test hash-join-order-by-build-table-ascending { + SELECT a.id + FROM a JOIN b ON b.k = a.k + WHERE a.id > 0 + ORDER BY a.id; +} +expect { + 1 + 2 + 3 +} + +@setup hash_join_order_by +test hash-join-order-by-build-table-descending-page { + SELECT a.id + FROM a JOIN b ON b.k = a.k + WHERE a.id > 0 + ORDER BY a.id DESC + LIMIT 1 OFFSET 1; +} +expect { + 2 +} + +@setup hash_join_order_by +test hash-join-window-order-by-build-table { + SELECT a.id, row_number() OVER (ORDER BY a.id) + FROM a JOIN b ON b.k = a.k + WHERE a.id > 0; +} +expect { + 1|1 + 2|2 + 3|3 +} + +@setup hash_join_order_by +@skip-if sqlite "query plan explanation output is different in sqlite" +test hash-join-order-by-build-table-uses-sorter { + EXPLAIN QUERY PLAN + SELECT a.id + FROM a JOIN b ON b.k = a.k + WHERE a.id > 0 + ORDER BY a.id; +} +expect pattern { + (?s)HASH JOIN b.*SCAN a.*USE SORTER FOR ORDER BY +} + +setup hash_join_with_outer_table { + CREATE TABLE c(id INTEGER PRIMARY KEY); + INSERT INTO c VALUES (3), (1), (2); + CREATE TABLE a(id INTEGER PRIMARY KEY, k INTEGER); + INSERT INTO a VALUES (1, 10), (2, 20), (3, 30); + CREATE TABLE b(k INTEGER); + INSERT INTO b VALUES (30), (10), (20); + INSERT INTO b SELECT 1000 + value FROM generate_series(1, 97); + ANALYZE; +} + +@setup hash_join_with_outer_table +@skip-if sqlite "query plan explanation output is different in sqlite" +test hash-join-keeps-outer-table-order-elision { + EXPLAIN QUERY PLAN + SELECT c.id, a.id + FROM c, a JOIN b ON b.k = a.k + WHERE c.id > 0 + ORDER BY c.id; +} +expect { + 1|0|0|SCAN c + 2|0|0|HASH JOIN b + 3|0|0|SCAN a +} + +@setup hash_join_with_outer_table +test hash-join-keeps-outer-table-runtime-order { + SELECT c.id + FROM c, a JOIN b ON b.k = a.k + WHERE c.id > 0 + ORDER BY c.id; +} +expect { + 1 + 1 + 1 + 2 + 2 + 2 + 3 + 3 + 3 +} + +setup materialized_hash_join_order_by { + CREATE TABLE c(id INTEGER PRIMARY KEY); + INSERT INTO c VALUES (1), (2), (3); + CREATE TABLE a(id INTEGER PRIMARY KEY, c_id INTEGER, k INTEGER); + INSERT INTO a + SELECT value, + ((value - 1) % 3) + 1, + (((value - 1) % 3) + 1) * 10 + FROM generate_series(1, 30); + CREATE INDEX a_c_id ON a(c_id); + CREATE TABLE b(k INTEGER); + INSERT INTO b VALUES (30), (10), (20); + INSERT INTO b SELECT 1000 + value FROM generate_series(1, 997); + ANALYZE; +} + +@setup materialized_hash_join_order_by +test hash-join-materialized-prefix-keeps-order { + SELECT c.id + FROM c + JOIN a ON a.c_id = c.id + JOIN b ON b.k = a.k + WHERE c.id > 0 + ORDER BY c.id; +} +expect { + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 3 + 3 + 3 + 3 + 3 + 3 + 3 + 3 + 3 + 3 +} + +@setup materialized_hash_join_order_by +@skip-if sqlite "query plan explanation output is different in sqlite" +test hash-join-materialized-prefix-uses-sorter { + EXPLAIN QUERY PLAN + SELECT c.id + FROM c + JOIN a ON a.c_id = c.id + JOIN b ON b.k = a.k + WHERE c.id > 0 + ORDER BY c.id; +} +expect pattern { + (?s)MATERIALIZE hash build input for a.*HASH JOIN b.*USE SORTER FOR ORDER BY +} diff --git a/conformance/sqlite-sqltests/join/memory.sqltest b/conformance/sqlite-sqltests/join/memory.sqltest index 20c983c6..68e775e6 100644 --- a/conformance/sqlite-sqltests/join/memory.sqltest +++ b/conformance/sqlite-sqltests/join/memory.sqltest @@ -1,5 +1,149 @@ @database :memory: +setup indexed-hash-build-rows { + CREATE TABLE expression_build_orders( + id INTEGER PRIMARY KEY, + customer_id INTEGER, + order_date TEXT + ); + CREATE INDEX expression_build_order_key + ON expression_build_orders(customer_id, lower(order_date)); + CREATE TABLE expression_build_inventory(product_id INTEGER, quantity INTEGER); + INSERT INTO expression_build_orders VALUES + (1, 100, 'ABC'), (2, 100, 'DeF'), (3, 100, NULL), + (4, 200, 'GHI'), (6, 100, 'xyz'); + INSERT INTO expression_build_inventory VALUES + (1, 2), (1, 3), (2, 4), (3, 5), (5, 6); + + CREATE TABLE covering_build_orders( + id INTEGER PRIMARY KEY, + customer_id INTEGER, + order_date TEXT + ); + INSERT INTO covering_build_orders SELECT * FROM expression_build_orders; + CREATE INDEX covering_build_order_key + ON covering_build_orders(customer_id, order_date); + + CREATE TABLE expression_covering_build_orders( + id INTEGER PRIMARY KEY, + customer_id INTEGER, + order_date TEXT + ); + INSERT INTO expression_covering_build_orders SELECT * FROM expression_build_orders; + CREATE INDEX expression_covering_build_order_key + ON expression_covering_build_orders(customer_id, lower(order_date), order_date); + + CREATE TABLE noncovering_build_orders( + id INTEGER PRIMARY KEY, + customer_id INTEGER, + order_date TEXT + ); + INSERT INTO noncovering_build_orders SELECT * FROM expression_build_orders; + CREATE INDEX noncovering_build_order_key + ON noncovering_build_orders(customer_id); + + CREATE TABLE partial_build_orders( + id INTEGER PRIMARY KEY, + customer_id INTEGER, + order_date TEXT + ); + INSERT INTO partial_build_orders SELECT * FROM expression_build_orders; + CREATE INDEX partial_build_order_key + ON partial_build_orders(customer_id, order_date) + WHERE customer_id = 100; + + CREATE TABLE expression_build_customers(id INTEGER PRIMARY KEY, enabled INTEGER); + INSERT INTO expression_build_customers VALUES (100, 1), (200, 0); +} + +@setup indexed-hash-build-rows +test expression-index-hash-build-keeps-source-column { + SELECT orders.id, lower(orders.order_date), inventory.quantity + FROM expression_build_orders AS orders + JOIN expression_build_inventory AS inventory + ON orders.id = inventory.product_id + WHERE orders.customer_id = 100 + ORDER BY orders.id, inventory.quantity; +} +expect { + 1|abc|2 + 1|abc|3 + 2|def|4 + 3||5 +} + +@setup indexed-hash-build-rows +@skip-if sqlite "SQLite does not have a hash join plan" +test expression-index-hash-build-reads-source-table { + EXPLAIN QUERY PLAN + SELECT orders.id, lower(orders.order_date), inventory.quantity + FROM expression_build_orders AS orders + JOIN expression_build_inventory AS inventory + ON orders.id = inventory.product_id + WHERE orders.customer_id = 100; +} +expect pattern { + (?s)MATERIALIZE hash build input for expression_build_orders AS orders.*HASH JOIN expression_build_inventory AS inventory.*SEARCH orders USING INTEGER PRIMARY KEY +} + +@setup indexed-hash-build-rows +@var build_table { expression_build_orders | covering_build_orders | expression_covering_build_orders | noncovering_build_orders | partial_build_orders } +@var join_type { JOIN | LEFT JOIN | FULL OUTER JOIN } +@var date_value { orders.order_date | lower(orders.order_date) } +matrix indexed-hash-build-join-results { + SELECT orders.id, $date_value, inventory.quantity + FROM $build_table AS orders + $join_type expression_build_inventory AS inventory + ON orders.id = inventory.product_id + WHERE orders.customer_id = 100 OR orders.id IS NULL + ORDER BY orders.id, inventory.quantity; +} + +setup expression-index-prefix-stats { + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('expression_build_customers', NULL, '1000'), + ('expression_build_orders', NULL, '100000'), + ('expression_build_orders', 'expression_build_order_key', '100000 100'), + ('expression_build_inventory', NULL, '100000'); + ANALYZE sqlite_schema; +} + +@setup indexed-hash-build-rows +@setup expression-index-prefix-stats +test expression-index-after-earlier-join-keeps-source-column { + SELECT customers.id, orders.id, lower(orders.order_date), inventory.quantity + FROM expression_build_customers AS customers + CROSS JOIN expression_build_orders AS orders + JOIN expression_build_inventory AS inventory + ON inventory.product_id = orders.id + WHERE orders.customer_id = customers.id AND customers.enabled = 1 + ORDER BY customers.id, orders.id, inventory.quantity; +} +expect { + 100|1|abc|2 + 100|1|abc|3 + 100|2|def|4 + 100|3||5 +} + +@setup indexed-hash-build-rows +@setup expression-index-prefix-stats +@skip-if sqlite "SQLite does not have a hash join plan" +test expression-index-after-earlier-join-reads-source-table { + EXPLAIN QUERY PLAN + SELECT customers.id, orders.id, lower(orders.order_date), inventory.quantity + FROM expression_build_customers AS customers + CROSS JOIN expression_build_orders AS orders + JOIN expression_build_inventory AS inventory + ON inventory.product_id = orders.id + WHERE orders.customer_id = customers.id AND customers.enabled = 1; +} +expect pattern { + (?s)MATERIALIZE hash build input for expression_build_orders AS orders.*SCAN expression_build_customers AS customers.*SEARCH orders USING INDEX expression_build_order_key.*HASH JOIN expression_build_inventory AS inventory +} + # regression test for issue 2794: not nulling out rowid properly when left join does not match @cross-check-integrity test min-null-regression-test { @@ -219,6 +363,41 @@ expect { aaaa|abcd } +@cross-check-integrity +test hash-join-offset-continues-with-the-next-match { + CREATE TABLE offset_hash_left(id INTEGER PRIMARY KEY, key INTEGER); + CREATE TABLE offset_hash_right(id INTEGER PRIMARY KEY, key INTEGER); + INSERT INTO offset_hash_left SELECT value, value % 20 FROM generate_series(1, 200); + INSERT INTO offset_hash_right SELECT value, value % 20 FROM generate_series(1, 220); + SELECT count(*) FROM ( + SELECT offset_hash_left.id, offset_hash_right.id + FROM offset_hash_left + JOIN offset_hash_right ON offset_hash_left.key = offset_hash_right.key + LIMIT 50 OFFSET 500 + ); +} +expect { + 50 +} + +@cross-check-integrity +test hash-left-join-offset-skips-unmatched-rows { + CREATE TABLE offset_unmatched_left(x INTEGER); + CREATE TABLE offset_unmatched_right(x INTEGER); + INSERT INTO offset_unmatched_left + VALUES (1), (2), (3), (4), (5), (6), (7), (8), (9), (10); + SELECT count(*) FROM ( + SELECT offset_unmatched_left.x, offset_unmatched_right.x + FROM offset_unmatched_left + LEFT JOIN offset_unmatched_right + ON offset_unmatched_right.x = offset_unmatched_left.x + LIMIT 1 OFFSET 1 + ); +} +expect { + 1 +} + @cross-check-integrity test inner-join-using-null { create table t(a, b); @@ -668,6 +847,27 @@ expect { 3|3|3 } +# Keep the selected build rows and their values when an IN seek feeds a hash join. +@cross-check-integrity +test hash-build-keeps-in-seek-rows-and-values { + CREATE TABLE requested(id INTEGER PRIMARY KEY, limit_value INTEGER); + CREATE TABLE allowed(id INTEGER PRIMARY KEY); + CREATE TABLE events(requested_id INTEGER, amount INTEGER, active INTEGER); + INSERT INTO requested VALUES (1, 10), (2, 20), (3, 30); + INSERT INTO allowed VALUES (1), (3); + INSERT INTO events VALUES (1, 4, 1), (1, 7, 0), (2, 8, 1), (3, 5, 1), (3, 6, 1); + SELECT requested.id, requested.limit_value, coalesce(sum(events.amount), 0) + FROM requested + LEFT JOIN events ON events.requested_id = requested.id AND events.active = 1 + WHERE requested.id IN (SELECT id FROM allowed) + GROUP BY requested.id + ORDER BY requested.id; +} +expect { + 1|10|4 + 3|30|11 +} + # The second hash join builds a3 after an earlier hash join reads a2. The a2 # table is no longer in the main join order, but a2.b still filters the a3 rows # used by the second hash join. Keep that filter when a4 has no matching row. @@ -749,3 +949,383 @@ test join-3-4-2 { expect error { cannot join using column d - column not present in both tables } + +@skip-if sqlite "Turso can replace the reversed primary-key lookup with a hash join" +test indexed-build-hashes-an-unfiltered-large-probe { + CREATE TABLE indexed_hash_parent(id INTEGER PRIMARY KEY, selected INTEGER); + CREATE TABLE indexed_hash_child(parent_id INTEGER, payload INTEGER); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('indexed_hash_parent', NULL, '200000'), + ('indexed_hash_child', NULL, '6000000'); + ANALYZE sqlite_schema; + + EXPLAIN QUERY PLAN + SELECT count(*) + FROM indexed_hash_parent AS parent + JOIN indexed_hash_child AS child ON child.parent_id = parent.id + WHERE parent.selected = 1; +} +expect pattern { + (?s)HASH JOIN indexed_hash_child AS child.*SCAN indexed_hash_parent AS parent +} + +@skip-if sqlite "Turso compares correlated subqueries with grouped joins" +test group-sort-cost-keeps-cheaper-correlated-seeks { + CREATE TABLE sort_cost_parent(id INTEGER PRIMARY KEY, selected INTEGER); + CREATE TABLE sort_cost_filter(id INTEGER PRIMARY KEY, selected INTEGER); + CREATE TABLE sort_cost_child( + parent_id INTEGER, + filter_id INTEGER, + value INTEGER, + PRIMARY KEY(parent_id, filter_id) + ); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('sort_cost_parent', NULL, '85000'), + ('sort_cost_filter', NULL, '100'), + ('sort_cost_child', NULL, '800000'), + ('sort_cost_child', 'sqlite_autoindex_sort_cost_child_1', '800000 4 1'); + ANALYZE sqlite_schema; + + EXPLAIN QUERY PLAN + SELECT parent.id + FROM sort_cost_parent AS parent + WHERE parent.selected = 1 + AND 0 = ( + SELECT min(child.value) + FROM sort_cost_child AS child + JOIN sort_cost_filter AS kept ON kept.id = child.filter_id + WHERE child.parent_id = parent.id AND kept.selected = 1 + ); +} +expect pattern { + (?s)CORRELATED SCALAR SUBQUERY.*SEARCH child USING INDEX sqlite_autoindex_sort_cost_child_1 \(parent_id=\?\) +} + +@skip-if sqlite "Turso can use a hash join when an index cannot seek by the join column" +test index-suffix-does-not-prevent-hash-join { + CREATE TABLE suffix_hash_parent(id INTEGER PRIMARY KEY, selected INTEGER); + CREATE TABLE suffix_hash_child( + group_id INTEGER, + parent_id INTEGER, + payload INTEGER, + PRIMARY KEY(group_id, parent_id) + ); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('suffix_hash_parent', NULL, '1000'), + ('suffix_hash_child', NULL, '5000000'), + ('suffix_hash_child', 'sqlite_autoindex_suffix_hash_child_1', '5000000 5000 1'); + ANALYZE sqlite_schema; + + EXPLAIN QUERY PLAN + SELECT count(*) + FROM suffix_hash_parent AS parent + JOIN suffix_hash_child AS child ON child.parent_id = parent.id + WHERE parent.selected = 1; +} +expect pattern { + (?s)HASH JOIN suffix_hash_child AS child.*SCAN suffix_hash_parent AS parent +} + +test index-suffix-join-keeps-null-and-duplicate-rows { + CREATE TABLE suffix_result_parent(id INTEGER PRIMARY KEY, selected INTEGER); + CREATE TABLE suffix_result_child( + group_id INTEGER, + parent_id INTEGER, + payload INTEGER, + PRIMARY KEY(group_id, parent_id) + ); + INSERT INTO suffix_result_parent VALUES (1, 1), (2, 1), (3, 0), (4, 1); + INSERT INTO suffix_result_child VALUES + (10, 1, 7), (20, 1, 8), (30, 2, 9), (40, NULL, 10), (NULL, 1, 11); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('suffix_result_parent', NULL, '1000'), + ('suffix_result_child', NULL, '5000000'), + ('suffix_result_child', 'sqlite_autoindex_suffix_result_child_1', '5000000 5000 1'); + ANALYZE sqlite_schema; + + SELECT parent.id, child.group_id, child.payload + FROM suffix_result_parent AS parent + LEFT JOIN suffix_result_child AS child ON child.parent_id = parent.id + WHERE parent.selected = 1 + ORDER BY parent.id, child.group_id, child.payload; +} +expect { + 1||11 + 1|10|7 + 1|20|8 + 2|30|9 + 4|| +} + +@skip-if sqlite "Turso can use a hash join when the current join prefix cannot seek an index" +test future-table-cannot-make-index-usable { + CREATE TABLE future_index_parent(id INTEGER PRIMARY KEY, selected INTEGER); + CREATE TABLE future_index_group(id INTEGER PRIMARY KEY); + CREATE TABLE future_index_child( + group_id INTEGER, + parent_id INTEGER, + PRIMARY KEY(group_id, parent_id) + ); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('future_index_parent', NULL, '1000'), + ('future_index_group', NULL, '1000000'), + ('future_index_child', NULL, '5000000'), + ('future_index_child', 'sqlite_autoindex_future_index_child_1', '5000000 5000 1'); + ANALYZE sqlite_schema; + + EXPLAIN QUERY PLAN + SELECT count(*) + FROM future_index_parent AS parent + CROSS JOIN future_index_child AS child ON child.parent_id = parent.id + CROSS JOIN future_index_group AS group_table ON child.group_id = group_table.id + WHERE parent.selected = 1; +} +expect pattern { + (?s)HASH JOIN future_index_child AS child.*SEARCH group_table USING INTEGER PRIMARY KEY.*SCAN future_index_parent AS parent +} + +test future-table-index-does-not-change-join-results { + CREATE TABLE future_result_parent(id INTEGER PRIMARY KEY, selected INTEGER); + CREATE TABLE future_result_group(id INTEGER PRIMARY KEY); + CREATE TABLE future_result_child( + group_id INTEGER, + parent_id INTEGER, + PRIMARY KEY(group_id, parent_id) + ); + INSERT INTO future_result_parent VALUES (1, 1), (2, 1), (3, 0); + INSERT INTO future_result_group VALUES (10), (30); + INSERT INTO future_result_child VALUES + (10, 1), (20, 1), (10, 2), (30, 2), (NULL, 1); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('future_result_parent', NULL, '1000'), + ('future_result_group', NULL, '1000000'), + ('future_result_child', NULL, '5000000'), + ('future_result_child', 'sqlite_autoindex_future_result_child_1', '5000000 5000 1'); + ANALYZE sqlite_schema; + + SELECT parent.id, child.group_id, group_table.id + FROM future_result_parent AS parent + CROSS JOIN future_result_child AS child ON child.parent_id = parent.id + CROSS JOIN future_result_group AS group_table ON child.group_id = group_table.id + WHERE parent.selected = 1 + ORDER BY parent.id, child.group_id, group_table.id; +} +expect { + 1|10|10 + 2|10|10 + 2|30|30 +} + +setup hash-index-prefix-rows { + CREATE TABLE prefix_parent(id INTEGER PRIMARY KEY, selected INTEGER); + CREATE TABLE prefix_child(group_id INTEGER, parent_id INTEGER, payload INTEGER, + PRIMARY KEY(group_id, parent_id)); + CREATE INDEX prefix_child_positive + ON prefix_child(group_id, parent_id) WHERE payload > 0; + INSERT INTO prefix_parent VALUES (1, 1), (2, 1), (3, 1), (4, 0); + INSERT INTO prefix_child VALUES + (10, 1, -1), (20, 1, 9), (30, 2, 8), (40, 3, -5), (50, NULL, 6); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('prefix_parent', NULL, '1000'), + ('prefix_child', NULL, '5000000'), + ('prefix_child', 'sqlite_autoindex_prefix_child_1', '5000000 5000 1'), + ('prefix_child', 'prefix_child_positive', '5000000 5000 1'); + ANALYZE sqlite_schema; +} + +@setup hash-index-prefix-rows +@skip-if sqlite "SQLite does not have a hash join plan" +test range-prefix-still-allows-hash-join { + EXPLAIN QUERY PLAN + SELECT parent.id, child.group_id, child.payload + FROM prefix_parent AS parent + LEFT JOIN prefix_child AS child + ON child.parent_id = parent.id AND child.group_id > 15 AND child.payload >= 8 + WHERE parent.selected = 1; +} +expect pattern { + HASH JOIN prefix_child AS child +} + +@setup hash-index-prefix-rows +@skip-if sqlite "SQLite does not have a hash join plan" +test equality-prefix-keeps-index-search { + EXPLAIN QUERY PLAN + SELECT count(*) + FROM prefix_parent AS parent + LEFT JOIN prefix_child AS child + ON child.parent_id = parent.id AND child.group_id = 10 + WHERE parent.selected = 1; +} +expect pattern { + SEARCH child USING COVERING INDEX sqlite_autoindex_prefix_child_1 \(group_id=\? AND parent_id=\?\) +} + +@setup hash-index-prefix-rows +@var join_type { JOIN | LEFT JOIN } +@var first_column { child.group_id > 15 | child.group_id IN (10, 20) | child.group_id = 10 | child.payload > 0 } +matrix index-prefix-join-results { + SELECT parent.id, child.group_id, child.payload + FROM prefix_parent AS parent + $join_type prefix_child AS child + ON child.parent_id = parent.id AND $first_column + WHERE parent.selected = 1 + ORDER BY parent.id, child.group_id, child.payload; +} + +@skip-if sqlite "Turso reports hash join estimates in its JSON plan" +test unique-composite-hash-key-uses-one-key-estimate { + CREATE TABLE composite_hash_parent(first INTEGER, second INTEGER, selected INTEGER); + CREATE UNIQUE INDEX composite_hash_parent_key + ON composite_hash_parent(first, second); + CREATE TABLE composite_hash_child(first INTEGER, second INTEGER); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('composite_hash_parent', NULL, '1000'), + ('composite_hash_parent', 'composite_hash_parent_key', '1000 10 1'), + ('composite_hash_child', NULL, '5000'); + ANALYZE sqlite_schema; + + EXPLAIN QUERY PLAN FORMAT=JSON + SELECT count(*) + FROM composite_hash_parent AS parent + JOIN composite_hash_child AS child + ON child.first = parent.first AND child.second = parent.second + WHERE parent.selected = 1; +} +expect pattern { + (?s)"detail":"HASH JOIN composite_hash_child AS child".*"rows_per_input":5 +} + +@skip-if sqlite "Turso reports hash join estimates in its JSON plan" +test unique-hash-key-does-not-apply-to-a-different-collation { + CREATE TABLE collation_unique_build(key TEXT UNIQUE); + CREATE TABLE collation_unique_probe(key TEXT); + INSERT INTO collation_unique_build VALUES + ('aaa'), ('aaA'), ('aAa'), ('aAA'), + ('Aaa'), ('AaA'), ('AAa'), ('AAA'); + INSERT INTO collation_unique_probe VALUES ('aaa'); + ANALYZE; + + EXPLAIN QUERY PLAN FORMAT=JSON + SELECT count(*) + FROM collation_unique_build + CROSS JOIN collation_unique_probe + ON collation_unique_probe.key COLLATE NOCASE = collation_unique_build.key; +} +expect pattern { + (?s)"detail":"HASH JOIN collation_unique_probe".*"rows_per_input":0.1 +} + +@skip-if sqlite "Turso reports hash join estimates in its JSON plan" +test hash-join-cost-includes-rows-visited { + CREATE TABLE hash_cost_left(key INTEGER); + CREATE TABLE hash_cost_right(key INTEGER); + + EXPLAIN QUERY PLAN FORMAT=JSON + SELECT count(*) + FROM hash_cost_left + JOIN hash_cost_right ON hash_cost_left.key = hash_cost_right.key; +} +expect pattern { + (?s)"detail":"HASH JOIN hash_cost_right".*"access_cost":300110000 +} + +@skip-if sqlite "Turso reports its selected plan in JSON" +test hash-build-row-limit-counts-prefix-rows { + CREATE TABLE prefix_row_limit_outer(value INTEGER); + CREATE TABLE prefix_row_limit_build(value INTEGER, key INTEGER); + CREATE INDEX prefix_row_limit_build_value + ON prefix_row_limit_build(value); + CREATE TABLE prefix_row_limit_probe(key INTEGER); + + EXPLAIN QUERY PLAN FORMAT=JSON + SELECT count(*) + FROM prefix_row_limit_outer AS outer_rows + CROSS JOIN prefix_row_limit_build AS build_rows + ON build_rows.value = outer_rows.value + CROSS JOIN prefix_row_limit_probe AS probe_rows + ON build_rows.key = probe_rows.key; +} +expect pattern { + "detail":"SEARCH probe_rows USING COVERING INDEX ephemeral_prefix_row_limit_probe_t3 +} + +@skip-if sqlite "Turso uses its own join cost model" +test no-stats-hash-join-keeps-unindexed-probe-fanout { + CREATE TABLE a(id INTEGER PRIMARY KEY, b_id INTEGER, chosen INTEGER); + CREATE TABLE b(id INTEGER PRIMARY KEY, c_key INTEGER); + CREATE TABLE c(c_key INTEGER, d_id INTEGER); + CREATE TABLE d(id INTEGER PRIMARY KEY); + + EXPLAIN QUERY PLAN FORMAT=JSON + SELECT count(*) + FROM a + CROSS JOIN b ON b.id = a.b_id + CROSS JOIN c ON c.c_key = b.c_key + CROSS JOIN d ON d.id = c.d_id + WHERE a.chosen = 1; +} +expect pattern { + (?s)"detail":"HASH JOIN c".*"rows_per_input":100000.*"detail":"SEARCH d USING INTEGER PRIMARY KEY +} + +@skip-if sqlite "Turso uses its own join cost model" +test partial-stats-hash-join-uses-known-unique-key-count { + CREATE TABLE a(id INTEGER PRIMARY KEY, b_id INTEGER, chosen INTEGER); + CREATE TABLE b(id INTEGER PRIMARY KEY, c_key INTEGER); + CREATE UNIQUE INDEX b_c_key ON b(c_key); + CREATE TABLE c(c_key INTEGER, d_id INTEGER); + CREATE TABLE d(id INTEGER PRIMARY KEY); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('b', NULL, '1000'), + ('b', 'b_c_key', '1000 1'), + ('c', NULL, '5000'); + ANALYZE sqlite_schema; + + EXPLAIN QUERY PLAN FORMAT=JSON + SELECT count(*) + FROM a + CROSS JOIN b ON b.id = a.b_id + CROSS JOIN c ON c.c_key = b.c_key + CROSS JOIN d ON d.id = c.d_id + WHERE a.chosen = 1; +} +expect pattern { + (?s)"detail":"HASH JOIN c".*"rows_per_input":5.*"detail":"SEARCH d USING INTEGER PRIMARY KEY +} + +@skip-if sqlite "Turso can materialize a hash build input" +test materialized-hash-build-keeps-index-search { + CREATE TABLE selected_hash_build(id INTEGER PRIMARY KEY, key INTEGER, selected INTEGER); + CREATE INDEX selected_hash_build_selected + ON selected_hash_build(selected); + CREATE TABLE selected_hash_probe(key INTEGER); + + EXPLAIN QUERY PLAN FORMAT=JSON + SELECT count(*) + FROM selected_hash_build AS build_rows + JOIN selected_hash_probe AS probe_rows + ON build_rows.key = probe_rows.key + WHERE build_rows.selected = 1; +} +expect pattern { + (?s)"detail":"MATERIALIZE hash build input for selected_hash_build AS build_rows".*"detail":"SEARCH build_rows USING INDEX selected_hash_build_selected.*"detail":"HASH JOIN selected_hash_probe AS probe_rows" +} diff --git a/conformance/sqlite-sqltests/json/default.sqltest b/conformance/sqlite-sqltests/json/default.sqltest index b042a33f..1954641e 100644 --- a/conformance/sqlite-sqltests/json/default.sqltest +++ b/conformance/sqlite-sqltests/json/default.sqltest @@ -1915,6 +1915,13 @@ expect { {"name":"John","age":30,"city":"New York"} } +test jsonb_null_is_sql_null { + SELECT typeof(jsonb(NULL)), jsonb(NULL) IS NULL; +} +expect { + null|1 +} + test json_complex_nested { SELECT json(jsonb('{"complex": {"nested": ["array", "of", "values"], "numbers": [1, 2, 3]}}')); } @@ -3461,3 +3468,23 @@ test jsonb-oversized-child-remove { } expect { } + +test json_each_over_null_column { + WITH t(id, tags) AS ( + VALUES (1, '["x"]'), (2, NULL) + ) + SELECT COUNT(*) FROM t, json_each(t.tags); +} +expect { + 1 +} + +test json_tree_over_null_column { + WITH t(id, tags) AS ( + VALUES (1, '["x"]'), (2, NULL) + ) + SELECT COUNT(*) FROM t, json_tree(t.tags); +} +expect { + 2 +} diff --git a/conformance/sqlite-sqltests/json/json-nested-array-path.sqltest b/conformance/sqlite-sqltests/json/json-nested-array-path.sqltest new file mode 100644 index 00000000..9fb6b3c8 --- /dev/null +++ b/conformance/sqlite-sqltests/json/json-nested-array-path.sqltest @@ -0,0 +1,50 @@ +@database :memory: + +# A path with two array locators in a row ($[0][2]) walks into the inner +# array through the outer array's element. The size update after a change +# used to resize the inner array's header twice, once as the outer entry's +# element and once as the inner entry's container, which corrupted the +# document and, when the double count went past the element's size, +# panicked while encoding a negative size. + +test json-remove-key-inside-nested-array { + SELECT json_remove('[[1,2,{"Z":5}]]','$[0][2].Z'); +} +expect { + [[1,2,{}]] +} + +test json-insert-key-inside-nested-array { + SELECT json_insert('[[1,2,{}]]','$[0][2].Z',5); +} +expect { + [[1,2,{"Z":5}]] +} + +test json-remove-then-insert-inside-nested-array { + SELECT json_insert(json_remove('[[1,2,{"Z":5}]]','$[0][2].Z'),'$[0][2].Z',5)->>'$[0][2].Z'; +} +expect { + 5 +} + +test json-remove-element-of-triple-nested-array { + SELECT json_remove('[[[1,2,3]]]','$[0][0][1]'); +} +expect { + [[[1,3]]] +} + +test json-set-deep-inside-nested-arrays { + SELECT json_set('[[{"a":[1,{"b":2}]}]]','$[0][0].a[1].b',3); +} +expect { + [[{"a":[1,{"b":3}]}]] +} + +test json-remove-inside-nested-array-with-siblings { + SELECT json_remove('[1,[{"eeee":1,"ffff":"bag"}],2]','$[1][0].eeee'); +} +expect { + [1,[{"ffff":"bag"}],2] +} diff --git a/conformance/sqlite-sqltests/json/json_tree_columns.sqltest b/conformance/sqlite-sqltests/json/json_tree_columns.sqltest new file mode 100644 index 00000000..e0c2ffc7 --- /dev/null +++ b/conformance/sqlite-sqltests/json/json_tree_columns.sqltest @@ -0,0 +1,72 @@ +@database :memory: + +# json_tree and json_each expose rowid like any virtual table, so a query +# can select or join on rt.rowid. The value itself is the engine's own +# row counter, so only its presence and type are checked here. + +test json-tree-rowid-is-selectable { + SELECT count(*) FROM (SELECT rt.rowid AS n FROM json_tree('{"a":[1,2]}') AS rt WHERE rt.type NOT IN ('object','array')); +} +expect { + 2 +} + +test json-tree-rowid-unqualified { + SELECT typeof(rowid) FROM json_tree('[5]') WHERE type='integer'; +} +expect { + integer +} + +test json-tree-rowid-into-table { + CREATE TABLE kv(n, key, val); + INSERT INTO kv SELECT rt.rowid, rt.fullkey, rt.atom FROM json_tree('{"a":[1,2]}') AS rt WHERE rt.type NOT IN ('object','array'); + SELECT key, val FROM kv ORDER BY key; +} +expect { + $.a[0]|1 + $.a[1]|2 +} + +# fullkey quotes an object key unless it starts with an ASCII letter and +# continues with ASCII letters and digits only, as SQLite does; the empty +# key is quoted too. + +test json-tree-fullkey-quotes-keys-like-sqlite { + SELECT fullkey FROM json_tree('{"a":{"":1,"$g":2,"_h":3,"a b":4,"x1":5,"é":6}}') WHERE atom IS NOT NULL; +} +expect { + $.a."" + $.a."$g" + $.a."_h" + $.a."a b" + $.a.x1 + $.a."é" +} + +test json-tree-fullkey-roundtrips-through-extract { + SELECT count(*) FROM json_tree('{"":{"a b":[1,{"$":2}]},"_":3}') AS rt WHERE rt.type NOT IN ('object','array') AND rt.atom IS NOT (json('{"":{"a b":[1,{"$":2}]},"_":3}') ->> rt.fullkey); +} +expect { + 0 +} + +# atom is the decoded string, as ->> returns it, not the JSON text with +# its escapes. + +test json-tree-atom-is-decoded-text { + SELECT atom, length(atom) FROM json_tree('["a\"b","é","x\ty","\\\\"]') WHERE type='text'; +} +expect { + a"b|3 + é|1 + x y|3 + \|1 +} + +test json-tree-atom-matches-extract { + SELECT count(*) FROM json_tree('{"a":"oat\r\n","b":["\fpan\b","gem\"hay","y㑘z"]}') AS rt WHERE rt.type='text' AND rt.atom IS NOT (json('{"a":"oat\r\n","b":["\fpan\b","gem\"hay","y㑘z"]}') ->> rt.fullkey); +} +expect { + 0 +} diff --git a/conformance/sqlite-sqltests/left-join-in-subquery-on-indexed-column.sqltest b/conformance/sqlite-sqltests/left-join-in-subquery-on-indexed-column.sqltest new file mode 100644 index 00000000..9ede1c08 --- /dev/null +++ b/conformance/sqlite-sqltests/left-join-in-subquery-on-indexed-column.sqltest @@ -0,0 +1,78 @@ +@database :memory: + +# WHERE u.k IN (SELECT ...) on the right table of a LEFT JOIN must also +# filter the null-extended rows. With an index on u.k the planner used to +# drive u's loop from the IN values (InSeek), so a left row with no match +# still came back null-extended, never checked against the IN term (#8753). +# The join key is INTEGER on one side and TEXT on the other so the ON term +# cannot become the seek; only the IN term is a candidate. + +setup tables { + CREATE TABLE t(id INTEGER PRIMARY KEY); + CREATE TABLE u(k TEXT); + CREATE TABLE v(k TEXT); + INSERT INTO t VALUES (5), (7); + INSERT INTO u VALUES ('7'); + INSERT INTO v VALUES ('7'), ('9'); + CREATE INDEX ui ON u(k); +} + +@setup tables +test left-join-in-subquery-filters-the-null-extended-row { + SELECT group_concat(t.id) FROM t LEFT JOIN u ON t.id = u.k + WHERE u.k IN (SELECT k FROM v); +} +expect { + 7 +} + +@setup tables +test left-join-in-empty-subquery-returns-no-rows { + SELECT count(*), count(u.k) FROM t LEFT JOIN u ON t.id = u.k + WHERE u.k IN (SELECT k FROM v WHERE k = 'a'); +} +expect { + 0|0 +} + +@setup tables +test left-join-in-subquery-as-a-delete-filter-spares-the-unmatched-row { + DELETE FROM t WHERE id IN ( + SELECT t.id FROM t LEFT JOIN u ON t.id = u.k + WHERE u.k IN (SELECT k FROM v) + ); + SELECT count(*) FROM t; +} +expect { + 1 +} + +@setup tables +test left-join-in-list-filters-the-null-extended-row { + SELECT group_concat(t.id) FROM t LEFT JOIN u ON t.id = u.k + WHERE u.k IN ('7', '9'); +} +expect { + 7 +} + +@setup tables +test left-join-in-subquery-in-the-on-clause-still-seeks { + -- ON-clause terms define what a match is, so this IN term may still + -- drive the seek; both left rows come back. + SELECT t.id, u.k FROM t LEFT JOIN u ON t.id = u.k AND u.k IN (SELECT k FROM v) + ORDER BY t.id; +} +expect { + 5| + 7|7 +} + +@setup tables +test inner-join-in-subquery-on-indexed-column-unchanged { + SELECT group_concat(t.id) FROM t JOIN u ON t.id = u.k + WHERE u.k IN (SELECT k FROM v); +} +expect { + 7 +} diff --git a/conformance/sqlite-sqltests/materialized_subquery.sqltest b/conformance/sqlite-sqltests/materialized_subquery.sqltest new file mode 100644 index 00000000..797de9d5 --- /dev/null +++ b/conformance/sqlite-sqltests/materialized_subquery.sqltest @@ -0,0 +1,20 @@ +@database :memory: + +@cross-check-integrity +test materialized-subquery-preserves-duplicate-rows { + CREATE TABLE t1(id INTEGER PRIMARY KEY, d INT); + CREATE TABLE t2(id INTEGER PRIMARY KEY, a INT, b INT, c INT, d INT); + INSERT INTO t1 VALUES (1, -5), (2, -5); + INSERT INTO t2 VALUES (10, 1, NULL, NULL, -5), (11, 1, NULL, NULL, -5); + SELECT sub_t1.id, sub_t2.a + FROM (SELECT * FROM t1) AS sub_t1 + JOIN (SELECT a, b, c, d, c + d AS cd FROM t2) AS sub_t2 + ON sub_t1.d = sub_t2.d + ORDER BY sub_t1.id, sub_t2.a; +} +expect { + 1|1 + 1|1 + 2|1 + 2|1 +} diff --git a/conformance/sqlite-sqltests/math/memory.sqltest b/conformance/sqlite-sqltests/math/memory.sqltest index 9e9b6e67..b8298432 100644 --- a/conformance/sqlite-sqltests/math/memory.sqltest +++ b/conformance/sqlite-sqltests/math/memory.sqltest @@ -3083,3 +3083,35 @@ test mod-huge-dividend-log10-modulus { expect { 1 } + +test arithmetic-on-integer-and-float-columns { + CREATE TABLE t(a, b); + INSERT INTO t VALUES (9223372036854775807, 1), (-9223372036854775808, 1), (1e308, 10), (1e308, 0.0), (0.25, 3), (3, 0.25), (1, 0.5), ('1.5', 2), (2, '0.5'), (NULL, 1.0), (1, NULL); + SELECT a + b, a - b, a * b, typeof(a * b), typeof((a * b) - (a * b)) FROM t; +} +expect { + 9.22337203685478e+18|9223372036854775806|9223372036854775807|integer|integer + -9223372036854775807|-9.22337203685478e+18|-9223372036854775808|integer|integer + 1.0e+308|1.0e+308|Inf|real|null + 1.0e+308|1.0e+308|0.0|real|real + 3.25|-2.75|0.75|real|real + 3.25|2.75|0.75|real|real + 1.5|0.5|0.5|real|real + 3.5|-0.5|3.0|real|real + 2.5|1.5|1.0|real|real + |||null|null + |||null|null +} +expect @js { + 9223372036854776000|9223372036854775806|9223372036854775807|integer|integer + -9223372036854775807|-9223372036854776000|-9223372036854775808|integer|integer + 1e+308|1e+308|Inf|real|null + 1e+308|1e+308|0|real|real + 3.25|-2.75|0.75|real|real + 3.25|2.75|0.75|real|real + 1.5|0.5|0.5|real|real + 3.5|-0.5|3|real|real + 2.5|1.5|1|real|real + |||null|null + |||null|null +} diff --git a/conformance/sqlite-sqltests/multi_index_intersection.sqltest b/conformance/sqlite-sqltests/multi_index_intersection.sqltest index 242f9901..45f2834e 100644 --- a/conformance/sqlite-sqltests/multi_index_intersection.sqltest +++ b/conformance/sqlite-sqltests/multi_index_intersection.sqltest @@ -139,3 +139,39 @@ expect { snapshot-eqp composite-index-preferred-over-intersection-eqp { SELECT id FROM t WHERE a = 10 AND b = 100; } + +setup multi_index_and_setup { + CREATE TABLE t( + a, b, + c0, c1, c2, c3, c4, c5, c6, c7, c8 + ); + + CREATE INDEX ia ON t(a); + CREATE INDEX ib ON t(b); + + WITH RECURSIVE n(x) AS ( + VALUES(0) + UNION ALL + SELECT x + 1 FROM n WHERE x < 9999 + ) + INSERT INTO t + SELECT + x % 10, + (x / 10) % 10, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL + FROM n; + + ANALYZE; +} + +@setup multi_index_and_setup +@skip-if mvcc "root page numbers are negative under mvcc" +snapshot-eqp multi_index_and_eqp { + SELECT * FROM t WHERE a = 1 AND b = 2; +} + +@skip-if mvcc "root page numbers are negative under mvcc" +@setup multi_index_and_setup +snapshot multi_index_and { + SELECT * FROM t WHERE a = 1 AND b = 2; +} \ No newline at end of file diff --git a/conformance/sqlite-sqltests/multi_index_or_compound.sqltest b/conformance/sqlite-sqltests/multi_index_or_compound.sqltest index b833e4be..cb6dee93 100644 --- a/conformance/sqlite-sqltests/multi_index_or_compound.sqltest +++ b/conformance/sqlite-sqltests/multi_index_or_compound.sqltest @@ -90,6 +90,25 @@ setup compound_branch_seek_schema { INSERT INTO item VALUES (6, 7, 'e'); } +setup or_join_rows { + CREATE TABLE or_left(id INTEGER PRIMARY KEY, x INTEGER, flag INTEGER); + CREATE INDEX or_left_x ON or_left(x); + INSERT INTO or_left VALUES + (1, 1, 1), (2, 1, 0), (3, 2, 1), (4, 2, 0), (5, NULL, 1), (6, 3, 1); + + CREATE TABLE or_right(id INTEGER PRIMARY KEY, left_id INTEGER, y INTEGER); + CREATE INDEX or_right_y ON or_right(y); + INSERT INTO or_right VALUES + (1, 1, 2), (2, 1, 1), (3, 2, 2), (4, 3, 1), + (5, 3, 2), (6, 4, 1), (7, 5, NULL), (8, 999, 1); +} + +setup implied_or_plan_schema { + CREATE TABLE a(x INTEGER, z INTEGER); + CREATE INDEX ax ON a(x); + CREATE TABLE b(y INTEGER); +} + # Basic: find entities connected to a specific node via 'knows' edges (bidirectional) @setup graph_schema test compound-or-basic { @@ -314,7 +333,6 @@ snapshot-eqp compound-or-benchmark-shape-eqp { # ANALYZE must not make a temporary type index win when it leaves the full # join condition to run for every matching pair. @skip-if sqlite "query plan explanation output is different in sqlite" -@skip-if mvcc "MVCC does not reload sqlite_stat1 after this test changes it" @setup benchmark_shape_schema test compound-or-analyzed-benchmark-shape-eqp { ANALYZE; @@ -344,6 +362,247 @@ expect { 4|0|0|USE HASH TABLE FOR DISTINCT } +@skip-if sqlite "Turso summarizes multi-index branches on one plan row" +test compound-or-keeps-correlated-index-columns-together { + CREATE TABLE t(id INTEGER PRIMARY KEY, x INTEGER, y INTEGER, payload TEXT); + CREATE INDEX txy ON t(x, y); + + EXPLAIN QUERY PLAN + SELECT * + FROM t + WHERE (x = 1 AND y = 10) OR (x = 2 AND y = 20); +} +expect { + 1|0|0|MULTI-INDEX OR t (txy, txy) +} + +@skip-if sqlite "Turso reports its row estimates in JSON" +test derived-or-filter-skips-correlated-columns { + CREATE TABLE correlated_values(x INTEGER, y INTEGER); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) + VALUES ('correlated_values', NULL, '1000'); + ANALYZE sqlite_schema; + + EXPLAIN QUERY PLAN FORMAT=JSON + SELECT count(*) + FROM correlated_values + WHERE (x = 1 AND y = 1) OR (x = 2 AND y = 2); +} +expect pattern { + "rows_per_input":1000 +} + +@setup implied_or_plan_schema +@skip-if sqlite "Turso uses one index search for the implied IN filter" +test implied-or-filter-accepts-parenthesized-equalities { + EXPLAIN QUERY PLAN + SELECT * FROM a JOIN b ON 1 = 1 + WHERE ((a.x = 1) AND b.y = 1) + OR ((a.x = 2) AND b.y = 2); +} +expect { + 1|0|0|SCAN b + 2|0|0|SEARCH a USING INDEX ax (x=?) +} + +@setup implied_or_plan_schema +@skip-if sqlite "Turso uses one index search for the implied IN filter" +test implied-or-filter-accepts-negative-numbers { + EXPLAIN QUERY PLAN + SELECT * FROM a JOIN b ON 1 = 1 + WHERE (a.x = -1 AND b.y = 1) + OR (a.x = -2 AND b.y = 2); +} +expect { + 1|0|0|SCAN b + 2|0|0|SEARCH a USING INDEX ax (x=?) +} + +@setup implied_or_plan_schema +@skip-if sqlite "Turso uses one index search for the implied IN filter" +test implied-or-filter-uses-only-indexed-column { + EXPLAIN QUERY PLAN + SELECT * FROM a JOIN b ON 1 = 1 + WHERE (a.x = 1 AND a.z = 10 AND b.y = 1) + OR (a.x = 2 AND a.z = 20 AND b.y = 2); +} +expect { + 1|0|0|SCAN b + 2|0|0|SEARCH a USING INDEX ax (x=?) +} + +@skip-if sqlite "Turso summarizes multi-index branches on one plan row" +test implied-or-filter-keeps-compound-index-searches { + CREATE TABLE a(x INTEGER, z INTEGER); + CREATE INDEX axz ON a(x, z); + CREATE TABLE b(y INTEGER); + + EXPLAIN QUERY PLAN + SELECT * FROM a JOIN b ON 1 = 1 + WHERE (a.x = 1 AND a.z = 10 AND b.y = 1) + OR (a.x = 2 AND a.z = 20 AND b.y = 2); +} +expect { + 1|0|0|SCAN b + 2|0|0|MULTI-INDEX OR a (axz, axz) +} + +test implied-or-filter-keeps-negative-number-results { + CREATE TABLE a(id INTEGER PRIMARY KEY, x INTEGER); + CREATE INDEX ax ON a(x); + INSERT INTO a VALUES (1, -1), (2, -2), (3, 1), (4, NULL); + CREATE TABLE b(y INTEGER); + INSERT INTO b VALUES (1), (2); + + SELECT a.id, b.y FROM a JOIN b ON 1 = 1 + WHERE ((a.x) = (-1) AND b.y = 1) + OR ((a.x) = (-2) AND b.y = 2) + ORDER BY a.id, b.y; +} +expect { + 1|1 + 2|2 +} + +test implied-or-filter-keeps-correlated-column-results { + CREATE TABLE a(id INTEGER PRIMARY KEY, x INTEGER, z INTEGER); + CREATE INDEX ax ON a(x); + INSERT INTO a VALUES (1, 1, 10), (2, 1, 20), (3, 2, 10), (4, 2, 20); + CREATE TABLE b(y INTEGER); + INSERT INTO b VALUES (1), (2); + + SELECT a.id, b.y FROM a JOIN b ON 1 = 1 + WHERE (a.x = 1 AND a.z = 10 AND b.y = 1) + OR (a.x = 2 AND a.z = 20 AND b.y = 2) + ORDER BY a.id, b.y; +} +expect { + 1|1 + 4|2 +} + +@skip-if sqlite "Turso summarizes repeated IN searches on one plan row" +test nested-or-branches-use-one-index-search { + CREATE TABLE t(x INTEGER); + CREATE INDEX tx ON t(x); + + EXPLAIN QUERY PLAN + SELECT * + FROM t + WHERE x = 1 OR (x = 2 OR x = 3); +} +expect { + 1|0|0|SEARCH t USING COVERING INDEX tx (x=?) +} + +@skip-if sqlite "Turso summarizes repeated IN searches on one plan row" +test nested-and-branches-use-one-index-search { + CREATE TABLE t(x INTEGER, y INTEGER, z INTEGER); + CREATE INDEX tx ON t(x); + + EXPLAIN QUERY PLAN + SELECT * + FROM t + WHERE (y > 0 AND (x = 1 AND z > 0)) + OR (y < 0 AND (x = 2 AND z < 0)); +} +expect { + 1|0|0|SEARCH t USING INDEX tx (x=?) +} + +@setup or_join_rows +@var join_type { JOIN | LEFT JOIN | FULL OUTER JOIN } +@var filter { + (l.x = 1 AND r.y = 2) OR (l.x = 2 AND r.y = 1) + | (1 = l.x AND 2 = r.y) OR (2 = l.x AND 1 = r.y) + | (l.x = 1 AND r.y = 2) OR r.y = 1 + | (l.x = 1 AND (r.y = 2 AND l.flag = 1)) OR (l.x = 2 AND (r.y = 1 AND l.flag = 1)) +} +matrix inferred-or-filters-keep-join-results { + SELECT l.id, r.id, l.x, r.y + FROM or_left AS l $join_type or_right AS r ON l.id = r.left_id + WHERE $filter + ORDER BY l.id, r.id; +} + +@setup or_join_rows +test outer-join-or-condition-keeps-unmatched-left-rows { + SELECT l.id, r.id + FROM or_left AS l + LEFT JOIN or_right AS r + ON l.id = r.left_id + AND ((l.x = 1 AND r.y = 2) OR (l.x = 2 AND r.y = 1)) + ORDER BY l.id, r.id; +} +expect { + 1|1 + 2|3 + 3|4 + 4|6 + 5| + 6| +} + +@setup or_join_rows +test inferred-or-filter-keeps-update-results { + UPDATE or_left SET flag = flag + 10 WHERE x = 1 OR x = 2; + SELECT id, flag FROM or_left ORDER BY id; +} +expect { + 1|11 + 2|10 + 3|11 + 4|10 + 5|1 + 6|1 +} + +@setup or_join_rows +test two-column-or-keeps-delete-results { + DELETE FROM or_left + WHERE (x = 1 AND flag = 0) OR (x = 2 AND flag = 1); + SELECT id FROM or_left ORDER BY id; +} +expect { + 1 + 4 + 5 + 6 +} + +test inferred-or-filter-keeps-text-collation { + CREATE TABLE collated_values(id INTEGER PRIMARY KEY, value TEXT COLLATE NOCASE); + CREATE INDEX collated_value_index ON collated_values(value); + INSERT INTO collated_values VALUES + (1, 'abc'), (2, 'ABC'), (3, 'def'), (4, 'xyz'), (5, NULL); + + SELECT id FROM collated_values + WHERE value = 'AbC' OR value = 'DeF' + ORDER BY id; +} +expect { + 1 + 2 + 3 +} + +test inferred-or-filter-keeps-blob-and-integer-values { + CREATE TABLE mixed_values(id INTEGER PRIMARY KEY, value BLOB); + CREATE INDEX mixed_value_index ON mixed_values(value); + INSERT INTO mixed_values VALUES + (1, X'31'), (2, 1), (3, '1'), (4, X'32'), (5, NULL); + + SELECT id FROM mixed_values + WHERE value = X'31' OR value = 1 + ORDER BY id; +} +expect { + 1 + 2 +} + # Snapshot EQP: correlated subquery residuals should fall back to a regular scan @setup correlated_subquery_schema snapshot-eqp compound-or-correlated-subquery-eqp { diff --git a/conformance/sqlite-sqltests/mvcc-analyze-refreshes-planner-stats.sqltest b/conformance/sqlite-sqltests/mvcc-analyze-refreshes-planner-stats.sqltest new file mode 100644 index 00000000..7664a13e --- /dev/null +++ b/conformance/sqlite-sqltests/mvcc-analyze-refreshes-planner-stats.sqltest @@ -0,0 +1,55 @@ +@database :temp: +@skip-file-if sqlite "MVCC and EXPLAIN QUERY PLAN FORMAT=JSON are not supported in SQLite" + +# Regression coverage for https://github.com/tursodatabase/turso/issues/9109: +# in MVCC mode an autocommit ANALYZE wrote sqlite_stat1 but did not reload +# the planner statistics on the same connection, so the next plan still used +# the default 1000000 row estimate instead of the 100 rows ANALYZE counted. + +test mvcc-analyze-writes-sqlite-stat1 { + PRAGMA journal_mode='MVCC'; + CREATE TABLE t(id INTEGER PRIMARY KEY, value INTEGER); + WITH RECURSIVE seq(n) AS ( + VALUES(1) + UNION ALL + SELECT n + 1 FROM seq WHERE n < 100 + ) + INSERT INTO t SELECT n, n FROM seq; + ANALYZE; + SELECT tbl, idx, stat FROM sqlite_stat1; +} +expect { + mvcc + t||100 +} + +test mvcc-analyze-next-plan-uses-new-row-estimate { + PRAGMA journal_mode='MVCC'; + CREATE TABLE t(id INTEGER PRIMARY KEY, value INTEGER); + WITH RECURSIVE seq(n) AS ( + VALUES(1) + UNION ALL + SELECT n + 1 FROM seq WHERE n < 100 + ) + INSERT INTO t SELECT n, n FROM seq; + ANALYZE; + EXPLAIN QUERY PLAN FORMAT=JSON SELECT * FROM t; +} +expect pattern { + "rows_per_input":100,"output_rows":100, +} + +test wal-analyze-next-plan-uses-new-row-estimate { + CREATE TABLE t(id INTEGER PRIMARY KEY, value INTEGER); + WITH RECURSIVE seq(n) AS ( + VALUES(1) + UNION ALL + SELECT n + 1 FROM seq WHERE n < 100 + ) + INSERT INTO t SELECT n, n FROM seq; + ANALYZE; + EXPLAIN QUERY PLAN FORMAT=JSON SELECT * FROM t; +} +expect pattern { + "rows_per_input":100,"output_rows":100, +} diff --git a/conformance/sqlite-sqltests/offset/memory.sqltest b/conformance/sqlite-sqltests/offset/memory.sqltest index 2d8bc1e2..0c82caee 100644 --- a/conformance/sqlite-sqltests/offset/memory.sqltest +++ b/conformance/sqlite-sqltests/offset/memory.sqltest @@ -16,6 +16,52 @@ expect { 5 } +@cross-check-integrity +test offset-continues-the-innermost-join-loop { + CREATE TABLE offset_left(x INTEGER); + CREATE TABLE offset_right(y INTEGER); + INSERT INTO offset_left VALUES (1), (2); + INSERT INTO offset_right VALUES (10), (20), (30); + SELECT count(*) FROM ( + SELECT * + FROM offset_left CROSS JOIN offset_right + LIMIT 2 OFFSET 2 + ); +} +expect { + 2 +} + +# https://github.com/tursodatabase/turso/issues/8727 +@cross-check-integrity +test offset-over-join-skips-result-rows-not-outer-rows { + CREATE TABLE offset_join_a(k INTEGER PRIMARY KEY); + CREATE TABLE offset_join_b(k INTEGER PRIMARY KEY); + INSERT INTO offset_join_a VALUES (1), (2), (3); + INSERT INTO offset_join_b VALUES (1), (2); + SELECT offset_join_a.k, offset_join_b.k + FROM offset_join_a, offset_join_b + ORDER BY offset_join_a.k, offset_join_b.k + LIMIT 10 OFFSET 2; + CREATE TABLE offset_join_dest(x, y); + INSERT INTO offset_join_dest + SELECT offset_join_a.k, offset_join_b.k + FROM offset_join_a, offset_join_b + ORDER BY offset_join_a.k, offset_join_b.k + LIMIT 10 OFFSET 2; + SELECT x, y FROM offset_join_dest ORDER BY x, y; +} +expect { + 2|1 + 2|2 + 3|1 + 3|2 + 2|1 + 2|2 + 3|1 + 3|2 +} + # https://github.com/tursodatabase/turso/issues/3300 @cross-check-integrity test select-ungrouped-aggregate-with-offset-limit { @@ -103,4 +149,3 @@ test offset-expr-invalid-data-type-3 { expect error { datatype mismatch } - diff --git a/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_a_desc_uses_sorter.snap b/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_a_desc_uses_sorter.snap index 46301213..8ab3de04 100644 --- a/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_a_desc_uses_sorter.snap +++ b/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_a_desc_uses_sorter.snap @@ -22,7 +22,7 @@ addr opcode p1 p2 p3 p4 p5 comment 4 Column 1 0 3 0 r[3]=t.a 5 MakeRecord 3 1 2 0 r[2]=mkrec(r[3..3]) 6 SorterInsert 0 2 0 0 0 key=r[2] - 7 Next 1 4 0 0 + 7 Next 1 4 0 1 8 OpenPseudo 2 2 1 0 1 columns in r[2] 9 SorterSort 0 14 0 0 10 SorterData 0 2 2 0 r[2]=data diff --git a/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_a_uses_sorter.snap b/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_a_uses_sorter.snap index 83f77004..167b13ca 100644 --- a/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_a_uses_sorter.snap +++ b/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_a_uses_sorter.snap @@ -22,7 +22,7 @@ addr opcode p1 p2 p3 p4 p5 comment 4 Column 1 0 3 0 r[3]=t.a 5 MakeRecord 3 1 2 0 r[2]=mkrec(r[3..3]) 6 SorterInsert 0 2 0 0 0 key=r[2] - 7 Next 1 4 0 0 + 7 Next 1 4 0 1 8 OpenPseudo 2 2 1 0 1 columns in r[2] 9 SorterSort 0 14 0 0 10 SorterData 0 2 2 0 r[2]=data diff --git a/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_rowid_desc_no_sorter.snap b/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_rowid_desc_no_sorter.snap index 6964b76b..c6f39c2b 100644 --- a/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_rowid_desc_no_sorter.snap +++ b/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_rowid_desc_no_sorter.snap @@ -19,7 +19,7 @@ addr opcode p1 p2 p3 p4 p5 comment 2 Last 0 6 0 0 3 Column 0 0 1 0 r[1]=t.a 4 ResultRow 1 1 0 0 output=r[1] - 5 Prev 0 3 0 0 + 5 Prev 0 3 0 1 6 Halt 0 0 0 0 7 Transaction 0 1 1 0 iDb=0 tx_mode=Read 8 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_rowid_no_sorter.snap b/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_rowid_no_sorter.snap index cdf845f1..0ff8e9b9 100644 --- a/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_rowid_no_sorter.snap +++ b/conformance/sqlite-sqltests/orderby/snapshots/orderby_plan__orderby_rowid_no_sorter.snap @@ -19,7 +19,7 @@ addr opcode p1 p2 p3 p4 p5 comment 2 Rewind 0 6 0 0 Rewind table t 3 Column 0 0 1 0 r[1]=t.a 4 ResultRow 1 1 0 0 output=r[1] - 5 Next 0 3 0 0 + 5 Next 0 3 0 1 6 Halt 0 0 0 0 7 Transaction 0 1 1 0 iDb=0 tx_mode=Read 8 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/page1.sqltest b/conformance/sqlite-sqltests/page1.sqltest new file mode 100644 index 00000000..1881b29b --- /dev/null +++ b/conformance/sqlite-sqltests/page1.sqltest @@ -0,0 +1,16 @@ +@database :memory: + +@cross-check-integrity +test page1-intersect-on-fresh-database { + values (1) intersect values (1); +} +expect { + 1 +} + +@cross-check-integrity +test page1-group-concat-window-over-empty-sqlite-schema { + select group_concat(name) over () from sqlite_schema; +} +expect { +} diff --git a/conformance/sqlite-sqltests/partial_idx.sqltest b/conformance/sqlite-sqltests/partial_idx.sqltest index f684e115..0c56f471 100644 --- a/conformance/sqlite-sqltests/partial_idx.sqltest +++ b/conformance/sqlite-sqltests/partial_idx.sqltest @@ -59,6 +59,86 @@ expect error { UNIQUE constraint failed: products.sku } +@cross-check-integrity +test partial-index-where-nocase-column-refuses-duplicate { + CREATE TABLE t (s TEXT COLLATE NOCASE); + CREATE UNIQUE INDEX i ON t(s) WHERE s = 'a'; + INSERT INTO t VALUES ('A'); + INSERT INTO t VALUES ('a'); +} +expect error { + UNIQUE constraint failed: t.s +} + +@cross-check-integrity +test partial-index-where-rtrim-column-refuses-duplicate { + CREATE TABLE t (s TEXT COLLATE RTRIM); + CREATE UNIQUE INDEX i ON t(s) WHERE s = 'a'; + INSERT INTO t VALUES ('a '); + INSERT INTO t VALUES ('a'); +} +expect error { + UNIQUE constraint failed: t.s +} + +@cross-check-integrity +test partial-index-where-nocase-column-other-than-indexed { + CREATE TABLE t (k INTEGER, s TEXT COLLATE NOCASE); + CREATE UNIQUE INDEX i ON t(k) WHERE s = 'a'; + INSERT INTO t VALUES (1, 'A'); + INSERT INTO t VALUES (1, 'a'); +} +expect error { + UNIQUE constraint failed: t.k +} + +@cross-check-integrity +test partial-index-where-nocase-column-update-into-predicate { + CREATE TABLE t (s TEXT COLLATE NOCASE); + CREATE UNIQUE INDEX i ON t(s) WHERE s = 'a'; + INSERT INTO t VALUES ('z'); + UPDATE t SET s = 'A'; + SELECT s FROM t WHERE s = 'a'; +} +expect { + A +} + +@cross-check-integrity +test partial-index-where-binary-column-keeps-binary { + CREATE TABLE t (s TEXT); + CREATE UNIQUE INDEX i ON t(s) WHERE s = 'a'; + INSERT INTO t VALUES ('A'); + INSERT INTO t VALUES ('a'); + SELECT count(*) FROM t; +} +expect { + 2 +} + +@cross-check-integrity +test partial-index-where-explicit-collate-beats-column { + CREATE TABLE t (s TEXT COLLATE NOCASE); + CREATE UNIQUE INDEX i ON t(s) WHERE s = 'a' COLLATE BINARY; + INSERT INTO t VALUES ('A'); + INSERT INTO t VALUES ('a'); + SELECT count(*) FROM t; +} +expect { + 2 +} + +@cross-check-integrity +test expression-index-nocase-column-refuses-duplicate { + CREATE TABLE t (s TEXT COLLATE NOCASE); + CREATE UNIQUE INDEX i ON t(s = 'a'); + INSERT INTO t VALUES ('A'); + INSERT INTO t VALUES ('a'); +} +expect error { + UNIQUE constraint failed +} + @cross-check-integrity test partial-index-expensive-violation-update { CREATE TABLE products (id INTEGER PRIMARY KEY, sku TEXT, price INTEGER); @@ -1207,6 +1287,64 @@ expect { 1|0|0|SCAN samples USING INDEX samples_user_processed_idx } +# Use the partial index when one side of its OR also constrains the index key. +@skip-if sqlite "query plan explanation output is different in sqlite" +test partial-index-or-term-with-equality-search { + CREATE TABLE t (a, b); + CREATE INDEX idx ON t(a) WHERE b < 5 OR a > 5; + INSERT INTO t VALUES (-1, 1), (-3, 3), (-5, 5), (-8, 8), (10, 8); + EXPLAIN QUERY PLAN SELECT a FROM t WHERE b < 5 AND a = -3; + SELECT a FROM t WHERE b < 5 AND a = -3; +} +expect { + 1|0|0|SEARCH t USING INDEX idx (a=?) + -3 +} + +# Use the partial index when it provides the requested order. +@skip-if sqlite "query plan explanation output is different in sqlite" +test partial-index-or-term-with-order { + CREATE TABLE t (a, b); + CREATE INDEX idx ON t(a) WHERE b < 5 OR a > 5; + INSERT INTO t VALUES (-1, 1), (-3, 3), (-5, 5), (-8, 8), (10, 8); + EXPLAIN QUERY PLAN SELECT a FROM t WHERE b < 5 ORDER BY a; + SELECT a FROM t WHERE b < 5 ORDER BY a; +} +expect { + 1|0|0|SCAN t USING INDEX idx + -3 + -1 +} + +# Use the partial index without table lookups when it contains the filtered column. +@skip-if sqlite "query plan explanation output is different in sqlite" +test partial-index-or-term-with-covering-scan { + CREATE TABLE t (a, b); + CREATE INDEX idx ON t(a, b) WHERE b < 5 OR a > 5; + INSERT INTO t VALUES (-1, 1), (-3, 3), (-5, 5), (-8, 8), (10, 8); + EXPLAIN QUERY PLAN SELECT a FROM t WHERE b < 5; + SELECT a FROM t WHERE b < 5 ORDER BY a; +} +expect { + 1|0|0|SCAN t USING COVERING INDEX idx + -3 + -1 +} + +# Use either side of the partial index OR to prove that the index is safe. +@skip-if sqlite "query plan explanation output is different in sqlite" +test partial-index-or-right-term-with-range-search { + CREATE TABLE t (a, b); + CREATE INDEX idx ON t(a) WHERE b < 5 OR a > 5; + INSERT INTO t VALUES (-1, 1), (-3, 3), (-5, 5), (-8, 8), (10, 8); + EXPLAIN QUERY PLAN SELECT a FROM t WHERE a > 5; + SELECT a FROM t WHERE a > 5 ORDER BY a; +} +expect { + 1|0|0|SEARCH t USING COVERING INDEX idx (a>?) + 10 +} + # Partial index must NOT be used when the query only satisfies a subset of the # index WHERE conjuncts — the partial index would skip rows the query needs. @skip-if sqlite "query plan explanation output is different in sqlite" @@ -1596,4 +1734,25 @@ test eqp-use-partial-idx-predicate-with-residual-predicate-for-covering-index-sc } expect { 1|0|0|SEARCH t USING COVERING INDEX idx (a=?) -} \ No newline at end of file +} + +# An outer join condition does not filter rows from its left table. +@cross-check-integrity +test outer-join-condition-does-not-enable-left-table-partial-index { + CREATE TABLE items(name TEXT, group_name TEXT); + INSERT INTO items VALUES ('one', 'keep'), ('two', 'keep'); + CREATE INDEX only_missing ON items(name, group_name) + WHERE name = 'missing'; + CREATE TABLE optional(name TEXT); + + SELECT items.name + FROM items + LEFT JOIN optional + ON optional.name = items.name AND items.name = 'missing' + WHERE items.group_name = 'keep' + ORDER BY items.name; +} +expect { + one + two +} diff --git a/conformance/sqlite-sqltests/pragma-name-case-insensitive.sqltest b/conformance/sqlite-sqltests/pragma-name-case-insensitive.sqltest new file mode 100644 index 00000000..90bdc90d --- /dev/null +++ b/conformance/sqlite-sqltests/pragma-name-case-insensitive.sqltest @@ -0,0 +1,56 @@ +@database :memory: + +# Regression test for https://github.com/tursodatabase/turso/issues/8736 +# SQLite matches PRAGMA names without regard to case, so PRAGMA FOREIGN_KEYS +# and PRAGMA foreign_keys name the same pragma. + +test pragma-uppercase-setter-is-applied { + PRAGMA USER_VERSION = 9; + PRAGMA user_version; +} +expect { + 9 +} + +test pragma-mixed-case-reader-sees-value { + PRAGMA user_version = 9; + PRAGMA User_Version; +} +expect { + 9 +} + +test pragma-lowercase-still-works { + PRAGMA user_version = 7; + PRAGMA USER_VERSION; +} +expect { + 7 +} + +test pragma-uppercase-table-info-returns-columns { + CREATE TABLE t1(a); + PRAGMA TABLE_INFO(t1); +} +expect { + 0|a||0||0 +} + +test pragma-uppercase-foreign-keys-enforces { + PRAGMA FOREIGN_KEYS = ON; + CREATE TABLE t1(a INTEGER PRIMARY KEY); + CREATE TABLE t2(b INTEGER REFERENCES t1(a)); + INSERT INTO t2 VALUES(999); +} +expect error { + FOREIGN KEY constraint failed +} + +test unknown-pragma-in-any-case-is-ignored { + PRAGMA NOT_A_REAL_PRAGMA = 1; + PRAGMA Not_A_Real_Pragma; + SELECT 1; +} +expect { + 1 +} diff --git a/conformance/sqlite-sqltests/pragma/memory.sqltest b/conformance/sqlite-sqltests/pragma/memory.sqltest index 3861fbf6..d7609bcd 100644 --- a/conformance/sqlite-sqltests/pragma/memory.sqltest +++ b/conformance/sqlite-sqltests/pragma/memory.sqltest @@ -24,6 +24,36 @@ expect { -2000 } +@cross-check-integrity +test pragma-attached-database-has-its-own-synchronous-mode { + PRAGMA main.synchronous=OFF; + ATTACH ':memory:' AS aux; + PRAGMA main.synchronous; + PRAGMA aux.synchronous; + PRAGMA aux.synchronous=NORMAL; + PRAGMA main.synchronous; + PRAGMA aux.synchronous; +} +expect { + 0 + 2 + 0 + 1 +} + +@cross-check-integrity +test pragma-temp-synchronous-stays-off { + PRAGMA temp.synchronous; + PRAGMA temp.synchronous=FULL; + PRAGMA temp.synchronous; + PRAGMA main.synchronous; +} +expect { + 0 + 0 + 2 +} + test pragma-function-cache-size { SELECT * FROM pragma_cache_size() } @@ -293,5 +323,3 @@ test pragma-page-size-set-uninitialized-db { expect { 1024 } - - diff --git a/conformance/sqlite-sqltests/pragma_query_only_rejects_metadata_and_drop_trigger.sqltest b/conformance/sqlite-sqltests/pragma_query_only_rejects_metadata_and_drop_trigger.sqltest new file mode 100644 index 00000000..d219e339 --- /dev/null +++ b/conformance/sqlite-sqltests/pragma_query_only_rejects_metadata_and_drop_trigger.sqltest @@ -0,0 +1,52 @@ +@database :memory: + +# Regression test for https://github.com/tursodatabase/turso/issues/8862 +# PRAGMA query_only=1 must refuse every statement that writes to the +# database, including schema cookie updates and DROP TRIGGER. + +setup table_with_trigger { + CREATE TABLE items(id INTEGER PRIMARY KEY); + CREATE TABLE audit(id INTEGER); + CREATE TRIGGER changed AFTER INSERT ON items BEGIN INSERT INTO audit VALUES (new.id); END; +} + +@setup table_with_trigger +test query-only-rejects-insert { + PRAGMA query_only = 1; + INSERT INTO items VALUES (1); +} +expect error { +} + +@setup table_with_trigger +test query-only-rejects-user-version-write { + PRAGMA query_only = 1; + PRAGMA user_version = 123; +} +expect error { +} + +@setup table_with_trigger +test query-only-rejects-application-id-write { + PRAGMA query_only = 1; + PRAGMA application_id = 123; +} +expect error { +} + +@setup table_with_trigger +test query-only-rejects-drop-trigger { + PRAGMA query_only = 1; + DROP TRIGGER changed; +} +expect error { +} + +@setup table_with_trigger +test query-only-still-allows-reading-user-version { + PRAGMA query_only = 1; + PRAGMA user_version; +} +expect { + 0 +} diff --git a/conformance/sqlite-sqltests/row-value-in.sqltest b/conformance/sqlite-sqltests/row-value-in.sqltest index e33e9455..a7b1dd36 100644 --- a/conformance/sqlite-sqltests/row-value-in.sqltest +++ b/conformance/sqlite-sqltests/row-value-in.sqltest @@ -161,3 +161,130 @@ expect { m5|-1 m6|-1 } + +setup row_values_with_null_schema { + CREATE TABLE w(a, b); + INSERT INTO w VALUES (NULL,1),(1,1),(2,NULL),(2,2); +} + +@setup row_values_with_null_schema +test row-value-filter-nulls-literal-list-in { + SELECT a, b FROM w WHERE (a,b) IN ((1,1),(2,2)) ORDER BY 1,2; +} +expect { + 1|1 + 2|2 +} + +@setup row_values_with_null_schema +test row-value-filter-nulls-literal-list-not-in { + SELECT a, b FROM w WHERE (a,b) NOT IN ((1,1),(2,2)) ORDER BY 1,2; +} +expect { +} + +@setup row_values_with_null_schema +test row-value-filter-nulls-literal-list-containing-null-in { + SELECT a, b FROM w WHERE (a,b) IN ((2,NULL),(2,2)) ORDER BY 1,2; +} +expect { + 2|2 +} + +test row-value-false-null-first-in { + SELECT ((NULL,1) IN ((1,1))) IS NULL; +} +expect { + 1 +} + +test row-value-false-null-second-in { + SELECT ((2,NULL) IN ((2,2))) IS NULL; +} +expect { + 1 +} + +test row-value-false-null-second-not-in { + SELECT ((2,NULL) NOT IN ((2,2))) IS NULL; +} +expect { + 1 +} + +test row-value-false-null-three-values-in { + SELECT ((1,NULL,3) IN ((1,2,3))) IS NULL; +} +expect { + 1 +} + +test row-value-false-null-multiple-tuples-in { + SELECT ((NULL,1) IN ((1,1),(9,9))) IS NULL; +} +expect { + 1 +} + +test row-value-false-null-multiple-tuples-with-null-in { + SELECT ((2,2) IN ((2,NULL),(9,9))) IS NULL; +} +expect { + 1 +} + +test row-value-false-null-multiple-tuples-with-null-not-in { + SELECT ((2,2) NOT IN ((2,NULL),(9,9))) IS NULL; +} +expect { + 1 +} + +test row-value-false-null-check-inequality-after-null-in { + SELECT ((1,2,3) IN ((1,NULL,9))) IS FALSE; +} +expect { + 1 +} + +test row-value-false-null-check-matches-after-null-in { + SELECT ((2,2) IN ((2,NULL),(2,2))) IS TRUE; +} +expect { + 1 +} + +test row-value-null-in-lhs-resolves-to-false-on-mismatch-after-in { + SELECT (NULL,1) IN ((2,2)) IS FALSE; +} +expect { + 1 +} + +test row-value-null-in-lhs-resolves-to-true-on-mismatch-after-not-in { + SELECT (NULL,1) NOT IN ((2,2)) IS TRUE; +} +expect { + 1 +} + +test row-value-mismatch-in-lhs-resolves-to-false-on-null-after-in { + SELECT (1,NULL) IN ((2,2)) IS FALSE; +} +expect { + 1 +} + +test row-value-null-in-lhs-resolves-to-false-on-mismatch-after-multiple-rhs-in { + SELECT (NULL,1) IN ((2,2),(3,3)) IS FALSE; +} +expect { + 1 +} + +test row-value-null-in-both-lhs-rhs-in { + SELECT (NULL,1) IN ((NULL,2)) IS FALSE; +} +expect { + 1 +} \ No newline at end of file diff --git a/conformance/sqlite-sqltests/scalar-functions-datetime.sqltest b/conformance/sqlite-sqltests/scalar-functions-datetime.sqltest index 13e36a8b..1f78edb9 100644 --- a/conformance/sqlite-sqltests/scalar-functions-datetime.sqltest +++ b/conformance/sqlite-sqltests/scalar-functions-datetime.sqltest @@ -29,6 +29,13 @@ expect { 2023-05-18 } +test date-unixepoch-at-1-millisecond-before-epochtime { + SELECT date(unixepoch('1969-12-31T23:59:59.999Z'), 'unixepoch'); +} +expect { + 1969-12-31 +} + test date-with-milliseconds { SELECT date('2023-05-18 15:30:45.123'); } @@ -639,6 +646,13 @@ expect { -1 } +test unixepoch-at-1-millisecond-before-epochtime { + SELECT unixepoch('1969-12-31 23:59:59.999'); +} +expect { + -1 +} + test unixepoch-at-future { SELECT unixepoch('9999-12-31 23:59:59'); } @@ -1556,6 +1570,13 @@ expect { 1737638070 } +test strftime-at-1-millisecond-before-epochtime { + SELECT strftime('%s', '1969-12-31T23:59:59.999'); +} +expect { + -1 +} + test strftime-with-subsec { SELECT strftime('%s', '2000-01-01 12:00:00.123', 'subsec'); } @@ -2558,3 +2579,45 @@ test unixepoch-just-past-lower-boundary { } expect { } + +test date-blob-time-value { + SELECT date(CAST('2024-01-01' AS BLOB)); +} +expect { + 2024-01-01 +} + +test unixepoch-blob-time-value { + SELECT unixepoch(X'323032342D30312D3031'); +} +expect { + 1704067200 +} + +test date-blob-modifier { + SELECT date('2024-06-15', X'2B3120646179'); +} +expect { + 2024-06-16 +} + +test strftime-blob-time-value { + SELECT strftime('%Y-%m-%d', X'323032342D30312D3031'); +} +expect { + 2024-01-01 +} + +test timediff-blob-time-values { + SELECT timediff(X'323032342D30312D3032', X'323032342D30312D3031'); +} +expect { + +0000-00-01 00:00:00.000 +} + +test date-blob-that-is-not-utf8 { + SELECT typeof(date(X'FF00')); +} +expect { + null +} diff --git a/conformance/sqlite-sqltests/scalar-functions.sqltest b/conformance/sqlite-sqltests/scalar-functions.sqltest index cf1a2fa9..491e47ef 100644 --- a/conformance/sqlite-sqltests/scalar-functions.sqltest +++ b/conformance/sqlite-sqltests/scalar-functions.sqltest @@ -1,6 +1,31 @@ @database :default: @database :default-no-rowidalias: +test abs-integer-boundaries { + SELECT abs(-9223372036854775807), abs(9223372036854775807), abs(0), abs(NULL) IS NULL; +} +expect { + 9223372036854775807|9223372036854775807|0|1 +} + +test abs-integer-overflow { + SELECT abs(-9223372036854775808); +} +expect error { + integer overflow +} + +test scalar-result-followed-by-null { + WITH t(id, v) AS (VALUES (1, 'ABC'), (2, NULL), (3, 'DEF')) + SELECT id, coalesce(lower(v), '-'), coalesce(sign(v), 9), + coalesce(length(v), -1) FROM t ORDER BY id; +} +expect { + 1|abc|9|3 + 2|-|9|-1 + 3|def|9|3 +} + test concat-chars { select concat('l', 'i'); } @@ -1058,6 +1083,23 @@ expect { 1 } +test min-int-equals-float { + select min(1, 1.0) +} +expect { + 1.0 +} +expect @js { + 1 +} + +test min-float-equals-int { + select min(1.0, 1) +} +expect { + 1 +} + test max-int-float { select max(1,5.0) } @@ -1114,6 +1156,39 @@ expect { 0 } +# upper, lower and unhex each read their argument as text, so every value type +# has to reach the same answer as SQLite. +test upper-lower-unhex-of-every-type { + SELECT quote(upper('abc')); + SELECT quote(upper(123)); + SELECT quote(upper(1.25)); + SELECT quote(upper(NULL)); + SELECT quote(upper(x'6162')); + SELECT quote(upper('héllo')); + SELECT quote(lower('ABC')); + SELECT quote(lower(x'4142')); + SELECT quote(lower(NULL)); + SELECT quote(unhex('4142')); + SELECT quote(unhex(4142)); + SELECT quote(unhex(NULL)); + SELECT quote(unhex('41zz')); +} +expect { + 'ABC' + '123' + '1.25' + NULL + 'AB' + 'HéLLO' + 'abc' + 'ab' + NULL + X'4142' + X'4142' + NULL + NULL +} + test substr-3-args { SELECT substr('limbo', 1, 3); } diff --git a/conformance/sqlite-sqltests/simple-count-optimization.sqltest b/conformance/sqlite-sqltests/simple-count-optimization.sqltest index 0a92fc0f..a935e3d9 100644 --- a/conformance/sqlite-sqltests/simple-count-optimization.sqltest +++ b/conformance/sqlite-sqltests/simple-count-optimization.sqltest @@ -109,3 +109,36 @@ expect { 1 1 } + +@skip-if sqlite "Deliberately break up with sqlite here to return the actual value instead of error" +test simple-count-unusable-partial-index-column-correctness { + CREATE TABLE t(x NOT NULL); + CREATE INDEX pi ON t(x) WHERE x > 5; + INSERT INTO t VALUES (9), (1), (2); + SELECT count(x) FROM t INDEXED BY pi; +} +expect { + 3 +} + +test simple-count-correlated-outer-join-null-extension { + CREATE TABLE t0(z); + CREATE TABLE t1(a INTEGER NOT NULL); + CREATE TABLE t2(b); + INSERT INTO t0 VALUES (1); + INSERT INTO t2 VALUES (1),(2); + SELECT (SELECT count(t2.rowid + t1.a) FROM t2) FROM t0 LEFT JOIN t1 ON t1.a = t0.z; +} +expect { + 0 +} + +@skip-if sqlite "sqlite errors on an invalid regexp pattern instead of returning NULL" +test simple-count-regexp-invalid-pattern-correctness { + CREATE TABLE t(x TEXT NOT NULL); + INSERT INTO t VALUES ('a'),('b'); + SELECT count(x REGEXP '(') FROM t; +} +expect { + 0 +} diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/aggregation.sqltest b/conformance/sqlite-sqltests/snapshot_tests/aggregation/aggregation.sqltest index 50cfd180..f0f82de5 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/aggregation.sqltest +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/aggregation.sqltest @@ -347,6 +347,17 @@ snapshot count-column { SELECT count(x) FROM t1; } +setup simple_table_notnull { + CREATE TABLE t3(x NOT NULL); +} + +# count(col) can use the Count opcode when col is NOT NULL, since it can +# never skip a row +@setup simple_table_notnull +snapshot count-notnull-column { + SELECT count(x) FROM t3; +} + # Mixed case must also use the Count opcode @setup simple_table snapshot simple-count-mixed-case { diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__aggregation-with-join.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__aggregation-with-join.snap index 1801d39e..b3396463 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__aggregation-with-join.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__aggregation-with-join.snap @@ -31,13 +31,13 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 78 0 0 Start at 78 + 0 Init 0 75 0 0 Start at 75 1 SorterOpen 0 1 0 k(1,-B) 0 cursor=0 2 Null 0 11 13 0 r[11..13]=NULL 3 SorterOpen 1 4 0 k(1,-B) 0 cursor=1 4 Integer 0 8 0 0 r[8]=0; clear group by abort flag 5 Null 0 9 0 0 r[9]=NULL; initialize group by comparison registers to NULL - 6 Gosub 19 65 0 0 ; go to clear accumulator subroutine + 6 Gosub 19 63 0 0 ; go to clear accumulator subroutine 7 OpenRead 3 6 0 k(4,B,B,B,B) 0 table=products, root=6, iDb=0 8 OpenRead 4 7 0 k(6,B,B,B,B,B,B) 0 table=orders, root=7, iDb=0 9 OpenRead 5 9 0 k(2,B) 0 index=idx_orders_product, root=9, iDb=0 @@ -59,55 +59,52 @@ addr opcode p1 p2 p3 p4 p5 comment 25 NullRow 4 0 0 0 Set cursor 4 to a (pseudo) NULL row 26 NullRow 5 0 0 0 Set cursor 5 to a (pseudo) NULL row 27 Goto 0 16 0 0 - 28 Next 3 11 0 0 + 28 Next 3 11 0 1 29 OpenPseudo 2 14 4 0 4 columns in r[14] - 30 SorterSort 1 49 0 0 + 30 SorterSort 1 47 0 0 31 SorterData 1 14 2 0 r[14]=data 32 Column 2 0 22 0 r[22]=pseudo.column 0 33 Compare 9 22 1 k(1, Binary) 0 r[9..9]==r[22..22] 34 Jump 35 39 35 0 ; start new group if comparison is not equal - 35 Gosub 6 53 0 0 ; check if ended group had data, and output if so + 35 Gosub 6 51 0 0 ; check if ended group had data, and output if so 36 Move 22 9 1 0 r[9..9]=r[22..22] - 37 IfPos 8 68 0 0 r[8]>0 -> r[8]-=0, goto 68; check abort flag - 38 Gosub 19 65 0 0 ; goto clear accumulator subroutine - 39 Column 2 1 23 0 r[23]=pseudo.column 1 - 40 Column 2 2 24 0 r[24]=pseudo.column 2 - 41 Column 2 3 25 0 r[25]=pseudo.column 3 - 42 AggStep 0 23 11 count 0 accum=r[11] step(r[23]) - 43 AggStep 0 24 12 sum 0 accum=r[12] step(r[24]) - 44 AggStep 0 25 13 avg 0 accum=r[13] step(r[25]) - 45 If 7 47 0 0 if r[7] goto 47; don't emit group columns if continuing existing group - 46 Column 2 0 10 0 r[10]=pseudo.column 0 - 47 Integer 1 7 0 0 r[7]=1; indicate data in accumulator - 48 SorterNext 1 31 0 0 - 49 Gosub 6 53 0 0 ; emit row for final group - 50 Goto 0 68 0 0 ; group by finished - 51 Integer 1 8 0 0 r[8]=1 - 52 Return 6 0 0 0 - 53 IfPos 7 55 0 0 r[7]>0 -> r[7]-=0, goto 55; output group by row subroutine start - 54 Return 6 0 0 0 - 55 AggFinal 0 11 0 count 0 accum=r[11] - 56 AggFinal 0 12 0 sum 0 accum=r[12] - 57 AggFinal 0 13 0 avg 0 accum=r[13] - 58 Copy 12 26 0 0 r[26]=r[12] - 59 Copy 10 27 0 0 r[27]=r[10] - 60 Copy 11 28 0 0 r[28]=r[11] - 61 Copy 13 29 0 0 r[29]=r[13] - 62 MakeRecord 26 4 5 0 r[5]=mkrec(r[26..29]) - 63 SorterInsert 0 5 0 0 0 key=r[5] - 64 Return 6 0 0 0 - 65 Null 0 10 13 0 r[10..13]=NULL; clear accumulator subroutine start - 66 Integer 0 7 0 0 r[7]=0 - 67 Return 19 0 0 0 - 68 OpenPseudo 6 5 4 0 4 columns in r[5] - 69 SorterSort 0 77 0 0 - 70 SorterData 0 5 6 0 r[5]=data - 71 Column 6 1 1 0 r[1]=pseudo.column 1 - 72 Column 6 2 2 0 r[2]=pseudo.column 2 - 73 Column 6 0 3 0 r[3]=pseudo.column 0 - 74 Column 6 3 4 0 r[4]=pseudo.column 3 - 75 ResultRow 1 4 0 0 output=r[1..4] - 76 SorterNext 0 70 0 0 - 77 Halt 0 0 0 0 - 78 Transaction 0 1 8 0 iDb=0 tx_mode=Read - 79 Goto 0 1 0 0 + 37 IfPos 8 66 0 0 r[8]>0 -> r[8]-=0, goto 66; check abort flag + 38 Gosub 19 63 0 0 ; goto clear accumulator subroutine + 39 ColumnRange 2 1 23 3 0 r[23..25]=pseudo.column 1..column 3 + 40 AggStep 0 23 11 count 0 accum=r[11] step(r[23]) + 41 AggStep 0 24 12 sum 0 accum=r[12] step(r[24]) + 42 AggStep 0 25 13 avg 0 accum=r[13] step(r[25]) + 43 If 7 45 0 0 if r[7] goto 45; don't emit group columns if continuing existing group + 44 Column 2 0 10 0 r[10]=pseudo.column 0 + 45 Integer 1 7 0 0 r[7]=1; indicate data in accumulator + 46 SorterNext 1 31 0 0 + 47 Gosub 6 51 0 0 ; emit row for final group + 48 Goto 0 66 0 0 ; group by finished + 49 Integer 1 8 0 0 r[8]=1 + 50 Return 6 0 0 0 + 51 IfPos 7 53 0 0 r[7]>0 -> r[7]-=0, goto 53; output group by row subroutine start + 52 Return 6 0 0 0 + 53 AggFinal 0 11 0 count 0 accum=r[11] + 54 AggFinal 0 12 0 sum 0 accum=r[12] + 55 AggFinal 0 13 0 avg 0 accum=r[13] + 56 Copy 12 26 0 0 r[26]=r[12] + 57 Copy 10 27 0 0 r[27]=r[10] + 58 Copy 11 28 0 0 r[28]=r[11] + 59 Copy 13 29 0 0 r[29]=r[13] + 60 MakeRecord 26 4 5 0 r[5]=mkrec(r[26..29]) + 61 SorterInsert 0 5 0 0 0 key=r[5] + 62 Return 6 0 0 0 + 63 Null 0 10 13 0 r[10..13]=NULL; clear accumulator subroutine start + 64 Integer 0 7 0 0 r[7]=0 + 65 Return 19 0 0 0 + 66 OpenPseudo 6 5 4 0 4 columns in r[5] + 67 SorterSort 0 74 0 0 + 68 SorterData 0 5 6 0 r[5]=data + 69 ColumnRange 6 1 1 2 0 r[1..2]=pseudo.column 1..column 2 + 70 Column 6 0 3 0 r[3]=pseudo.column 0 + 71 Column 6 3 4 0 r[4]=pseudo.column 3 + 72 ResultRow 1 4 0 0 output=r[1..4] + 73 SorterNext 0 68 0 0 + 74 Halt 0 0 0 0 + 75 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 76 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__column-free-scan-covering-index.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__column-free-scan-covering-index.snap index e029b971..38f00dd8 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__column-free-scan-covering-index.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__column-free-scan-covering-index.snap @@ -18,7 +18,7 @@ addr opcode p1 p2 p3 p4 p5 comment 1 OpenRead 0 4 0 k(2,B) 0 index=indexed_count_narrow_idx, root=4, iDb=0 2 Rewind 0 5 0 0 Rewind index indexed_count_narrow_idx 3 ResultRow 1 1 0 0 output=r[1] - 4 Next 0 3 0 0 + 4 Next 0 3 0 1 5 Halt 0 0 0 0 6 Transaction 0 1 4 0 iDb=0 tx_mode=Read 7 Integer 1 1 0 0 r[1]=1 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-column.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-column.snap index 34c555e4..2622de5a 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-column.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-column.snap @@ -20,7 +20,7 @@ addr opcode p1 p2 p3 p4 p5 comment 3 Rewind 0 7 0 0 Rewind table t1 4 Column 0 0 3 0 r[3]=t1.x 5 AggStep 0 3 2 count 0 accum=r[2] step(r[3]) - 6 Next 0 4 0 0 + 6 Next 0 4 0 1 7 AggFinal 0 2 0 count 0 accum=r[2] 8 Copy 2 1 0 0 r[1]=r[2] 9 ResultRow 1 1 0 0 output=r[1] diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-distinct.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-distinct.snap index cf118261..92be9b32 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-distinct.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-distinct.snap @@ -55,7 +55,7 @@ addr opcode p1 p2 p3 p4 p5 commen 25 If 6 27 0 0 if r[6] goto 27; don't emit group columns if continuing existing group 26 Column 2 0 9 0 r[9]=idx_sales_category.category 27 Integer 1 6 0 0 r[6]=1; indicate data in accumulator - 28 Next 2 11 0 0 + 28 Next 2 11 0 1 29 Gosub 5 33 0 0 ; emit row for final group 30 Goto 0 48 0 0 ; group by finished 31 Integer 1 7 0 0 r[7]=1 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-notnull-column.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-notnull-column.snap new file mode 100644 index 00000000..d791b56a --- /dev/null +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-notnull-column.snap @@ -0,0 +1,26 @@ +--- +source: aggregation.sqltest +expression: SELECT count(x) FROM t3; +info: + statement_type: SELECT + tables: + - t3 + setup_blocks: + - simple_table_notnull + database: ':memory:' +--- +QUERY PLAN +`--SCAN t3 + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 8 0 0 Start at 8 + 1 Null 0 2 0 0 r[2]=NULL + 2 OpenRead 0 2 0 k(2,B) 0 table=t3, root=2, iDb=0 + 3 Count 0 3 0 0 + 4 Close 0 0 0 0 + 5 Copy 3 1 0 0 r[1]=r[3] + 6 ResultRow 1 1 0 0 output=r[1] + 7 Halt 0 0 0 0 + 8 Transaction 0 1 1 0 iDb=0 tx_mode=Read + 9 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-star-plus-expr.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-star-plus-expr.snap index 6095d2d1..4dcea2ca 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-star-plus-expr.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__count-star-plus-expr.snap @@ -19,7 +19,7 @@ addr opcode p1 p2 p3 p4 p5 comment 2 OpenRead 0 2 0 k(2,B) 0 table=t1, root=2, iDb=0 3 Rewind 0 6 0 0 Rewind table t1 4 AggStep 0 3 2 count 0 accum=r[2] step(r[3]) - 5 Next 0 4 0 0 + 5 Next 0 4 0 1 6 AggFinal 0 2 0 count 0 accum=r[2] 7 Copy 2 4 0 0 r[4]=r[2] 8 Add 4 5 1 0 r[1]=r[4]+r[5] diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__distinct-count-with-join.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__distinct-count-with-join.snap index 9fbefa94..96519cd4 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__distinct-count-with-join.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__distinct-count-with-join.snap @@ -32,13 +32,13 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 68 0 0 Start at 68 + 0 Init 0 67 0 0 Start at 67 1 SorterOpen 0 1 0 k(1,-B) 0 cursor=0 2 Null 0 10 11 0 r[10..11]=NULL 3 SorterOpen 1 3 0 k(1,-B) 0 cursor=1 4 Integer 0 7 0 0 r[7]=0; clear group by abort flag 5 Null 0 8 0 0 r[8]=NULL; initialize group by comparison registers to NULL - 6 Gosub 16 54 0 0 ; go to clear accumulator subroutine + 6 Gosub 16 53 0 0 ; go to clear accumulator subroutine 7 HashClear 1073741824 0 0 0 8 HashClear 1073741825 0 0 0 9 OpenRead 3 6 0 k(4,B,B,B,B) 0 table=products, root=6, iDb=0 @@ -51,54 +51,53 @@ addr opcode p1 p2 p3 p4 p5 comment 16 RowId 4 15 0 0 r[15]=orders.rowid 17 MakeRecord 13 3 12 0 r[12]=mkrec(r[13..15]) 18 SorterInsert 1 12 0 0 0 key=r[12] - 19 Next 4 12 0 0 + 19 Next 4 12 0 1 20 OpenPseudo 2 12 3 0 3 columns in r[12] - 21 SorterSort 1 40 0 0 + 21 SorterSort 1 39 0 0 22 SorterData 1 12 2 0 r[12]=data 23 Column 2 0 18 0 r[18]=pseudo.column 0 24 Compare 8 18 1 k(1, Binary) 0 r[8..8]==r[18..18] 25 Jump 26 30 26 0 ; start new group if comparison is not equal - 26 Gosub 5 44 0 0 ; check if ended group had data, and output if so + 26 Gosub 5 43 0 0 ; check if ended group had data, and output if so 27 Move 18 8 1 0 r[8..8]=r[18..18] - 28 IfPos 7 59 0 0 r[7]>0 -> r[7]-=0, goto 59; check abort flag - 29 Gosub 16 54 0 0 ; goto clear accumulator subroutine - 30 Column 2 1 19 0 r[19]=pseudo.column 1 - 31 Column 2 2 20 0 r[20]=pseudo.column 2 - 32 HashDistinct 1073741824 19 1 jmp=34 0 - 33 AggStep 0 19 10 count 0 accum=r[10] step(r[19]) - 34 HashDistinct 1073741825 20 1 jmp=36 0 - 35 AggStep 0 20 11 count 0 accum=r[11] step(r[20]) - 36 If 6 38 0 0 if r[6] goto 38; don't emit group columns if continuing existing group - 37 Column 2 0 9 0 r[9]=pseudo.column 0 - 38 Integer 1 6 0 0 r[6]=1; indicate data in accumulator - 39 SorterNext 1 22 0 0 - 40 Gosub 5 44 0 0 ; emit row for final group - 41 Goto 0 59 0 0 ; group by finished - 42 Integer 1 7 0 0 r[7]=1 - 43 Return 5 0 0 0 - 44 IfPos 6 46 0 0 r[6]>0 -> r[6]-=0, goto 46; output group by row subroutine start - 45 Return 5 0 0 0 - 46 AggFinal 0 10 0 count 0 accum=r[10] - 47 AggFinal 0 11 0 count 0 accum=r[11] - 48 Copy 10 21 0 0 r[21]=r[10] - 49 Copy 9 22 0 0 r[22]=r[9] - 50 Copy 11 23 0 0 r[23]=r[11] - 51 MakeRecord 21 3 4 0 r[4]=mkrec(r[21..23]) - 52 SorterInsert 0 4 0 0 0 key=r[4] - 53 Return 5 0 0 0 - 54 Null 0 9 11 0 r[9..11]=NULL; clear accumulator subroutine start - 55 HashClear 1073741824 0 0 0 - 56 HashClear 1073741825 0 0 0 - 57 Integer 0 6 0 0 r[6]=0 - 58 Return 16 0 0 0 - 59 OpenPseudo 5 4 3 0 3 columns in r[4] - 60 SorterSort 0 67 0 0 - 61 SorterData 0 4 5 0 r[4]=data - 62 Column 5 1 1 0 r[1]=pseudo.column 1 - 63 Column 5 0 2 0 r[2]=pseudo.column 0 - 64 Column 5 2 3 0 r[3]=pseudo.column 2 - 65 ResultRow 1 3 0 0 output=r[1..3] - 66 SorterNext 0 61 0 0 - 67 Halt 0 0 0 0 - 68 Transaction 0 1 8 0 iDb=0 tx_mode=Read - 69 Goto 0 1 0 0 + 28 IfPos 7 58 0 0 r[7]>0 -> r[7]-=0, goto 58; check abort flag + 29 Gosub 16 53 0 0 ; goto clear accumulator subroutine + 30 ColumnRange 2 1 19 2 0 r[19..20]=pseudo.column 1..column 2 + 31 HashDistinct 1073741824 19 1 jmp=33 0 + 32 AggStep 0 19 10 count 0 accum=r[10] step(r[19]) + 33 HashDistinct 1073741825 20 1 jmp=35 0 + 34 AggStep 0 20 11 count 0 accum=r[11] step(r[20]) + 35 If 6 37 0 0 if r[6] goto 37; don't emit group columns if continuing existing group + 36 Column 2 0 9 0 r[9]=pseudo.column 0 + 37 Integer 1 6 0 0 r[6]=1; indicate data in accumulator + 38 SorterNext 1 22 0 0 + 39 Gosub 5 43 0 0 ; emit row for final group + 40 Goto 0 58 0 0 ; group by finished + 41 Integer 1 7 0 0 r[7]=1 + 42 Return 5 0 0 0 + 43 IfPos 6 45 0 0 r[6]>0 -> r[6]-=0, goto 45; output group by row subroutine start + 44 Return 5 0 0 0 + 45 AggFinal 0 10 0 count 0 accum=r[10] + 46 AggFinal 0 11 0 count 0 accum=r[11] + 47 Copy 10 21 0 0 r[21]=r[10] + 48 Copy 9 22 0 0 r[22]=r[9] + 49 Copy 11 23 0 0 r[23]=r[11] + 50 MakeRecord 21 3 4 0 r[4]=mkrec(r[21..23]) + 51 SorterInsert 0 4 0 0 0 key=r[4] + 52 Return 5 0 0 0 + 53 Null 0 9 11 0 r[9..11]=NULL; clear accumulator subroutine start + 54 HashClear 1073741824 0 0 0 + 55 HashClear 1073741825 0 0 0 + 56 Integer 0 6 0 0 r[6]=0 + 57 Return 16 0 0 0 + 58 OpenPseudo 5 4 3 0 3 columns in r[4] + 59 SorterSort 0 66 0 0 + 60 SorterData 0 4 5 0 r[4]=data + 61 Column 5 1 1 0 r[1]=pseudo.column 1 + 62 Column 5 0 2 0 r[2]=pseudo.column 0 + 63 Column 5 2 3 0 r[3]=pseudo.column 2 + 64 ResultRow 1 3 0 0 output=r[1..3] + 65 SorterNext 0 60 0 0 + 66 Halt 0 0 0 0 + 67 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 68 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__group-by-with-having.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__group-by-with-having.snap index f6af6c8b..af4b1a6d 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__group-by-with-having.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__group-by-with-having.snap @@ -51,7 +51,7 @@ addr opcode p1 p2 p3 p4 p5 comment 21 If 6 23 0 0 if r[6] goto 23; don't emit group columns if continuing existing group 22 Column 2 0 9 0 r[9]=idx_sales_category.category 23 Integer 1 6 0 0 r[6]=1; indicate data in accumulator - 24 Next 2 9 0 0 + 24 Next 2 9 0 1 25 Gosub 5 29 0 0 ; emit row for final group 26 Goto 0 45 0 0 ; group by finished 27 Integer 1 7 0 0 r[7]=1 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__min-max-per-group.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__min-max-per-group.snap index f5b0c632..010ca367 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__min-max-per-group.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__min-max-per-group.snap @@ -47,7 +47,7 @@ addr opcode p1 p2 p3 p4 p5 comment 20 If 5 22 0 0 if r[5] goto 22; don't emit group columns if continuing existing group 21 Column 1 0 8 0 r[8]=idx_sales_category.category 22 Integer 1 5 0 0 r[5]=1; indicate data in accumulator - 23 Next 1 8 0 0 + 23 Next 1 8 0 1 24 Gosub 4 28 0 0 ; emit row for final group 25 Goto 0 40 0 0 ; group by finished 26 Integer 1 6 0 0 r[6]=1 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__min-max-with-index.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__min-max-with-index.snap index d060e107..5b3580b8 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__min-max-with-index.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__min-max-with-index.snap @@ -27,7 +27,7 @@ addr opcode p1 p2 p3 p4 p5 comment 5 AggStep 0 5 3 min(Binary) 0 accum=r[3] step(r[5]) 6 Column 0 0 6 0 r[6]=idx_sales_amount.amount 7 AggStep 0 6 4 max(Binary) 0 accum=r[4] step(r[6]) - 8 Next 0 4 0 0 + 8 Next 0 4 0 1 9 AggFinal 0 3 0 min 0 accum=r[3] 10 AggFinal 0 4 0 max 0 accum=r[4] 11 Copy 3 1 0 0 r[1]=r[3] diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__multi-column-group-by.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__multi-column-group-by.snap index 56a2327e..1ae48a3c 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__multi-column-group-by.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__multi-column-group-by.snap @@ -27,49 +27,47 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 43 0 0 Start at 43 + 0 Init 0 41 0 0 Start at 41 1 Null 0 12 13 0 r[12..13]=NULL 2 Integer 0 7 0 0 r[7]=0; clear group by abort flag 3 Null 0 8 9 0 r[8..9]=NULL; initialize group by comparison registers to NULL - 4 Gosub 18 39 0 0 ; go to clear accumulator subroutine + 4 Gosub 18 37 0 0 ; go to clear accumulator subroutine 5 OpenRead 0 2 0 k(7,B,B,B,B,B,B,B) 0 table=sales, root=2, iDb=0 6 OpenRead 1 5 0 k(3,B,B) 0 index=idx_sales_category_region, root=5, iDb=0 - 7 Rewind 1 25 0 0 Rewind index idx_sales_category_region + 7 Rewind 1 23 0 0 Rewind index idx_sales_category_region 8 DeferredSeek 1 0 0 0 - 9 Column 1 0 15 0 r[15]=idx_sales_category_region.category - 10 Column 1 1 16 0 r[16]=idx_sales_category_region.region - 11 Column 0 4 17 0 r[17]=sales.amount - 12 Compare 8 15 2 k(2, Binary, Binary) 0 r[8..9]==r[15..16] - 13 Jump 14 18 14 0 ; start new group if comparison is not equal - 14 Gosub 5 29 0 0 ; check if ended group had data, and output if so - 15 Move 15 8 2 0 r[8..9]=r[15..16] - 16 IfPos 7 42 0 0 r[7]>0 -> r[7]-=0, goto 42; check abort flag - 17 Gosub 18 39 0 0 ; goto clear accumulator subroutine - 18 AggStep 0 17 12 sum 0 accum=r[12] step(r[17]) - 19 AggStep 0 19 13 count 0 accum=r[13] step(r[19]) - 20 If 6 23 0 0 if r[6] goto 23; don't emit group columns if continuing existing group - 21 Column 1 0 10 0 r[10]=idx_sales_category_region.category - 22 Column 1 1 11 0 r[11]=idx_sales_category_region.region - 23 Integer 1 6 0 0 r[6]=1; indicate data in accumulator - 24 Next 1 8 0 0 - 25 Gosub 5 29 0 0 ; emit row for final group - 26 Goto 0 42 0 0 ; group by finished - 27 Integer 1 7 0 0 r[7]=1 - 28 Return 5 0 0 0 - 29 IfPos 6 31 0 0 r[6]>0 -> r[6]-=0, goto 31; output group by row subroutine start - 30 Return 5 0 0 0 - 31 AggFinal 0 12 0 sum 0 accum=r[12] - 32 AggFinal 0 13 0 count 0 accum=r[13] - 33 Copy 10 1 0 0 r[1]=r[10] - 34 Copy 11 2 0 0 r[2]=r[11] - 35 Copy 12 3 0 0 r[3]=r[12] - 36 Copy 13 4 0 0 r[4]=r[13] - 37 ResultRow 1 4 0 0 output=r[1..4] - 38 Return 5 0 0 0 - 39 Null 0 10 13 0 r[10..13]=NULL; clear accumulator subroutine start - 40 Integer 0 6 0 0 r[6]=0 - 41 Return 18 0 0 0 - 42 Halt 0 0 0 0 - 43 Transaction 0 1 8 0 iDb=0 tx_mode=Read - 44 Integer 1 19 0 0 r[19]=1 - 45 Goto 0 1 0 0 + 9 ColumnRange 1 0 15 2 0 r[15..16]=idx_sales_category_region.category..region + 10 Column 0 4 17 0 r[17]=sales.amount + 11 Compare 8 15 2 k(2, Binary, Binary) 0 r[8..9]==r[15..16] + 12 Jump 13 17 13 0 ; start new group if comparison is not equal + 13 Gosub 5 27 0 0 ; check if ended group had data, and output if so + 14 Move 15 8 2 0 r[8..9]=r[15..16] + 15 IfPos 7 40 0 0 r[7]>0 -> r[7]-=0, goto 40; check abort flag + 16 Gosub 18 37 0 0 ; goto clear accumulator subroutine + 17 AggStep 0 17 12 sum 0 accum=r[12] step(r[17]) + 18 AggStep 0 19 13 count 0 accum=r[13] step(r[19]) + 19 If 6 21 0 0 if r[6] goto 21; don't emit group columns if continuing existing group + 20 ColumnRange 1 0 10 2 0 r[10..11]=idx_sales_category_region.category..region + 21 Integer 1 6 0 0 r[6]=1; indicate data in accumulator + 22 Next 1 8 0 1 + 23 Gosub 5 27 0 0 ; emit row for final group + 24 Goto 0 40 0 0 ; group by finished + 25 Integer 1 7 0 0 r[7]=1 + 26 Return 5 0 0 0 + 27 IfPos 6 29 0 0 r[6]>0 -> r[6]-=0, goto 29; output group by row subroutine start + 28 Return 5 0 0 0 + 29 AggFinal 0 12 0 sum 0 accum=r[12] + 30 AggFinal 0 13 0 count 0 accum=r[13] + 31 Copy 10 1 0 0 r[1]=r[10] + 32 Copy 11 2 0 0 r[2]=r[11] + 33 Copy 12 3 0 0 r[3]=r[12] + 34 Copy 13 4 0 0 r[4]=r[13] + 35 ResultRow 1 4 0 0 output=r[1..4] + 36 Return 5 0 0 0 + 37 Null 0 10 13 0 r[10..13]=NULL; clear accumulator subroutine start + 38 Integer 0 6 0 0 r[6]=0 + 39 Return 18 0 0 0 + 40 Halt 0 0 0 0 + 41 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 42 Integer 1 19 0 0 r[19]=1 + 43 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__multiple-aggregates.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__multiple-aggregates.snap index 7f8f9839..48af8e4e 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__multiple-aggregates.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__multiple-aggregates.snap @@ -28,71 +28,67 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 65 0 0 Start at 65 + 0 Init 0 61 0 0 Start at 61 1 SorterOpen 0 1 0 k(1,-B) 0 cursor=0 2 Null 0 13 17 0 r[13..17]=NULL 3 Integer 0 10 0 0 r[10]=0; clear group by abort flag 4 Null 0 11 0 0 r[11]=NULL; initialize group by comparison registers to NULL - 5 Gosub 24 50 0 0 ; go to clear accumulator subroutine + 5 Gosub 24 49 0 0 ; go to clear accumulator subroutine 6 OpenRead 1 2 0 k(7,B,B,B,B,B,B,B) 0 table=sales, root=2, iDb=0 7 OpenRead 2 4 0 k(2,B) 0 index=idx_sales_category, root=4, iDb=0 - 8 Rewind 2 30 0 0 Rewind index idx_sales_category + 8 Rewind 2 29 0 0 Rewind index idx_sales_category 9 DeferredSeek 2 1 0 0 10 Column 2 0 19 0 r[19]=idx_sales_category.category 11 Column 1 4 20 0 r[20]=sales.amount - 12 Column 1 4 21 0 r[21]=sales.amount - 13 Column 1 5 22 0 r[22]=sales.quantity - 14 Column 1 5 23 0 r[23]=sales.quantity - 15 Compare 11 19 1 k(1, Binary) 0 r[11..11]==r[19..19] - 16 Jump 17 21 17 0 ; start new group if comparison is not equal - 17 Gosub 8 34 0 0 ; check if ended group had data, and output if so - 18 Move 19 11 1 0 r[11..11]=r[19..19] - 19 IfPos 10 53 0 0 r[10]>0 -> r[10]-=0, goto 53; check abort flag - 20 Gosub 24 50 0 0 ; goto clear accumulator subroutine - 21 AggStep 0 25 13 count 0 accum=r[13] step(r[25]) - 22 AggStep 0 20 14 sum 0 accum=r[14] step(r[20]) - 23 AggStep 0 21 15 avg 0 accum=r[15] step(r[21]) - 24 AggStep 0 22 16 sum 0 accum=r[16] step(r[22]) - 25 AggStep 0 23 17 avg 0 accum=r[17] step(r[23]) - 26 If 9 28 0 0 if r[9] goto 28; don't emit group columns if continuing existing group - 27 Column 2 0 12 0 r[12]=idx_sales_category.category - 28 Integer 1 9 0 0 r[9]=1; indicate data in accumulator - 29 Next 2 9 0 0 - 30 Gosub 8 34 0 0 ; emit row for final group - 31 Goto 0 53 0 0 ; group by finished - 32 Integer 1 10 0 0 r[10]=1 - 33 Return 8 0 0 0 - 34 IfPos 9 36 0 0 r[9]>0 -> r[9]-=0, goto 36; output group by row subroutine start - 35 Return 8 0 0 0 - 36 AggFinal 0 13 0 count 0 accum=r[13] - 37 AggFinal 0 14 0 sum 0 accum=r[14] - 38 AggFinal 0 15 0 avg 0 accum=r[15] - 39 AggFinal 0 16 0 sum 0 accum=r[16] - 40 AggFinal 0 17 0 avg 0 accum=r[17] - 41 Copy 14 26 0 0 r[26]=r[14] - 42 Copy 12 27 0 0 r[27]=r[12] - 43 Copy 13 28 0 0 r[28]=r[13] - 44 Copy 15 29 0 0 r[29]=r[15] - 45 Copy 16 30 0 0 r[30]=r[16] - 46 Copy 17 31 0 0 r[31]=r[17] - 47 MakeRecord 26 6 7 0 r[7]=mkrec(r[26..31]) - 48 SorterInsert 0 7 0 0 0 key=r[7] - 49 Return 8 0 0 0 - 50 Null 0 12 17 0 r[12..17]=NULL; clear accumulator subroutine start - 51 Integer 0 9 0 0 r[9]=0 - 52 Return 24 0 0 0 - 53 OpenPseudo 3 7 6 0 6 columns in r[7] - 54 SorterSort 0 64 0 0 - 55 SorterData 0 7 3 0 r[7]=data - 56 Column 3 1 1 0 r[1]=pseudo.column 1 - 57 Column 3 2 2 0 r[2]=pseudo.column 2 - 58 Column 3 0 3 0 r[3]=pseudo.column 0 - 59 Column 3 3 4 0 r[4]=pseudo.column 3 - 60 Column 3 4 5 0 r[5]=pseudo.column 4 - 61 Column 3 5 6 0 r[6]=pseudo.column 5 - 62 ResultRow 1 6 0 0 output=r[1..6] - 63 SorterNext 0 55 0 0 - 64 Halt 0 0 0 0 - 65 Transaction 0 1 8 0 iDb=0 tx_mode=Read - 66 Integer 1 25 0 0 r[25]=1 - 67 Goto 0 1 0 0 + 12 ColumnRange 1 4 21 2 0 r[21..22]=sales.amount..quantity + 13 Column 1 5 23 0 r[23]=sales.quantity + 14 Compare 11 19 1 k(1, Binary) 0 r[11..11]==r[19..19] + 15 Jump 16 20 16 0 ; start new group if comparison is not equal + 16 Gosub 8 33 0 0 ; check if ended group had data, and output if so + 17 Move 19 11 1 0 r[11..11]=r[19..19] + 18 IfPos 10 52 0 0 r[10]>0 -> r[10]-=0, goto 52; check abort flag + 19 Gosub 24 49 0 0 ; goto clear accumulator subroutine + 20 AggStep 0 25 13 count 0 accum=r[13] step(r[25]) + 21 AggStep 0 20 14 sum 0 accum=r[14] step(r[20]) + 22 AggStep 0 21 15 avg 0 accum=r[15] step(r[21]) + 23 AggStep 0 22 16 sum 0 accum=r[16] step(r[22]) + 24 AggStep 0 23 17 avg 0 accum=r[17] step(r[23]) + 25 If 9 27 0 0 if r[9] goto 27; don't emit group columns if continuing existing group + 26 Column 2 0 12 0 r[12]=idx_sales_category.category + 27 Integer 1 9 0 0 r[9]=1; indicate data in accumulator + 28 Next 2 9 0 1 + 29 Gosub 8 33 0 0 ; emit row for final group + 30 Goto 0 52 0 0 ; group by finished + 31 Integer 1 10 0 0 r[10]=1 + 32 Return 8 0 0 0 + 33 IfPos 9 35 0 0 r[9]>0 -> r[9]-=0, goto 35; output group by row subroutine start + 34 Return 8 0 0 0 + 35 AggFinal 0 13 0 count 0 accum=r[13] + 36 AggFinal 0 14 0 sum 0 accum=r[14] + 37 AggFinal 0 15 0 avg 0 accum=r[15] + 38 AggFinal 0 16 0 sum 0 accum=r[16] + 39 AggFinal 0 17 0 avg 0 accum=r[17] + 40 Copy 14 26 0 0 r[26]=r[14] + 41 Copy 12 27 0 0 r[27]=r[12] + 42 Copy 13 28 0 0 r[28]=r[13] + 43 Copy 15 29 0 0 r[29]=r[15] + 44 Copy 16 30 0 0 r[30]=r[16] + 45 Copy 17 31 0 0 r[31]=r[17] + 46 MakeRecord 26 6 7 0 r[7]=mkrec(r[26..31]) + 47 SorterInsert 0 7 0 0 0 key=r[7] + 48 Return 8 0 0 0 + 49 Null 0 12 17 0 r[12..17]=NULL; clear accumulator subroutine start + 50 Integer 0 9 0 0 r[9]=0 + 51 Return 24 0 0 0 + 52 OpenPseudo 3 7 6 0 6 columns in r[7] + 53 SorterSort 0 60 0 0 + 54 SorterData 0 7 3 0 r[7]=data + 55 ColumnRange 3 1 1 2 0 r[1..2]=pseudo.column 1..column 2 + 56 Column 3 0 3 0 r[3]=pseudo.column 0 + 57 ColumnRange 3 3 4 3 0 r[4..6]=pseudo.column 3..column 5 + 58 ResultRow 1 6 0 0 output=r[1..6] + 59 SorterNext 0 54 0 0 + 60 Halt 0 0 0 0 + 61 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 62 Integer 1 25 0 0 r[25]=1 + 63 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__nested-aggregation.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__nested-aggregation.snap index 7ff3d3a2..6afc52cd 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__nested-aggregation.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__nested-aggregation.snap @@ -51,7 +51,7 @@ addr opcode p1 p2 p3 p4 p5 comment 19 If 5 21 0 0 if r[5] goto 21; don't emit group columns if continuing existing group 20 Column 1 0 8 0 r[8]=idx_sales_category.category 21 Integer 1 5 0 0 r[5]=1; indicate data in accumulator - 22 Next 1 9 0 0 + 22 Next 1 9 0 1 23 Gosub 4 27 0 0 ; emit row for final group 24 Goto 0 37 0 0 ; group by finished 25 Integer 1 6 0 0 r[6]=1 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__partial-index-count-star.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__partial-index-count-star.snap index d70ddda3..b69e9af8 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__partial-index-count-star.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__partial-index-count-star.snap @@ -10,22 +10,20 @@ info: database: ':memory:' --- QUERY PLAN -`--SCAN t2 USING INDEX t2_partial_aux_idx +`--SCAN t2 USING COVERING INDEX t2_partial_aux_idx BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 12 0 0 Start at 12 - 1 Null 0 2 0 0 r[2]=NULL - 2 OpenRead 0 2 0 k(4,B,B,B,B) 0 table=t2, root=2, iDb=0 - 3 OpenRead 1 4 0 k(2,B) 0 index=t2_partial_aux_idx, root=4, iDb=0 - 4 Rewind 1 8 0 0 Rewind index t2_partial_aux_idx - 5 DeferredSeek 1 0 0 0 - 6 AggStep 0 3 2 count 0 accum=r[2] step(r[3]) - 7 Next 1 5 0 0 - 8 AggFinal 0 2 0 count 0 accum=r[2] - 9 Copy 2 1 0 0 r[1]=r[2] - 10 ResultRow 1 1 0 0 output=r[1] - 11 Halt 0 0 0 0 - 12 Transaction 0 1 3 0 iDb=0 tx_mode=Read - 13 Integer 1 3 0 0 r[3]=1 - 14 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 10 0 0 Start at 10 + 1 Null 0 2 0 0 r[2]=NULL + 2 OpenRead 0 4 0 k(2,B) 0 index=t2_partial_aux_idx, root=4, iDb=0 + 3 Rewind 0 6 0 0 Rewind index t2_partial_aux_idx + 4 AggStep 0 3 2 count 0 accum=r[2] step(r[3]) + 5 Next 0 4 0 1 + 6 AggFinal 0 2 0 count 0 accum=r[2] + 7 Copy 2 1 0 0 r[1]=r[2] + 8 ResultRow 1 1 0 0 output=r[1] + 9 Halt 0 0 0 0 + 10 Transaction 0 1 3 0 iDb=0 tx_mode=Read + 11 Integer 1 3 0 0 r[3]=1 + 12 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__simple-group-by-single-column.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__simple-group-by-single-column.snap index f87de39c..956ff6c9 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__simple-group-by-single-column.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__simple-group-by-single-column.snap @@ -46,7 +46,7 @@ addr opcode p1 p2 p3 p4 p5 comment 19 If 5 21 0 0 if r[5] goto 21; don't emit group columns if continuing existing group 20 Column 2 0 8 0 r[8]=idx_sales_category.category 21 Integer 1 5 0 0 r[5]=1; indicate data in accumulator - 22 Next 2 9 0 0 + 22 Next 2 9 0 1 23 Gosub 4 27 0 0 ; emit row for final group 24 Goto 0 38 0 0 ; group by finished 25 Integer 1 6 0 0 r[6]=1 diff --git a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__whole-table-aggregates.snap b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__whole-table-aggregates.snap index 4c9bbfae..ee60dcba 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__whole-table-aggregates.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/aggregation/snapshots/aggregation__whole-table-aggregates.snap @@ -38,7 +38,7 @@ addr opcode p1 p2 p3 p4 p5 comment 12 AggStep 0 17 11 max(Binary) 0 accum=r[11] step(r[17]) 13 Column 0 5 18 0 r[18]=sales.quantity 14 AggStep 0 18 12 sum 0 accum=r[12] step(r[18]) - 15 Next 0 4 0 0 + 15 Next 0 4 0 1 16 AggFinal 0 7 0 count 0 accum=r[7] 17 AggFinal 0 8 0 sum 0 accum=r[8] 18 AggFinal 0 9 0 avg 0 accum=r[9] diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__aggregate-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__aggregate-after-analyze.snap index 0a78fcab..b97b3367 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__aggregate-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__aggregate-after-analyze.snap @@ -18,54 +18,53 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 48 0 0 Start at 48 + 0 Init 0 47 0 0 Start at 47 1 Null 0 9 10 0 r[9..10]=NULL 2 SorterOpen 0 2 0 k(1,B) 0 cursor=0 3 Integer 0 6 0 0 r[6]=0; clear group by abort flag 4 Null 0 7 0 0 r[7]=NULL; initialize group by comparison registers to NULL - 5 Gosub 14 44 0 0 ; go to clear accumulator subroutine + 5 Gosub 14 43 0 0 ; go to clear accumulator subroutine 6 OpenRead 2 2 0 k(6,B,B,B,B,B,B) 0 table=products, root=2, iDb=0 - 7 Rewind 2 14 0 0 Rewind table products - 8 Column 2 3 12 0 r[12]=products.category - 9 Column 2 4 13 0 r[13]=products.price - 10 RealAffinity 13 0 0 0 - 11 MakeRecord 12 2 11 0 r[11]=mkrec(r[12..13]) - 12 SorterInsert 0 11 0 0 0 key=r[11] - 13 Next 2 8 0 0 - 14 OpenPseudo 1 11 2 0 2 columns in r[11] - 15 SorterSort 0 31 0 0 - 16 SorterData 0 11 1 0 r[11]=data - 17 Column 1 0 15 0 r[15]=pseudo.column 0 - 18 Compare 7 15 1 k(1, Binary) 0 r[7..7]==r[15..15] - 19 Jump 20 24 20 0 ; start new group if comparison is not equal - 20 Gosub 4 35 0 0 ; check if ended group had data, and output if so - 21 Move 15 7 1 0 r[7..7]=r[15..15] - 22 IfPos 6 47 0 0 r[6]>0 -> r[6]-=0, goto 47; check abort flag - 23 Gosub 14 44 0 0 ; goto clear accumulator subroutine - 24 Column 1 1 16 0 r[16]=pseudo.column 1 - 25 AggStep 0 17 9 count 0 accum=r[9] step(r[17]) - 26 AggStep 0 16 10 avg 0 accum=r[10] step(r[16]) - 27 If 5 29 0 0 if r[5] goto 29; don't emit group columns if continuing existing group - 28 Column 1 0 8 0 r[8]=pseudo.column 0 - 29 Integer 1 5 0 0 r[5]=1; indicate data in accumulator - 30 SorterNext 0 16 0 0 - 31 Gosub 4 35 0 0 ; emit row for final group - 32 Goto 0 47 0 0 ; group by finished - 33 Integer 1 6 0 0 r[6]=1 - 34 Return 4 0 0 0 - 35 IfPos 5 37 0 0 r[5]>0 -> r[5]-=0, goto 37; output group by row subroutine start - 36 Return 4 0 0 0 - 37 AggFinal 0 9 0 count 0 accum=r[9] - 38 AggFinal 0 10 0 avg 0 accum=r[10] - 39 Copy 8 1 0 0 r[1]=r[8] - 40 Copy 9 2 0 0 r[2]=r[9] - 41 Copy 10 3 0 0 r[3]=r[10] - 42 ResultRow 1 3 0 0 output=r[1..3] - 43 Return 4 0 0 0 - 44 Null 0 8 10 0 r[8..10]=NULL; clear accumulator subroutine start - 45 Integer 0 5 0 0 r[5]=0 - 46 Return 14 0 0 0 - 47 Halt 0 0 0 0 - 48 Transaction 0 1 5 0 iDb=0 tx_mode=Read - 49 Integer 1 17 0 0 r[17]=1 - 50 Goto 0 1 0 0 + 7 Rewind 2 13 0 0 Rewind table products + 8 ColumnRange 2 3 12 2 0 r[12..13]=products.category..price + 9 RealAffinity 13 0 0 0 + 10 MakeRecord 12 2 11 0 r[11]=mkrec(r[12..13]) + 11 SorterInsert 0 11 0 0 0 key=r[11] + 12 Next 2 8 0 1 + 13 OpenPseudo 1 11 2 0 2 columns in r[11] + 14 SorterSort 0 30 0 0 + 15 SorterData 0 11 1 0 r[11]=data + 16 Column 1 0 15 0 r[15]=pseudo.column 0 + 17 Compare 7 15 1 k(1, Binary) 0 r[7..7]==r[15..15] + 18 Jump 19 23 19 0 ; start new group if comparison is not equal + 19 Gosub 4 34 0 0 ; check if ended group had data, and output if so + 20 Move 15 7 1 0 r[7..7]=r[15..15] + 21 IfPos 6 46 0 0 r[6]>0 -> r[6]-=0, goto 46; check abort flag + 22 Gosub 14 43 0 0 ; goto clear accumulator subroutine + 23 Column 1 1 16 0 r[16]=pseudo.column 1 + 24 AggStep 0 17 9 count 0 accum=r[9] step(r[17]) + 25 AggStep 0 16 10 avg 0 accum=r[10] step(r[16]) + 26 If 5 28 0 0 if r[5] goto 28; don't emit group columns if continuing existing group + 27 Column 1 0 8 0 r[8]=pseudo.column 0 + 28 Integer 1 5 0 0 r[5]=1; indicate data in accumulator + 29 SorterNext 0 15 0 0 + 30 Gosub 4 34 0 0 ; emit row for final group + 31 Goto 0 46 0 0 ; group by finished + 32 Integer 1 6 0 0 r[6]=1 + 33 Return 4 0 0 0 + 34 IfPos 5 36 0 0 r[5]>0 -> r[5]-=0, goto 36; output group by row subroutine start + 35 Return 4 0 0 0 + 36 AggFinal 0 9 0 count 0 accum=r[9] + 37 AggFinal 0 10 0 avg 0 accum=r[10] + 38 Copy 8 1 0 0 r[1]=r[8] + 39 Copy 9 2 0 0 r[2]=r[9] + 40 Copy 10 3 0 0 r[3]=r[10] + 41 ResultRow 1 3 0 0 output=r[1..3] + 42 Return 4 0 0 0 + 43 Null 0 8 10 0 r[8..10]=NULL; clear accumulator subroutine start + 44 Integer 0 5 0 0 r[5]=0 + 45 Return 14 0 0 0 + 46 Halt 0 0 0 0 + 47 Transaction 0 1 5 0 iDb=0 tx_mode=Read + 48 Integer 1 17 0 0 r[17]=1 + 49 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__join-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__join-after-analyze.snap index 4f019010..c0fa0859 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__join-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__join-after-analyze.snap @@ -17,35 +17,24 @@ info: --- QUERY PLAN |--SEARCH c USING INTEGER PRIMARY KEY (rowid=?) -`--SEARCH p USING INDEX ephemeral_purchases_t2 (customer_id=?) +`--SCAN purchases AS p BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 25 0 0 Start at 25 + 0 Init 0 14 0 0 Start at 14 1 OpenRead 0 2 0 k(2,B,B) 0 table=customers, root=2, iDb=0 2 OpenRead 1 3 0 k(3,B,B,B) 0 table=purchases, root=3, iDb=0 - 3 SeekRowid 0 3 24 0 if (r[3]!=cursor 0 for table customers.rowid) goto 24 - 4 Once 14 0 0 0 goto 14 - 5 OpenAutoindex 2 0 0 0 cursor=2 - 6 Rewind 1 7 0 0 Rewind table purchases - 7 Column 1 1 4 0 r[4]=purchases.customer_id - 8 Column 1 2 5 0 r[5]=purchases.amount - 9 RowId 1 6 0 0 r[6]=purchases.rowid - 10 MakeRecord 4 3 7 0 r[7]=mkrec(r[4..6]); for ephemeral_purchases_t2 - 11 FilterAdd 2 4 1 0 bloom_filter_add(r[4..5]) - 12 IdxInsert 2 7 4 0 key=r[7] - 13 Next 1 7 0 0 - 14 RowId 0 8 0 0 r[8]=customers.rowid - 15 Filter 2 24 8 1 if !bloom_filter(r[8..9]) goto 24 - 16 SeekGE 2 24 8 0 key=[8..8] - 17 IdxGT 2 24 8 0 key=[8..8] - 18 DeferredSeek 2 1 0 0 - 19 Column 0 1 1 0 r[1]=customers.name - 20 Column 2 1 2 0 r[2]=ephemeral(ephemeral_purchases_t2).amount - 21 RealAffinity 2 0 0 0 - 22 ResultRow 1 2 0 0 output=r[1..2] - 23 Next 2 17 0 0 - 24 Halt 0 0 0 0 - 25 Transaction 0 1 4 0 iDb=0 tx_mode=Read - 26 Integer 2 3 0 0 r[3]=2 - 27 Goto 0 1 0 0 + 3 SeekRowid 0 3 13 0 if (r[3]!=cursor 0 for table customers.rowid) goto 13 + 4 Rewind 1 13 0 0 Rewind table purchases + 5 RowId 0 5 0 0 r[5]=customers.rowid + 6 Column 1 1 6 0 r[6]=purchases.customer_id + 7 Ne 5 6 12 Binary 0 if r[5]!=r[6] goto 12 + 8 Column 0 1 1 0 r[1]=customers.name + 9 Column 1 2 2 0 r[2]=purchases.amount + 10 RealAffinity 2 0 0 0 + 11 ResultRow 1 2 0 0 output=r[1..2] + 12 Next 1 5 0 1 + 13 Halt 0 0 0 0 + 14 Transaction 0 1 4 0 iDb=0 tx_mode=Read + 15 Integer 2 3 0 0 r[3]=2 + 16 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__join-aggregate-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__join-aggregate-after-analyze.snap index 1e3c95bf..67cdd777 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__join-aggregate-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__join-aggregate-after-analyze.snap @@ -17,67 +17,66 @@ info: --- QUERY PLAN |--SCAN customers AS c -`--SEARCH p USING INDEX ephemeral_purchases_t2 (customer_id=?) +`--SEARCH p USING COVERING INDEX ephemeral_purchases_t2 (customer_id=?) BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 57 0 0 Start at 57 + 0 Init 0 56 0 0 Start at 56 1 Null 0 10 11 0 r[10..11]=NULL 2 Integer 0 6 0 0 r[6]=0; clear group by abort flag 3 Null 0 7 0 0 r[7]=NULL; initialize group by comparison registers to NULL - 4 Gosub 16 53 0 0 ; go to clear accumulator subroutine + 4 Gosub 16 52 0 0 ; go to clear accumulator subroutine 5 OpenRead 0 2 0 k(2,B,B) 0 table=customers, root=2, iDb=0 6 OpenRead 1 3 0 k(3,B,B,B) 0 table=purchases, root=3, iDb=0 - 7 Rewind 0 40 0 0 Rewind table customers - 8 Once 18 0 0 0 goto 18 + 7 Rewind 0 39 0 0 Rewind table customers + 8 Once 17 0 0 0 goto 17 9 OpenAutoindex 2 0 0 0 cursor=2 10 Rewind 1 11 0 0 Rewind table purchases - 11 Column 1 1 17 0 r[17]=purchases.customer_id - 12 Column 1 2 18 0 r[18]=purchases.amount - 13 RowId 1 19 0 0 r[19]=purchases.rowid - 14 MakeRecord 17 3 20 0 r[20]=mkrec(r[17..19]); for ephemeral_purchases_t2 - 15 FilterAdd 2 17 1 0 bloom_filter_add(r[17..18]) - 16 IdxInsert 2 20 17 0 key=r[20] - 17 Next 1 11 0 0 - 18 RowId 0 21 0 0 r[21]=customers.rowid - 19 Filter 2 39 21 1 if !bloom_filter(r[21..22]) goto 39 - 20 SeekGE 2 39 21 0 key=[21..21] - 21 IdxGT 2 39 21 0 key=[21..21] - 22 DeferredSeek 2 1 0 0 - 23 RowId 0 13 0 0 r[13]=customers.rowid - 24 Column 0 1 14 0 r[14]=customers.name - 25 Column 2 1 15 0 r[15]=ephemeral(ephemeral_purchases_t2).amount - 26 RealAffinity 15 0 0 0 - 27 Compare 7 13 1 k(1, Binary) 0 r[7..7]==r[13..13] - 28 Jump 29 33 29 0 ; start new group if comparison is not equal - 29 Gosub 4 44 0 0 ; check if ended group had data, and output if so - 30 Move 13 7 1 0 r[7..7]=r[13..13] - 31 IfPos 6 56 0 0 r[6]>0 -> r[6]-=0, goto 56; check abort flag - 32 Gosub 16 53 0 0 ; goto clear accumulator subroutine - 33 AggStep 0 22 10 count 0 accum=r[10] step(r[22]) - 34 AggStep 0 15 11 sum 0 accum=r[11] step(r[15]) - 35 If 5 37 0 0 if r[5] goto 37; don't emit group columns if continuing existing group - 36 Column 0 1 8 0 r[8]=customers.name - 37 Integer 1 5 0 0 r[5]=1; indicate data in accumulator - 38 Next 2 21 0 0 - 39 Next 0 8 0 0 - 40 Gosub 4 44 0 0 ; emit row for final group - 41 Goto 0 56 0 0 ; group by finished - 42 Integer 1 6 0 0 r[6]=1 - 43 Return 4 0 0 0 - 44 IfPos 5 46 0 0 r[5]>0 -> r[5]-=0, goto 46; output group by row subroutine start - 45 Return 4 0 0 0 - 46 AggFinal 0 10 0 count 0 accum=r[10] - 47 AggFinal 0 11 0 sum 0 accum=r[11] - 48 Copy 8 1 0 0 r[1]=r[8] - 49 Copy 10 2 0 0 r[2]=r[10] - 50 Copy 11 3 0 0 r[3]=r[11] - 51 ResultRow 1 3 0 0 output=r[1..3] - 52 Return 4 0 0 0 - 53 Null 0 8 11 0 r[8..11]=NULL; clear accumulator subroutine start - 54 Integer 0 5 0 0 r[5]=0 - 55 Return 16 0 0 0 - 56 Halt 0 0 0 0 - 57 Transaction 0 1 4 0 iDb=0 tx_mode=Read - 58 Integer 1 22 0 0 r[22]=1 - 59 Goto 0 1 0 0 + 11 ColumnRange 1 1 17 2 0 r[17..18]=purchases.customer_id..amount + 12 RowId 1 19 0 0 r[19]=purchases.rowid + 13 MakeRecord 17 3 20 0 r[20]=mkrec(r[17..19]); for ephemeral_purchases_t2 + 14 FilterAdd 2 17 1 0 bloom_filter_add(r[17..18]) + 15 IdxInsert 2 20 17 0 key=r[20] + 16 Next 1 11 0 0 + 17 RowId 0 21 0 0 r[21]=customers.rowid + 18 Filter 2 38 21 1 if !bloom_filter(r[21..22]) goto 38 + 19 SeekGE 2 38 21 0 key=[21..21] + 20 IdxGT 2 38 21 0 key=[21..21] + 21 DeferredSeek 2 1 0 0 + 22 RowId 0 13 0 0 r[13]=customers.rowid + 23 Column 0 1 14 0 r[14]=customers.name + 24 Column 2 1 15 0 r[15]=ephemeral(ephemeral_purchases_t2).amount + 25 RealAffinity 15 0 0 0 + 26 Compare 7 13 1 k(1, Binary) 0 r[7..7]==r[13..13] + 27 Jump 28 32 28 0 ; start new group if comparison is not equal + 28 Gosub 4 43 0 0 ; check if ended group had data, and output if so + 29 Move 13 7 1 0 r[7..7]=r[13..13] + 30 IfPos 6 55 0 0 r[6]>0 -> r[6]-=0, goto 55; check abort flag + 31 Gosub 16 52 0 0 ; goto clear accumulator subroutine + 32 AggStep 0 22 10 count 0 accum=r[10] step(r[22]) + 33 AggStep 0 15 11 sum 0 accum=r[11] step(r[15]) + 34 If 5 36 0 0 if r[5] goto 36; don't emit group columns if continuing existing group + 35 Column 0 1 8 0 r[8]=customers.name + 36 Integer 1 5 0 0 r[5]=1; indicate data in accumulator + 37 Next 2 20 0 0 + 38 Next 0 8 0 1 + 39 Gosub 4 43 0 0 ; emit row for final group + 40 Goto 0 55 0 0 ; group by finished + 41 Integer 1 6 0 0 r[6]=1 + 42 Return 4 0 0 0 + 43 IfPos 5 45 0 0 r[5]>0 -> r[5]-=0, goto 45; output group by row subroutine start + 44 Return 4 0 0 0 + 45 AggFinal 0 10 0 count 0 accum=r[10] + 46 AggFinal 0 11 0 sum 0 accum=r[11] + 47 Copy 8 1 0 0 r[1]=r[8] + 48 Copy 10 2 0 0 r[2]=r[10] + 49 Copy 11 3 0 0 r[3]=r[11] + 50 ResultRow 1 3 0 0 output=r[1..3] + 51 Return 4 0 0 0 + 52 Null 0 8 11 0 r[8..11]=NULL; clear accumulator subroutine start + 53 Integer 0 5 0 0 r[5]=0 + 54 Return 16 0 0 0 + 55 Halt 0 0 0 0 + 56 Transaction 0 1 4 0 iDb=0 tx_mode=Read + 57 Integer 1 22 0 0 r[22]=1 + 58 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__max-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__max-after-analyze.snap index 913ddee9..bc9b7655 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__max-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__max-after-analyze.snap @@ -21,7 +21,7 @@ addr opcode p1 p2 p3 p4 p5 comment 4 Column 0 4 3 0 r[3]=products.price 5 RealAffinity 3 0 0 0 6 AggStep 0 3 2 max(Binary) 0 accum=r[2] step(r[3]) - 7 Next 0 4 0 0 + 7 Next 0 4 0 1 8 AggFinal 0 2 0 max 0 accum=r[2] 9 Copy 2 1 0 0 r[1]=r[2] 10 ResultRow 1 1 0 0 output=r[1] diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__min-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__min-after-analyze.snap index 80602322..baf1a486 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__min-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__min-after-analyze.snap @@ -21,7 +21,7 @@ addr opcode p1 p2 p3 p4 p5 comment 4 Column 0 4 3 0 r[3]=products.price 5 RealAffinity 3 0 0 0 6 AggStep 0 3 2 min(Binary) 0 accum=r[2] step(r[3]) - 7 Next 0 4 0 0 + 7 Next 0 4 0 1 8 AggFinal 0 2 0 min 0 accum=r[2] 9 Copy 2 1 0 0 r[1]=r[2] 10 ResultRow 1 1 0 0 output=r[1] diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__order-by-index-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__order-by-index-after-analyze.snap index 6f42d0a4..6bfd3a47 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__order-by-index-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__order-by-index-after-analyze.snap @@ -15,7 +15,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 23 0 0 Start at 23 + 0 Init 0 22 0 0 Start at 22 1 SorterOpen 0 1 0 k(1,B) 0 cursor=0 2 Integer 5 5 0 0 r[5]=5; LIMIT counter 3 IfNot 5 13 0 0 if !r[5] goto 13 @@ -27,16 +27,15 @@ addr opcode p1 p2 p3 p4 p5 comment 9 Column 1 1 8 0 r[8]=products.sku 10 MakeRecord 6 3 4 0 r[4]=mkrec(r[6..8]) 11 SorterInsert 0 4 0 0 0 key=r[4] - 12 Next 1 6 0 0 + 12 Next 1 6 0 1 13 OpenPseudo 2 4 3 0 3 columns in r[4] - 14 SorterSort 0 22 0 0 + 14 SorterSort 0 21 0 0 15 SorterData 0 4 2 0 r[4]=data - 16 Column 2 1 1 0 r[1]=pseudo.column 1 - 17 Column 2 2 2 0 r[2]=pseudo.column 2 - 18 Column 2 0 3 0 r[3]=pseudo.column 0 - 19 ResultRow 1 3 0 0 output=r[1..3] - 20 DecrJumpZero 5 22 0 0 if (--r[5]==0) goto 22 - 21 SorterNext 0 15 0 0 - 22 Halt 0 0 0 0 - 23 Transaction 0 1 5 0 iDb=0 tx_mode=Read - 24 Goto 0 1 0 0 + 16 ColumnRange 2 1 1 2 0 r[1..2]=pseudo.column 1..column 2 + 17 Column 2 0 3 0 r[3]=pseudo.column 0 + 18 ResultRow 1 3 0 0 output=r[1..3] + 19 DecrJumpZero 5 21 0 0 if (--r[5]==0) goto 21 + 20 SorterNext 0 15 0 0 + 21 Halt 0 0 0 0 + 22 Transaction 0 1 5 0 iDb=0 tx_mode=Read + 23 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-after-analyze-category.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-after-analyze-category.snap index a0f2fe83..afebf745 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-after-analyze-category.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-after-analyze-category.snap @@ -24,7 +24,7 @@ addr opcode p1 p2 p3 p4 p5 comment 7 Column 0 4 3 0 r[3]=products.price 8 RealAffinity 3 0 0 0 9 ResultRow 1 3 0 0 output=r[1..3] - 10 Next 0 3 0 0 + 10 Next 0 3 0 1 11 Halt 0 0 0 0 12 Transaction 0 1 5 0 iDb=0 tx_mode=Read 13 String8 0 6 0 Office 0 r[6]='Office' diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-after-analyze-range.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-after-analyze-range.snap index 515e4730..c96621a6 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-after-analyze-range.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-after-analyze-range.snap @@ -28,7 +28,7 @@ addr opcode p1 p2 p3 p4 p5 comment 11 Column 0 4 3 0 r[3]=products.price 12 RealAffinity 3 0 0 0 13 ResultRow 1 3 0 0 output=r[1..3] - 14 Next 0 3 0 0 + 14 Next 0 3 0 1 15 Halt 0 0 0 0 16 Transaction 0 1 5 0 iDb=0 tx_mode=Read 17 Real 0 6 0 20.0 0 r[6]=20 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-composite-full-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-composite-full-after-analyze.snap index 5979fc8a..35a01bac 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-composite-full-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-composite-full-after-analyze.snap @@ -17,20 +17,19 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 15 0 0 Start at 15 + 0 Init 0 14 0 0 Start at 14 1 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=orders, root=2, iDb=0 2 OpenRead 1 3 0 k(3,B,B) 0 index=idx_orders_customer_date, root=3, iDb=0 3 Integer 100 4 0 0 r[4]=100 4 String8 0 5 0 2024-01-15 0 r[5]='2024-01-15' - 5 SeekGE 1 14 4 0 key=[4..5] - 6 IdxGT 1 14 4 0 key=[4..5] + 5 SeekGE 1 13 4 0 key=[4..5] + 6 IdxGT 1 13 4 0 key=[4..5] 7 DeferredSeek 1 0 0 0 8 IdxRowId 1 1 0 0 r[1]=cursor 1 for index idx_orders_customer_date.rowid - 9 Column 0 3 2 0 r[2]=orders.status - 10 Column 0 4 3 0 r[3]=orders.total_amount - 11 RealAffinity 3 0 0 0 - 12 ResultRow 1 3 0 0 output=r[1..3] - 13 Next 1 6 0 0 - 14 Halt 0 0 0 0 - 15 Transaction 0 1 4 0 iDb=0 tx_mode=Read - 16 Goto 0 1 0 0 + 9 ColumnRange 0 3 2 2 0 r[2..3]=orders.status..total_amount + 10 RealAffinity 3 0 0 0 + 11 ResultRow 1 3 0 0 output=r[1..3] + 12 Next 1 6 0 0 + 13 Halt 0 0 0 0 + 14 Transaction 0 1 4 0 iDb=0 tx_mode=Read + 15 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-composite-partial-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-composite-partial-after-analyze.snap index 61be48e7..b72eb9e5 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-composite-partial-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-composite-partial-after-analyze.snap @@ -27,7 +27,7 @@ addr opcode p1 p2 p3 p4 p5 comment 7 Column 0 4 3 0 r[3]=orders.total_amount 8 RealAffinity 3 0 0 0 9 ResultRow 1 3 0 0 output=r[1..3] - 10 Next 0 3 0 0 + 10 Next 0 3 0 1 11 Halt 0 0 0 0 12 Transaction 0 1 4 0 iDb=0 tx_mode=Read 13 Integer 100 6 0 0 r[6]=100 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-for-nulls-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-for-nulls-after-analyze.snap index 3f0398c0..8503a288 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-for-nulls-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-for-nulls-after-analyze.snap @@ -10,20 +10,20 @@ info: database: ':memory:' --- QUERY PLAN -`--SCAN items +`--SEARCH items USING COVERING INDEX idx_items_cat (category=?) BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 11 0 0 Start at 11 - 1 OpenRead 0 2 0 k(3,B,B,B) 0 table=items, root=2, iDb=0 - 2 Rewind 0 10 0 0 Rewind table items - 3 Column 0 1 4 0 r[4]=items.category - 4 NotNull 4 9 0 0 r[4]!=NULL -> goto 9 - 5 RowId 0 1 0 0 r[1]=items.rowid - 6 Column 0 1 2 0 r[2]=items.category - 7 Column 0 2 3 0 r[3]=items.subcategory - 8 ResultRow 1 3 0 0 output=r[1..3] - 9 Next 0 3 0 0 - 10 Halt 0 0 0 0 - 11 Transaction 0 1 3 0 iDb=0 tx_mode=Read - 12 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 11 0 0 Start at 11 + 1 OpenRead 0 3 0 k(3,B,B) 0 index=idx_items_cat, root=3, iDb=0 + 2 Null 0 4 0 0 r[4]=NULL + 3 Affinity 4 1 0 0 r[4..5] = B + 4 SeekGE 0 10 4 0 key=[4..4] + 5 IdxGT 0 10 4 0 key=[4..4] + 6 IdxRowId 0 1 0 0 r[1]=cursor 0 for index idx_items_cat.rowid + 7 ColumnRange 0 0 2 2 0 r[2..3]=idx_items_cat.category..subcategory + 8 ResultRow 1 3 0 0 output=r[1..3] + 9 Next 0 5 0 0 + 10 Halt 0 0 0 0 + 11 Transaction 0 1 3 0 iDb=0 tx_mode=Read + 12 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-identical-composite-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-identical-composite-after-analyze.snap index d75833fd..e5df4ea6 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-identical-composite-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-identical-composite-after-analyze.snap @@ -24,7 +24,7 @@ addr opcode p1 p2 p3 p4 p5 comment 7 RowId 0 1 0 0 r[1]=uniform.rowid 8 Column 0 2 2 0 r[2]=uniform.value 9 ResultRow 1 2 0 0 output=r[1..2] - 10 Next 0 3 0 0 + 10 Next 0 3 0 1 11 Halt 0 0 0 0 12 Transaction 0 1 4 0 iDb=0 tx_mode=Read 13 String8 0 5 0 active 0 r[5]='active' diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-identical-values-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-identical-values-after-analyze.snap index 77256ac3..13266c24 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-identical-values-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-identical-values-after-analyze.snap @@ -13,18 +13,17 @@ QUERY PLAN `--SCAN uniform BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 11 0 0 Start at 11 - 1 OpenRead 0 2 0 k(3,B,B,B) 0 table=uniform, root=2, iDb=0 - 2 Rewind 0 10 0 0 Rewind table uniform - 3 Column 0 1 5 0 r[5]=uniform.status - 4 Ne 5 6 9 Binary 0 if r[5]!=r[6] goto 9 - 5 RowId 0 1 0 0 r[1]=uniform.rowid - 6 Column 0 1 2 0 r[2]=uniform.status - 7 Column 0 2 3 0 r[3]=uniform.value - 8 ResultRow 1 3 0 0 output=r[1..3] - 9 Next 0 3 0 0 - 10 Halt 0 0 0 0 - 11 Transaction 0 1 4 0 iDb=0 tx_mode=Read - 12 String8 0 6 0 active 0 r[6]='active' - 13 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 10 0 0 Start at 10 + 1 OpenRead 0 2 0 k(3,B,B,B) 0 table=uniform, root=2, iDb=0 + 2 Rewind 0 9 0 0 Rewind table uniform + 3 Column 0 1 5 0 r[5]=uniform.status + 4 Ne 5 6 8 Binary 0 if r[5]!=r[6] goto 8 + 5 RowId 0 1 0 0 r[1]=uniform.rowid + 6 ColumnRange 0 1 2 2 0 r[2..3]=uniform.status..value + 7 ResultRow 1 3 0 0 output=r[1..3] + 8 Next 0 3 0 1 + 9 Halt 0 0 0 0 + 10 Transaction 0 1 4 0 iDb=0 tx_mode=Read + 11 String8 0 6 0 active 0 r[6]='active' + 12 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-three-column-first-only-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-three-column-first-only-after-analyze.snap index 50a5a3a9..35bda280 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-three-column-first-only-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-three-column-first-only-after-analyze.snap @@ -16,19 +16,17 @@ QUERY PLAN `--SCAN inventory BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 12 0 0 Start at 12 - 1 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=inventory, root=2, iDb=0 - 2 Rewind 0 11 0 0 Rewind table inventory - 3 Column 0 1 6 0 r[6]=inventory.warehouse_id - 4 Ne 6 7 10 Binary 0 if r[6]!=r[7] goto 10 - 5 RowId 0 1 0 0 r[1]=inventory.rowid - 6 Column 0 2 2 0 r[2]=inventory.product_id - 7 Column 0 3 3 0 r[3]=inventory.bin_location - 8 Column 0 4 4 0 r[4]=inventory.quantity - 9 ResultRow 1 4 0 0 output=r[1..4] - 10 Next 0 3 0 0 - 11 Halt 0 0 0 0 - 12 Transaction 0 1 3 0 iDb=0 tx_mode=Read - 13 Integer 2 7 0 0 r[7]=2 - 14 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 10 0 0 Start at 10 + 1 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=inventory, root=2, iDb=0 + 2 Rewind 0 9 0 0 Rewind table inventory + 3 Column 0 1 6 0 r[6]=inventory.warehouse_id + 4 Ne 6 7 8 Binary 0 if r[6]!=r[7] goto 8 + 5 RowId 0 1 0 0 r[1]=inventory.rowid + 6 ColumnRange 0 2 2 3 0 r[2..4]=inventory.product_id..quantity + 7 ResultRow 1 4 0 0 output=r[1..4] + 8 Next 0 3 0 1 + 9 Halt 0 0 0 0 + 10 Transaction 0 1 3 0 iDb=0 tx_mode=Read + 11 Integer 2 7 0 0 r[7]=2 + 12 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-three-column-partial-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-three-column-partial-after-analyze.snap index c99745bc..054ac9a4 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-three-column-partial-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-three-column-partial-after-analyze.snap @@ -16,21 +16,20 @@ QUERY PLAN `--SCAN inventory BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 13 0 0 Start at 13 - 1 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=inventory, root=2, iDb=0 - 2 Rewind 0 12 0 0 Rewind table inventory - 3 Column 0 1 5 0 r[5]=inventory.warehouse_id - 4 Ne 5 6 11 Binary 0 if r[5]!=r[6] goto 11 - 5 Column 0 2 8 0 r[8]=inventory.product_id - 6 Ne 8 9 11 Binary 0 if r[8]!=r[9] goto 11 - 7 RowId 0 1 0 0 r[1]=inventory.rowid - 8 Column 0 3 2 0 r[2]=inventory.bin_location - 9 Column 0 4 3 0 r[3]=inventory.quantity - 10 ResultRow 1 3 0 0 output=r[1..3] - 11 Next 0 3 0 0 - 12 Halt 0 0 0 0 - 13 Transaction 0 1 3 0 iDb=0 tx_mode=Read - 14 Integer 1 6 0 0 r[6]=1 - 15 Integer 100 9 0 0 r[9]=100 - 16 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 12 0 0 Start at 12 + 1 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=inventory, root=2, iDb=0 + 2 Rewind 0 11 0 0 Rewind table inventory + 3 Column 0 1 5 0 r[5]=inventory.warehouse_id + 4 Ne 5 6 10 Binary 0 if r[5]!=r[6] goto 10 + 5 Column 0 2 8 0 r[8]=inventory.product_id + 6 Ne 8 9 10 Binary 0 if r[8]!=r[9] goto 10 + 7 RowId 0 1 0 0 r[1]=inventory.rowid + 8 ColumnRange 0 3 2 2 0 r[2..3]=inventory.bin_location..quantity + 9 ResultRow 1 3 0 0 output=r[1..3] + 10 Next 0 3 0 1 + 11 Halt 0 0 0 0 + 12 Transaction 0 1 3 0 iDb=0 tx_mode=Read + 13 Integer 1 6 0 0 r[6]=1 + 14 Integer 100 9 0 0 r[9]=100 + 15 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-with-nulls-after-analyze.snap b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-with-nulls-after-analyze.snap index b29a8e35..28022c37 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-with-nulls-after-analyze.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/analyze/snapshots/analyze__query-with-nulls-after-analyze.snap @@ -13,18 +13,17 @@ QUERY PLAN `--SCAN items BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 11 0 0 Start at 11 - 1 OpenRead 0 2 0 k(3,B,B,B) 0 table=items, root=2, iDb=0 - 2 Rewind 0 10 0 0 Rewind table items - 3 Column 0 1 5 0 r[5]=items.category - 4 Ne 5 6 9 Binary 0 if r[5]!=r[6] goto 9 - 5 RowId 0 1 0 0 r[1]=items.rowid - 6 Column 0 1 2 0 r[2]=items.category - 7 Column 0 2 3 0 r[3]=items.subcategory - 8 ResultRow 1 3 0 0 output=r[1..3] - 9 Next 0 3 0 0 - 10 Halt 0 0 0 0 - 11 Transaction 0 1 3 0 iDb=0 tx_mode=Read - 12 String8 0 6 0 cat1 0 r[6]='cat1' - 13 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 10 0 0 Start at 10 + 1 OpenRead 0 2 0 k(3,B,B,B) 0 table=items, root=2, iDb=0 + 2 Rewind 0 9 0 0 Rewind table items + 3 Column 0 1 5 0 r[5]=items.category + 4 Ne 5 6 8 Binary 0 if r[5]!=r[6] goto 8 + 5 RowId 0 1 0 0 r[1]=items.rowid + 6 ColumnRange 0 1 2 2 0 r[2..3]=items.category..subcategory + 7 ResultRow 1 3 0 0 output=r[1..3] + 8 Next 0 3 0 1 + 9 Halt 0 0 0 0 + 10 Transaction 0 1 3 0 iDb=0 tx_mode=Read + 11 String8 0 6 0 cat1 0 r[6]='cat1' + 12 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/expressions/snapshots/expressions__constant-parameter-hoisted-out-of-scan.snap b/conformance/sqlite-sqltests/snapshot_tests/expressions/snapshots/expressions__constant-parameter-hoisted-out-of-scan.snap index 2e0abaa3..d21f7eb6 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/expressions/snapshots/expressions__constant-parameter-hoisted-out-of-scan.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/expressions/snapshots/expressions__constant-parameter-hoisted-out-of-scan.snap @@ -20,7 +20,7 @@ addr opcode p1 p2 p3 p4 p5 comment 3 Ne 4 5 7 0 if r[4]!=r[5] goto 7 4 Rewind 0 7 0 0 Rewind table t 5 AggStep 0 8 2 count 0 accum=r[2] step(r[8]) - 6 Next 0 5 0 0 + 6 Next 0 5 0 1 7 AggFinal 0 2 0 count 0 accum=r[2] 8 Copy 2 1 0 0 r[1]=r[2] 9 ResultRow 1 1 0 0 output=r[1] diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/indexes.sqltest b/conformance/sqlite-sqltests/snapshot_tests/indexes/indexes.sqltest index 6a4a088b..c38b307a 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/indexes.sqltest +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/indexes.sqltest @@ -677,6 +677,14 @@ snapshot-eqp join-with-index { WHERE o.customer_id = 100; } +@setup schema +snapshot join-with-covering-index-bytecode { + SELECT o.id, o.order_date, i.quantity + FROM orders o + JOIN inventory i ON o.id = i.product_id + WHERE o.customer_id = 100; +} + # Subquery with indexed lookup @setup schema snapshot-eqp subquery-with-index { diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-all-columns.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-all-columns.snap index ac525498..9dd060e4 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-all-columns.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-all-columns.snap @@ -13,4 +13,4 @@ info: database: ':memory:' --- QUERY PLAN -`--SEARCH inventory USING INDEX idx_inventory_covering (warehouse_id=? AND product_id=?) +`--SEARCH inventory USING COVERING INDEX idx_inventory_covering (warehouse_id=? AND product_id=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-subset.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-subset.snap index 1c836069..edd540dd 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-subset.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-subset.snap @@ -13,4 +13,4 @@ info: database: ':memory:' --- QUERY PLAN -`--SEARCH inventory USING INDEX idx_inventory_covering (warehouse_id=?) +`--SEARCH inventory USING COVERING INDEX idx_inventory_covering (warehouse_id=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-with-aggregation.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-with-aggregation.snap index bd64db2a..a5d5cc3c 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-with-aggregation.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__covering-index-with-aggregation.snap @@ -14,4 +14,4 @@ info: database: ':memory:' --- QUERY PLAN -`--SEARCH inventory USING INDEX idx_inventory_covering (warehouse_id=?) +`--SEARCH inventory USING COVERING INDEX idx_inventory_covering (warehouse_id=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__in-list-composite-prefix.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__in-list-composite-prefix.snap index 93b06736..e83097d4 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__in-list-composite-prefix.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__in-list-composite-prefix.snap @@ -13,4 +13,4 @@ info: database: ':memory:' --- QUERY PLAN -`--SEARCH orders USING INDEX idx_orders_customer_date (customer_id=?) +`--SEARCH orders USING COVERING INDEX idx_orders_customer_date (customer_id=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-correlated-max-materialized-cte-bytecode.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-correlated-max-materialized-cte-bytecode.snap index 75c87836..a1c5e46f 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-correlated-max-materialized-cte-bytecode.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-correlated-max-materialized-cte-bytecode.snap @@ -28,43 +28,44 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 37 0 0 Start at 37 - 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true - 2 OpenRead 1 5 0 k(2,B) 0 index=idx_products_price, root=5, iDb=0 - 3 Rewind 1 10 0 0 Rewind index idx_products_price - 4 Column 1 0 3 0 r[3]=idx_products_price.price - 5 RealAffinity 3 0 0 0 - 6 MakeRecord 3 1 4 0 r[4]=mkrec(r[3..3]); for - 7 NewRowid 0 5 0 0 r[5]=rowid - 8 Insert 0 4 5 4 intkey=r[5] data=r[4] - 9 Next 1 4 0 0 - 10 Rewind 0 36 0 0 Rewind ephemeral() - 11 Column 0 0 2 0 r[2]=ephemeral().price - 12 Copy 2 8 0 0 r[8]=r[2] - 13 Ge 8 9 35 Binary 0 if r[8]>=r[9] goto 35 - 14 BeginSubrtn 10 0 0 0 r[10]=NULL - 15 Null 0 1 0 0 r[1]=NULL - 16 OpenDup 2 0 0 0 new_cursor=2, original_cursor=0 - 17 Null 0 13 0 0 r[13]=NULL - 18 Integer 1 14 0 0 r[14]=1; LIMIT counter - 19 IfNot 14 29 0 0 if !r[14] goto 29 - 20 Last 2 29 0 0 - 21 Column 2 0 11 0 r[11]=ephemeral().price - 22 Copy 11 16 0 0 r[16]=r[11] - 23 Copy 2 17 0 0 r[17]=r[2] - 24 Ge 16 17 28 Binary 0 if r[16]>=r[17] goto 28 - 25 Copy 11 18 0 0 r[18]=r[11] - 26 AggStep 0 18 13 max(Binary) 0 accum=r[13] step(r[18]) - 27 Goto 0 29 0 0 - 28 Prev 2 21 0 0 - 29 AggFinal 0 13 0 max 0 accum=r[13] - 30 Copy 13 12 0 0 r[12]=r[13] - 31 Copy 12 1 0 0 r[1]=r[12] - 32 Return 10 0 1 0 - 33 Copy 1 6 0 0 r[6]=r[1] - 34 ResultRow 6 1 0 0 output=r[6] - 35 Next 0 11 0 0 - 36 Halt 0 0 0 0 - 37 Transaction 0 1 14 0 iDb=0 tx_mode=Read - 38 Integer 100 9 0 0 r[9]=100 - 39 Goto 0 1 0 0 + 0 Init 0 38 0 0 Start at 38 + 1 Once 11 0 0 0 goto 11 + 2 OpenEphemeral 0 1 0 0 cursor=0 is_table=true + 3 OpenRead 1 5 0 k(2,B) 0 index=idx_products_price, root=5, iDb=0 + 4 Rewind 1 11 0 0 Rewind index idx_products_price + 5 Column 1 0 3 0 r[3]=idx_products_price.price + 6 RealAffinity 3 0 0 0 + 7 MakeRecord 3 1 4 0 r[4]=mkrec(r[3..3]); for + 8 NewRowid 0 5 0 0 r[5]=rowid + 9 Insert 0 4 5 4 intkey=r[5] data=r[4] + 10 Next 1 5 0 1 + 11 Rewind 0 37 0 0 Rewind ephemeral() + 12 Column 0 0 2 0 r[2]=ephemeral().price + 13 Copy 2 8 0 0 r[8]=r[2] + 14 Ge 8 9 36 Binary 0 if r[8]>=r[9] goto 36 + 15 BeginSubrtn 10 0 0 0 r[10]=NULL + 16 Null 0 1 0 0 r[1]=NULL + 17 OpenDup 2 0 0 0 new_cursor=2, original_cursor=0 + 18 Null 0 13 0 0 r[13]=NULL + 19 Integer 1 14 0 0 r[14]=1; LIMIT counter + 20 IfNot 14 30 0 0 if !r[14] goto 30 + 21 Last 2 30 0 0 + 22 Column 2 0 11 0 r[11]=ephemeral().price + 23 Copy 11 16 0 0 r[16]=r[11] + 24 Copy 2 17 0 0 r[17]=r[2] + 25 Ge 16 17 29 Binary 0 if r[16]>=r[17] goto 29 + 26 Copy 11 18 0 0 r[18]=r[11] + 27 AggStep 0 18 13 max(Binary) 0 accum=r[13] step(r[18]) + 28 Goto 0 30 0 0 + 29 Prev 2 22 0 0 + 30 AggFinal 0 13 0 max 0 accum=r[13] + 31 Copy 13 12 0 0 r[12]=r[13] + 32 Copy 12 1 0 0 r[1]=r[12] + 33 Return 10 0 1 0 + 34 Copy 1 6 0 0 r[6]=r[1] + 35 ResultRow 6 1 0 0 output=r[6] + 36 Next 0 12 0 0 + 37 Halt 0 0 0 0 + 38 Transaction 0 1 14 0 iDb=0 tx_mode=Read + 39 Integer 100 9 0 0 r[9]=100 + 40 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-bytecode.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-bytecode.snap index 7b125389..451f0d3b 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-bytecode.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-bytecode.snap @@ -22,7 +22,7 @@ addr opcode p1 p2 p3 p4 p5 comment 5 RealAffinity 3 0 0 0 6 AggStep 0 3 2 max(Binary) 0 accum=r[2] step(r[3]) 7 Goto 0 9 0 0 - 8 Prev 0 4 0 0 + 8 Prev 0 4 0 1 9 AggFinal 0 2 0 max 0 accum=r[2] 10 Copy 2 1 0 0 r[1]=r[2] 11 ResultRow 1 1 0 0 output=r[1] diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-materialized-cte-bytecode.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-materialized-cte-bytecode.snap index bb93e540..cb1ec536 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-materialized-cte-bytecode.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-materialized-cte-bytecode.snap @@ -20,29 +20,30 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 23 0 0 Start at 23 - 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true - 2 OpenRead 1 5 0 k(2,B) 0 index=idx_products_price, root=5, iDb=0 - 3 Rewind 1 10 0 0 Rewind index idx_products_price - 4 Column 1 0 2 0 r[2]=idx_products_price.price - 5 RealAffinity 2 0 0 0 - 6 MakeRecord 2 1 3 0 r[3]=mkrec(r[2..2]); for - 7 NewRowid 0 4 0 0 r[4]=rowid - 8 Insert 0 3 4 4 intkey=r[4] data=r[3] - 9 Next 1 4 0 0 - 10 Null 0 6 0 0 r[6]=NULL - 11 Last 0 19 0 0 - 12 Column 0 0 1 0 r[1]=ephemeral().price - 13 Copy 1 8 0 0 r[8]=r[1] - 14 Ge 8 9 18 Binary 0 if r[8]>=r[9] goto 18 - 15 Copy 1 10 0 0 r[10]=r[1] - 16 AggStep 0 10 6 max(Binary) 0 accum=r[6] step(r[10]) - 17 Goto 0 19 0 0 - 18 Prev 0 12 0 0 - 19 AggFinal 0 6 0 max 0 accum=r[6] - 20 Copy 6 5 0 0 r[5]=r[6] - 21 ResultRow 5 1 0 0 output=r[5] - 22 Halt 0 0 0 0 - 23 Transaction 0 1 14 0 iDb=0 tx_mode=Read - 24 Integer 100 9 0 0 r[9]=100 - 25 Goto 0 1 0 0 + 0 Init 0 24 0 0 Start at 24 + 1 Once 11 0 0 0 goto 11 + 2 OpenEphemeral 0 1 0 0 cursor=0 is_table=true + 3 OpenRead 1 5 0 k(2,B) 0 index=idx_products_price, root=5, iDb=0 + 4 Rewind 1 11 0 0 Rewind index idx_products_price + 5 Column 1 0 2 0 r[2]=idx_products_price.price + 6 RealAffinity 2 0 0 0 + 7 MakeRecord 2 1 3 0 r[3]=mkrec(r[2..2]); for + 8 NewRowid 0 4 0 0 r[4]=rowid + 9 Insert 0 3 4 4 intkey=r[4] data=r[3] + 10 Next 1 5 0 1 + 11 Null 0 6 0 0 r[6]=NULL + 12 Last 0 20 0 0 + 13 Column 0 0 1 0 r[1]=ephemeral().price + 14 Copy 1 8 0 0 r[8]=r[1] + 15 Ge 8 9 19 Binary 0 if r[8]>=r[9] goto 19 + 16 Copy 1 10 0 0 r[10]=r[1] + 17 AggStep 0 10 6 max(Binary) 0 accum=r[6] step(r[10]) + 18 Goto 0 20 0 0 + 19 Prev 0 13 0 0 + 20 AggFinal 0 6 0 max 0 accum=r[6] + 21 Copy 6 5 0 0 r[5]=r[6] + 22 ResultRow 5 1 0 0 output=r[5] + 23 Halt 0 0 0 0 + 24 Transaction 0 1 14 0 iDb=0 tx_mode=Read + 25 Integer 100 9 0 0 r[9]=100 + 26 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-materialized-cte-no-where-bytecode.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-materialized-cte-no-where-bytecode.snap index 4535952c..4b0a6f0d 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-materialized-cte-no-where-bytecode.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-max-materialized-cte-no-where-bytecode.snap @@ -20,26 +20,27 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 21 0 0 Start at 21 - 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true - 2 OpenRead 1 5 0 k(2,B) 0 index=idx_products_price, root=5, iDb=0 - 3 Rewind 1 10 0 0 Rewind index idx_products_price - 4 Column 1 0 2 0 r[2]=idx_products_price.price - 5 RealAffinity 2 0 0 0 - 6 MakeRecord 2 1 3 0 r[3]=mkrec(r[2..2]); for - 7 NewRowid 0 4 0 0 r[4]=rowid - 8 Insert 0 3 4 4 intkey=r[4] data=r[3] - 9 Next 1 4 0 0 - 10 Null 0 6 0 0 r[6]=NULL - 11 Last 0 17 0 0 - 12 Column 0 0 1 0 r[1]=ephemeral().price - 13 Copy 1 7 0 0 r[7]=r[1] - 14 AggStep 0 7 6 max(Binary) 0 accum=r[6] step(r[7]) - 15 Goto 0 17 0 0 - 16 Prev 0 12 0 0 - 17 AggFinal 0 6 0 max 0 accum=r[6] - 18 Copy 6 5 0 0 r[5]=r[6] - 19 ResultRow 5 1 0 0 output=r[5] - 20 Halt 0 0 0 0 - 21 Transaction 0 1 14 0 iDb=0 tx_mode=Read - 22 Goto 0 1 0 0 + 0 Init 0 22 0 0 Start at 22 + 1 Once 11 0 0 0 goto 11 + 2 OpenEphemeral 0 1 0 0 cursor=0 is_table=true + 3 OpenRead 1 5 0 k(2,B) 0 index=idx_products_price, root=5, iDb=0 + 4 Rewind 1 11 0 0 Rewind index idx_products_price + 5 Column 1 0 2 0 r[2]=idx_products_price.price + 6 RealAffinity 2 0 0 0 + 7 MakeRecord 2 1 3 0 r[3]=mkrec(r[2..2]); for + 8 NewRowid 0 4 0 0 r[4]=rowid + 9 Insert 0 3 4 4 intkey=r[4] data=r[3] + 10 Next 1 5 0 1 + 11 Null 0 6 0 0 r[6]=NULL + 12 Last 0 18 0 0 + 13 Column 0 0 1 0 r[1]=ephemeral().price + 14 Copy 1 7 0 0 r[7]=r[1] + 15 AggStep 0 7 6 max(Binary) 0 accum=r[6] step(r[7]) + 16 Goto 0 18 0 0 + 17 Prev 0 13 0 0 + 18 AggFinal 0 6 0 max 0 accum=r[6] + 19 Copy 6 5 0 0 r[5]=r[6] + 20 ResultRow 5 1 0 0 output=r[5] + 21 Halt 0 0 0 0 + 22 Transaction 0 1 14 0 iDb=0 tx_mode=Read + 23 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-min-bytecode.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-min-bytecode.snap index 8c319201..94a31717 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-min-bytecode.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__index-for-min-bytecode.snap @@ -23,7 +23,7 @@ addr opcode p1 p2 p3 p4 p5 comment 6 IsNull 3 9 0 0 if (r[3]==NULL) goto 9 7 AggStep 0 3 2 min(Binary) 0 accum=r[2] step(r[3]) 8 Goto 0 10 0 0 - 9 Next 0 4 0 0 + 9 Next 0 4 0 1 10 AggFinal 0 2 0 min 0 accum=r[2] 11 Copy 2 1 0 0 r[1]=r[2] 12 ResultRow 1 1 0 0 output=r[1] diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-delete-category.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-delete-category.snap index d9095580..17bd9e27 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-delete-category.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-delete-category.snap @@ -10,4 +10,4 @@ info: database: ':memory:' --- QUERY PLAN -`--SEARCH products USING INDEX idx_products_category (category=?) +`--SEARCH products USING COVERING INDEX idx_products_category (category=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-delete-sku.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-delete-sku.snap index 6bf76d2a..3d8cbf81 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-delete-sku.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-delete-sku.snap @@ -10,4 +10,4 @@ info: database: ':memory:' --- QUERY PLAN -`--SEARCH products USING INDEX idx_products_sku (sku=?) +`--SEARCH products USING COVERING INDEX idx_products_sku (sku=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-join-both.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-join-both.snap index f89738e8..44712ca2 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-join-both.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-join-both.snap @@ -18,4 +18,4 @@ info: --- QUERY PLAN |--SEARCH p USING INDEX idx_products_sku (sku=?) -`--SEARCH o USING INDEX idx_orders_customer_date (customer_id=?) +`--SEARCH o USING COVERING INDEX idx_orders_customer_date (customer_id=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-join-one-side.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-join-one-side.snap index 9a4da162..eaa25822 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-join-one-side.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-join-one-side.snap @@ -16,5 +16,5 @@ info: database: ':memory:' --- QUERY PLAN -|--SEARCH o USING INDEX idx_orders_customer_date (customer_id=?) +|--SEARCH o USING COVERING INDEX idx_orders_customer_date (customer_id=?) `--SEARCH p USING INTEGER PRIMARY KEY (rowid=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-update-price.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-update-price.snap index 8f93b950..7435759e 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-update-price.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__indexed-by-update-price.snap @@ -10,4 +10,4 @@ info: database: ':memory:' --- QUERY PLAN -`--SEARCH products USING INDEX idx_products_price (price0 -> r[26]-=0, goto 48 + 38 HashNext 1 27 40 payload=r[21]..r[24] 0 + 39 Goto 0 33 0 0 + 40 Next 2 27 0 0 + 41 HashGraceInit 1 0 53 0 hash_table_id=1 + 42 Integer 1 26 0 0 r[26]=1 + 43 HashGraceLoadPart 1 0 52 0 hash_table_id=1 + 44 HashGraceNextProbe 1 20 50 0 hash_table_id=1 keys=r[20]..r[20] probe_rowid_dest=r[25] + 45 SeekRowid 2 25 44 0 if (r[25]!=cursor 2 for table inventory.rowid) goto 44 + 46 HashProbe 1 20 1 r[27]=44 payload=r[21]..r[24] 0 + 47 Goto 0 33 0 0 + 48 HashNext 1 27 44 payload=r[21]..r[24] 0 + 49 Goto 0 33 0 0 + 50 HashGraceAdvPart 1 0 52 0 hash_table_id=1 + 51 Goto 0 43 0 0 + 52 Integer 0 26 0 0 r[26]=0 + 53 HashClose 1 0 0 0 + 54 Halt 0 0 0 0 + 55 Transaction 0 1 14 0 iDb=0 tx_mode=Read + 56 Integer 100 9 0 0 r[9]=100 + 57 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__join-with-index.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__join-with-index.snap index e32da7e9..8415278e 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__join-with-index.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__join-with-index.snap @@ -16,5 +16,6 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN inventory AS i USING COVERING INDEX idx_inventory_covering -`--SEARCH o USING INTEGER PRIMARY KEY (rowid=?) +|--MATERIALIZE hash build input for orders AS o +| `--SEARCH o USING COVERING INDEX idx_orders_customer_date (customer_id=?) +`--HASH JOIN inventory AS i diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__max-with-equality-prefix-bytecode.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__max-with-equality-prefix-bytecode.snap index b1e3c0b1..3f40c8e1 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__max-with-equality-prefix-bytecode.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__max-with-equality-prefix-bytecode.snap @@ -10,7 +10,7 @@ info: database: ':memory:' --- QUERY PLAN -`--SEARCH measurements USING INDEX idx_measurements_category_value (category=?) +`--SEARCH measurements USING COVERING INDEX idx_measurements_category_value (category=?) BYTECODE addr opcode p1 p2 p3 p4 p5 comment diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__or-same-column.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__or-same-column.snap index cb36a12e..1e4af6db 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__or-same-column.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__or-same-column.snap @@ -13,4 +13,4 @@ info: database: ':memory:' --- QUERY PLAN -`--MULTI-INDEX OR events (idx_events_type, idx_events_type) +`--SEARCH events USING INDEX idx_events_type (event_type=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__or-with-and-combinations.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__or-with-and-combinations.snap index 4e854ddf..a27e199d 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__or-with-and-combinations.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__or-with-and-combinations.snap @@ -14,4 +14,4 @@ info: database: ':memory:' --- QUERY PLAN -`--MULTI-INDEX OR events (idx_events_type, idx_events_type) +`--SEARCH events USING INDEX idx_events_type (event_type=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__rowid-range-no-alias.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__rowid-range-no-alias.snap index 28362ff5..73600cfb 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__rowid-range-no-alias.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__rowid-range-no-alias.snap @@ -14,19 +14,18 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 14 0 0 Start at 14 + 0 Init 0 13 0 0 Start at 13 1 OpenRead 0 2 0 k(3,B,B) 0 table=items, root=2, iDb=0 2 Integer 10 4 0 0 r[4]=10 - 3 SeekGT 0 13 4 0 key=[4..4] + 3 SeekGT 0 12 4 0 key=[4..4] 4 Integer 20 4 0 0 r[4]=20 5 RowId 0 5 0 0 r[5]=items.rowid - 6 Ge 5 4 13 0 if r[5]>=r[4] goto 13 + 6 Ge 5 4 12 0 if r[5]>=r[4] goto 12 7 RowId 0 1 0 0 r[1]=items.rowid - 8 Column 0 0 2 0 r[2]=items.name - 9 Column 0 1 3 0 r[3]=items.value - 10 RealAffinity 3 0 0 0 - 11 ResultRow 1 3 0 0 output=r[1..3] - 12 Next 0 5 0 0 - 13 Halt 0 0 0 0 - 14 Transaction 0 1 1 0 iDb=0 tx_mode=Read - 15 Goto 0 1 0 0 + 8 ColumnRange 0 0 2 2 0 r[2..3]=items.name..value + 9 RealAffinity 3 0 0 0 + 10 ResultRow 1 3 0 0 output=r[1..3] + 11 Next 0 5 0 0 + 12 Halt 0 0 0 0 + 13 Transaction 0 1 1 0 iDb=0 tx_mode=Read + 14 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__subquery-with-index.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__subquery-with-index.snap index 9343e245..1f4bccc2 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__subquery-with-index.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__subquery-with-index.snap @@ -15,5 +15,5 @@ info: --- QUERY PLAN |--LIST SUBQUERY 1 -| `--SEARCH inventory USING INDEX idx_inventory_covering (warehouse_id=?) +| `--SEARCH inventory USING COVERING INDEX idx_inventory_covering (warehouse_id=?) `--SEARCH products USING INTEGER PRIMARY KEY (rowid=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__unordered-correlated-max-materialized-cte-bytecode.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__unordered-correlated-max-materialized-cte-bytecode.snap index 1c01548c..1d44b532 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__unordered-correlated-max-materialized-cte-bytecode.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__unordered-correlated-max-materialized-cte-bytecode.snap @@ -28,55 +28,56 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 49 0 0 Start at 49 - 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true - 2 OpenRead 1 2 0 k(2,B) 0 table=products, root=2, iDb=0 - 3 Rewind 1 10 0 0 Rewind table products - 4 Column 1 0 3 0 r[3]=products.price - 5 RealAffinity 3 0 0 0 - 6 MakeRecord 3 1 4 0 r[4]=mkrec(r[3..3]); for - 7 NewRowid 0 5 0 0 r[5]=rowid - 8 Insert 0 4 5 4 intkey=r[5] data=r[4] - 9 Next 1 4 0 0 - 10 Rewind 0 48 0 0 Rewind ephemeral() - 11 Column 0 0 2 0 r[2]=ephemeral().price - 12 Copy 2 8 0 0 r[8]=r[2] - 13 Ge 8 9 47 Binary 0 if r[8]>=r[9] goto 47 - 14 BeginSubrtn 10 0 0 0 r[10]=NULL - 15 Null 0 1 0 0 r[1]=NULL - 16 OpenDup 2 0 0 0 new_cursor=2, original_cursor=0 - 17 Null 0 13 0 0 r[13]=NULL - 18 Integer 1 14 0 0 r[14]=1; LIMIT counter - 19 IfNot 14 41 0 0 if !r[14] goto 41 - 20 Once 28 0 0 0 goto 28 - 21 OpenAutoindex 3 0 0 0 cursor=3 - 22 Rewind 2 23 0 0 Rewind ephemeral() - 23 Column 2 0 15 0 r[15]=ephemeral().price - 24 RowId 2 16 0 0 r[16]=ephemeral().rowid - 25 MakeRecord 15 2 17 0 r[17]=mkrec(r[15..16]); for ephemeral_subquery_t7 - 26 IdxInsert 3 17 15 0 key=r[17] - 27 Next 2 23 0 0 - 28 Copy 2 18 0 0 r[18]=r[2] - 29 IsNull 18 41 0 0 if (r[18]==NULL) goto 41 - 30 Affinity 18 1 0 0 r[18..19] = C - 31 SeekLT 3 41 18 0 key=[18..18] - 32 Null 0 18 0 0 r[18]=NULL - 33 IdxLE 3 41 18 0 key=[18..18] - 34 DeferredSeek 3 2 0 0 - 35 Column 2 0 11 0 r[11]=ephemeral().price - 36 Column 3 0 19 0 r[19]=ephemeral(ephemeral_subquery_t7).price - 37 RealAffinity 19 0 0 0 - 38 AggStep 0 19 13 max(Binary) 0 accum=r[13] step(r[19]) - 39 Goto 0 41 0 0 - 40 Prev 3 32 0 0 - 41 AggFinal 0 13 0 max 0 accum=r[13] - 42 Copy 13 12 0 0 r[12]=r[13] - 43 Copy 12 1 0 0 r[1]=r[12] - 44 Return 10 0 1 0 - 45 Copy 1 6 0 0 r[6]=r[1] - 46 ResultRow 6 1 0 0 output=r[6] - 47 Next 0 11 0 0 - 48 Halt 0 0 0 0 - 49 Transaction 0 1 1 0 iDb=0 tx_mode=Read - 50 Integer 100 9 0 0 r[9]=100 - 51 Goto 0 1 0 0 + 0 Init 0 50 0 0 Start at 50 + 1 Once 11 0 0 0 goto 11 + 2 OpenEphemeral 0 1 0 0 cursor=0 is_table=true + 3 OpenRead 1 2 0 k(2,B) 0 table=products, root=2, iDb=0 + 4 Rewind 1 11 0 0 Rewind table products + 5 Column 1 0 3 0 r[3]=products.price + 6 RealAffinity 3 0 0 0 + 7 MakeRecord 3 1 4 0 r[4]=mkrec(r[3..3]); for + 8 NewRowid 0 5 0 0 r[5]=rowid + 9 Insert 0 4 5 4 intkey=r[5] data=r[4] + 10 Next 1 5 0 1 + 11 Rewind 0 49 0 0 Rewind ephemeral() + 12 Column 0 0 2 0 r[2]=ephemeral().price + 13 Copy 2 8 0 0 r[8]=r[2] + 14 Ge 8 9 48 Binary 0 if r[8]>=r[9] goto 48 + 15 BeginSubrtn 10 0 0 0 r[10]=NULL + 16 Null 0 1 0 0 r[1]=NULL + 17 OpenDup 2 0 0 0 new_cursor=2, original_cursor=0 + 18 Null 0 13 0 0 r[13]=NULL + 19 Integer 1 14 0 0 r[14]=1; LIMIT counter + 20 IfNot 14 42 0 0 if !r[14] goto 42 + 21 Once 29 0 0 0 goto 29 + 22 OpenAutoindex 3 0 0 0 cursor=3 + 23 Rewind 2 24 0 0 Rewind ephemeral() + 24 Column 2 0 15 0 r[15]=ephemeral().price + 25 RowId 2 16 0 0 r[16]=ephemeral().rowid + 26 MakeRecord 15 2 17 0 r[17]=mkrec(r[15..16]); for ephemeral_subquery_t7 + 27 IdxInsert 3 17 15 0 key=r[17] + 28 Next 2 24 0 0 + 29 Copy 2 18 0 0 r[18]=r[2] + 30 IsNull 18 42 0 0 if (r[18]==NULL) goto 42 + 31 Affinity 18 1 0 0 r[18..19] = C + 32 SeekLT 3 42 18 0 key=[18..18] + 33 Null 0 18 0 0 r[18]=NULL + 34 IdxLE 3 42 18 0 key=[18..18] + 35 DeferredSeek 3 2 0 0 + 36 Column 2 0 11 0 r[11]=ephemeral().price + 37 Column 3 0 19 0 r[19]=ephemeral(ephemeral_subquery_t7).price + 38 RealAffinity 19 0 0 0 + 39 AggStep 0 19 13 max(Binary) 0 accum=r[13] step(r[19]) + 40 Goto 0 42 0 0 + 41 Prev 3 33 0 0 + 42 AggFinal 0 13 0 max 0 accum=r[13] + 43 Copy 13 12 0 0 r[12]=r[13] + 44 Copy 12 1 0 0 r[1]=r[12] + 45 Return 10 0 1 0 + 46 Copy 1 6 0 0 r[6]=r[1] + 47 ResultRow 6 1 0 0 output=r[6] + 48 Next 0 12 0 0 + 49 Halt 0 0 0 0 + 50 Transaction 0 1 1 0 iDb=0 tx_mode=Read + 51 Integer 100 9 0 0 r[9]=100 + 52 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__unordered-materialized-cte-max-bytecode.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__unordered-materialized-cte-max-bytecode.snap index 030ab734..0c2b8dfb 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__unordered-materialized-cte-max-bytecode.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__unordered-materialized-cte-max-bytecode.snap @@ -20,25 +20,26 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 20 0 0 Start at 20 - 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true - 2 OpenRead 1 2 0 k(2,B) 0 table=products, root=2, iDb=0 - 3 Rewind 1 10 0 0 Rewind table products - 4 Column 1 0 2 0 r[2]=products.price - 5 RealAffinity 2 0 0 0 - 6 MakeRecord 2 1 3 0 r[3]=mkrec(r[2..2]); for - 7 NewRowid 0 4 0 0 r[4]=rowid - 8 Insert 0 3 4 4 intkey=r[4] data=r[3] - 9 Next 1 4 0 0 - 10 Null 0 6 0 0 r[6]=NULL - 11 Rewind 0 16 0 0 Rewind ephemeral() - 12 Column 0 0 1 0 r[1]=ephemeral().price - 13 Copy 1 7 0 0 r[7]=r[1] - 14 AggStep 0 7 6 max(Binary) 0 accum=r[6] step(r[7]) - 15 Next 0 12 0 0 - 16 AggFinal 0 6 0 max 0 accum=r[6] - 17 Copy 6 5 0 0 r[5]=r[6] - 18 ResultRow 5 1 0 0 output=r[5] - 19 Halt 0 0 0 0 - 20 Transaction 0 1 1 0 iDb=0 tx_mode=Read - 21 Goto 0 1 0 0 + 0 Init 0 21 0 0 Start at 21 + 1 Once 11 0 0 0 goto 11 + 2 OpenEphemeral 0 1 0 0 cursor=0 is_table=true + 3 OpenRead 1 2 0 k(2,B) 0 table=products, root=2, iDb=0 + 4 Rewind 1 11 0 0 Rewind table products + 5 Column 1 0 2 0 r[2]=products.price + 6 RealAffinity 2 0 0 0 + 7 MakeRecord 2 1 3 0 r[3]=mkrec(r[2..2]); for + 8 NewRowid 0 4 0 0 r[4]=rowid + 9 Insert 0 3 4 4 intkey=r[4] data=r[3] + 10 Next 1 5 0 1 + 11 Null 0 6 0 0 r[6]=NULL + 12 Rewind 0 17 0 0 Rewind ephemeral() + 13 Column 0 0 1 0 r[1]=ephemeral().price + 14 Copy 1 7 0 0 r[7]=r[1] + 15 AggStep 0 7 6 max(Binary) 0 accum=r[6] step(r[7]) + 16 Next 0 13 0 0 + 17 AggFinal 0 6 0 max 0 accum=r[6] + 18 Copy 6 5 0 0 r[5]=r[6] + 19 ResultRow 5 1 0 0 output=r[5] + 20 Halt 0 0 0 0 + 21 Transaction 0 1 1 0 iDb=0 tx_mode=Read + 22 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__use-partial-idx-predicate-for-covering-index-scan.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__use-partial-idx-predicate-for-covering-index-scan.snap new file mode 100644 index 00000000..220f2d2d --- /dev/null +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__use-partial-idx-predicate-for-covering-index-scan.snap @@ -0,0 +1,13 @@ +--- +source: indexes.sqltest +expression: select a from t where b < 5; +info: + statement_type: SELECT + tables: + - t + setup_blocks: + - partial-idx-predicate + database: ':memory:' +--- +QUERY PLAN +`--SCAN t USING COVERING INDEX idx diff --git a/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__use-partial-idx-predicate-where-predicate-is-disjunction-term-from-index-where-clause-for-covering-index-scan.snap b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__use-partial-idx-predicate-where-predicate-is-disjunction-term-from-index-where-clause-for-covering-index-scan.snap new file mode 100644 index 00000000..fed02f2e --- /dev/null +++ b/conformance/sqlite-sqltests/snapshot_tests/indexes/snapshots/indexes__use-partial-idx-predicate-where-predicate-is-disjunction-term-from-index-where-clause-for-covering-index-scan.snap @@ -0,0 +1,13 @@ +--- +source: indexes.sqltest +expression: select a from t where b < 5; +info: + statement_type: SELECT + tables: + - t + setup_blocks: + - partial-idx-predicate-disjunction + database: ':memory:' +--- +QUERY PLAN +`--SCAN t diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__bigass-three-table-join-rowid-seek.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__bigass-three-table-join-rowid-seek.snap index 36435cc5..d7b1b472 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__bigass-three-table-join-rowid-seek.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__bigass-three-table-join-rowid-seek.snap @@ -23,8 +23,8 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN customer_support_tickets AS t -|--SEARCH o USING INTEGER PRIMARY KEY (rowid=?) +|--HASH JOIN customer_support_tickets AS t |--SEARCH oi USING INDEX idx_order_items_order_id (order_id=?) +|--SCAN orders AS o |--USE SORTER FOR GROUP BY `--USE HASH TABLE FOR count(DISTINCT) diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-explicit.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-explicit.snap index 9d408fc2..1f9635b5 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-explicit.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-explicit.snap @@ -32,8 +32,8 @@ addr opcode p1 p2 p3 p4 p5 comment 6 Column 1 1 2 0 r[2]=products.name 7 Column 1 3 3 0 r[3]=products.price 8 ResultRow 1 3 0 0 output=r[1..3] - 9 Next 1 5 0 0 - 10 Next 0 4 0 0 + 9 Next 1 5 0 1 + 10 Next 0 4 0 1 11 Halt 0 0 0 0 12 Transaction 0 1 10 0 iDb=0 tx_mode=Read 13 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-limited.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-limited.snap index bb40697c..7042b9ed 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-limited.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-limited.snap @@ -34,8 +34,8 @@ addr opcode p1 p2 p3 p4 p5 comment 8 Column 1 1 2 0 r[2]=products.name 9 ResultRow 1 2 0 0 output=r[1..2] 10 DecrJumpZero 3 13 0 0 if (--r[3]==0) goto 13 - 11 Next 1 7 0 0 - 12 Next 0 6 0 0 + 11 Next 1 7 0 1 + 12 Next 0 6 0 1 13 Halt 0 0 0 0 14 Transaction 0 1 10 0 iDb=0 tx_mode=Read 15 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-with-filter.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-with-filter.snap index 9bda6807..0dfa9676 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-with-filter.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__cross-join-with-filter.snap @@ -22,39 +22,41 @@ info: --- QUERY PLAN |--SCAN customers AS c -`--SEARCH p USING INDEX ephemeral_products_t2 (category=?) +`--SEARCH p USING COVERING INDEX ephemeral_products_t2 (category=?) BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 29 0 0 Start at 29 + 0 Init 0 30 0 0 Start at 30 1 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=customers, root=2, iDb=0 2 OpenRead 1 3 0 k(5,B,B,B,B,B) 0 table=products, root=3, iDb=0 - 3 Rewind 0 28 0 0 Rewind table customers + 3 Rewind 0 29 0 0 Rewind table customers 4 Column 0 3 5 0 r[5]=customers.city - 5 Ne 5 6 27 Binary 0 if r[5]!=r[6] goto 27 - 6 Once 17 0 0 0 goto 17 + 5 Ne 5 6 28 Binary 0 if r[5]!=r[6] goto 28 + 6 Once 19 0 0 0 goto 19 7 OpenAutoindex 2 0 0 0 cursor=2 8 Rewind 1 9 0 0 Rewind table products - 9 Column 1 2 7 0 r[7]=products.category - 10 Column 1 1 8 0 r[8]=products.name - 11 Column 1 3 9 0 r[9]=products.price - 12 RowId 1 10 0 0 r[10]=products.rowid - 13 MakeRecord 7 4 11 0 r[11]=mkrec(r[7..10]); for ephemeral_products_t2 - 14 FilterAdd 2 7 1 0 bloom_filter_add(r[7..8]) - 15 IdxInsert 2 11 7 0 key=r[11] - 16 Next 1 9 0 0 - 17 String8 0 12 0 Books 0 r[12]='Books' - 18 Filter 2 27 12 1 if !bloom_filter(r[12..13]) goto 27 - 19 SeekGE 2 27 12 0 key=[12..12] - 20 IdxGT 2 27 12 0 key=[12..12] - 21 DeferredSeek 2 1 0 0 - 22 Column 0 1 1 0 r[1]=customers.name - 23 Column 2 1 2 0 r[2]=ephemeral(ephemeral_products_t2).name - 24 Column 2 2 3 0 r[3]=ephemeral(ephemeral_products_t2).price - 25 ResultRow 1 3 0 0 output=r[1..3] - 26 Next 2 20 0 0 - 27 Next 0 4 0 0 - 28 Halt 0 0 0 0 - 29 Transaction 0 1 10 0 iDb=0 tx_mode=Read - 30 String8 0 6 0 Boston 0 r[6]='Boston' - 31 Goto 0 1 0 0 + 9 Column 1 2 8 0 r[8]=products.category + 10 Ne 8 9 18 Binary 0 if r[8]!=r[9] goto 18 + 11 Column 1 2 10 0 r[10]=products.category + 12 Column 1 1 11 0 r[11]=products.name + 13 Column 1 3 12 0 r[12]=products.price + 14 RowId 1 13 0 0 r[13]=products.rowid + 15 MakeRecord 10 4 14 0 r[14]=mkrec(r[10..13]); for ephemeral_products_t2 + 16 FilterAdd 2 10 1 0 bloom_filter_add(r[10..11]) + 17 IdxInsert 2 14 10 0 key=r[14] + 18 Next 1 9 0 0 + 19 String8 0 15 0 Books 0 r[15]='Books' + 20 Filter 2 28 15 1 if !bloom_filter(r[15..16]) goto 28 + 21 SeekGE 2 28 15 0 key=[15..15] + 22 IdxGT 2 28 15 0 key=[15..15] + 23 DeferredSeek 2 1 0 0 + 24 Column 0 1 1 0 r[1]=customers.name + 25 ColumnRange 2 1 2 2 0 r[2..3]=ephemeral(ephemeral_products_t2).name..price + 26 ResultRow 1 3 0 0 output=r[1..3] + 27 Next 2 22 0 0 + 28 Next 0 4 0 1 + 29 Halt 0 0 0 0 + 30 Transaction 0 1 10 0 iDb=0 tx_mode=Read + 31 String8 0 6 0 Boston 0 r[6]='Boston' + 32 String8 0 9 0 Books 0 r[9]='Books' + 33 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-inner-basic.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-inner-basic.snap index 28ba253a..ca2fcea0 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-inner-basic.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-inner-basic.snap @@ -20,7 +20,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 42 0 0 Start at 42 + 0 Init 0 41 0 0 Start at 41 1 OpenRead 0 2 0 k(3,B,B,B) 0 table=h_orders, root=2, iDb=0 2 OpenRead 1 3 0 k(4,B,B,B,B) 0 table=h_items, root=3, iDb=0 3 Once 13 0 0 0 goto 13 @@ -33,34 +33,33 @@ addr opcode p1 p2 p3 p4 10 HashBuild 2 5 1 r=[1] budget=67108864 payload=r[6]..r[7] 0 11 Next 2 6 0 0 12 HashBuildFinalize 1 0 0 0 - 13 Rewind 1 28 0 0 Rewind table h_items + 13 Rewind 1 27 0 0 Rewind table h_items 14 Column 1 1 8 0 r[8]=h_items.order_name 15 Affinity 8 1 0 0 r[8..9] = A 16 RowId 1 11 0 0 r[11]=h_items.rowid 17 Integer 0 12 0 0 r[12]=0 - 18 HashProbe 1 8 1 r[13]=27 payload=r[9]..r[10] 0 + 18 HashProbe 1 8 1 r[13]=26 payload=r[9]..r[10] 0 19 Copy 9 1 0 0 r[1]=r[9] 20 Copy 10 2 0 0 r[2]=r[10] - 21 Column 1 2 3 0 r[3]=h_items.product - 22 Column 1 3 4 0 r[4]=h_items.quantity - 23 ResultRow 1 4 0 0 output=r[1..4] - 24 IfPos 12 35 0 0 r[12]>0 -> r[12]-=0, goto 35 - 25 HashNext 1 13 27 payload=r[9]..r[10] 0 - 26 Goto 0 19 0 0 - 27 Next 1 14 0 0 - 28 HashGraceInit 1 0 40 0 hash_table_id=1 - 29 Integer 1 12 0 0 r[12]=1 - 30 HashGraceLoadPart 1 0 39 0 hash_table_id=1 - 31 HashGraceNextProbe 1 8 37 0 hash_table_id=1 keys=r[8]..r[8] probe_rowid_dest=r[11] - 32 SeekRowid 1 11 31 0 if (r[11]!=cursor 1 for table h_items.rowid) goto 31 - 33 HashProbe 1 8 1 r[13]=31 payload=r[9]..r[10] 0 - 34 Goto 0 19 0 0 - 35 HashNext 1 13 31 payload=r[9]..r[10] 0 - 36 Goto 0 19 0 0 - 37 HashGraceAdvPart 1 0 39 0 hash_table_id=1 - 38 Goto 0 30 0 0 - 39 Integer 0 12 0 0 r[12]=0 - 40 HashClose 1 0 0 0 - 41 Halt 0 0 0 0 - 42 Transaction 0 1 3 0 iDb=0 tx_mode=Read - 43 Goto 0 1 0 0 + 21 ColumnRange 1 2 3 2 0 r[3..4]=h_items.product..quantity + 22 ResultRow 1 4 0 0 output=r[1..4] + 23 IfPos 12 34 0 0 r[12]>0 -> r[12]-=0, goto 34 + 24 HashNext 1 13 26 payload=r[9]..r[10] 0 + 25 Goto 0 19 0 0 + 26 Next 1 14 0 0 + 27 HashGraceInit 1 0 39 0 hash_table_id=1 + 28 Integer 1 12 0 0 r[12]=1 + 29 HashGraceLoadPart 1 0 38 0 hash_table_id=1 + 30 HashGraceNextProbe 1 8 36 0 hash_table_id=1 keys=r[8]..r[8] probe_rowid_dest=r[11] + 31 SeekRowid 1 11 30 0 if (r[11]!=cursor 1 for table h_items.rowid) goto 30 + 32 HashProbe 1 8 1 r[13]=30 payload=r[9]..r[10] 0 + 33 Goto 0 19 0 0 + 34 HashNext 1 13 30 payload=r[9]..r[10] 0 + 35 Goto 0 19 0 0 + 36 HashGraceAdvPart 1 0 38 0 hash_table_id=1 + 37 Goto 0 29 0 0 + 38 Integer 0 12 0 0 r[12]=0 + 39 HashClose 1 0 0 0 + 40 Halt 0 0 0 0 + 41 Transaction 0 1 3 0 iDb=0 tx_mode=Read + 42 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-three-table.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-three-table.snap index b850a6d1..609b6b6c 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-three-table.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-three-table.snap @@ -17,109 +17,68 @@ info: database: ':memory:' --- QUERY PLAN -|--MATERIALIZE hash build input for h_orders AS h -| |--HASH JOIN h_orders AS h -| `--SCAN h_items AS i -|--HASH JOIN h_returns AS r -|--SCAN h_items AS i -`--HASH JOIN h_orders AS h +|--HASH JOIN h_items AS i +|--SEARCH r USING COVERING INDEX ephemeral_h_returns_t3 (order_name=?) +`--SCAN h_orders AS h BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 95 0 0 Start at 95 - 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true - 2 OpenRead 1 2 0 k(3,B,B,B) 0 table=h_orders, root=2, iDb=0 - 3 OpenRead 2 3 0 k(4,B,B,B,B) 0 table=h_items, root=3, iDb=0 - 4 Once 14 0 0 0 goto 14 - 5 OpenRead 3 3 0 k(4,B,B,B,B) 0 table=h_items, root=3, iDb=0 - 6 Rewind 3 13 0 0 Rewind table h_items - 7 Column 3 1 8 0 r[8]=h_items.order_name - 8 Affinity 8 1 0 0 r[8..9] = A - 9 Column 3 1 9 0 r[9]=h_items.order_name - 10 Column 3 2 10 0 r[10]=h_items.product - 11 HashBuild 3 8 1 r=[2] budget=67108864 payload=r[9]..r[10] 0 - 12 Next 3 7 0 0 - 13 HashBuildFinalize 2 0 0 0 - 14 Rewind 1 37 0 0 Rewind table h_orders - 15 Column 1 1 11 0 r[11]=h_orders.customer_name - 16 Affinity 11 1 0 0 r[11..12] = A - 17 RowId 1 14 0 0 r[14]=h_orders.rowid - 18 Integer 0 15 0 0 r[15]=0 - 19 HashProbe 2 11 1 r[16]=36 payload=r[12]..r[13] 0 - 20 Column 1 1 18 0 r[18]=h_orders.customer_name - 21 Copy 12 19 0 0 r[19]=r[12] - 22 Ne 18 19 33 Binary 0 if r[18]!=r[19] goto 33 - 23 Column 1 1 1 0 r[1]=h_orders.customer_name - 24 RowId 1 2 0 0 r[2]=h_orders.rowid - 25 Column 1 1 3 0 r[3]=h_orders.customer_name - 26 RowId 1 4 0 0 r[4]=h_orders.rowid - 27 Copy 12 5 0 0 r[5]=r[12] - 28 Copy 13 6 0 0 r[6]=r[13] - 29 Copy 16 7 0 0 r[7]=r[16] - 30 MakeRecord 1 7 20 0 r[20]=mkrec(r[1..7]); for hash_build_input_t4 - 31 NewRowid 0 21 0 0 r[21]=rowid - 32 Insert 0 20 21 hash_build_input_t4 4 intkey=r[21] data=r[20] - 33 IfPos 15 44 0 0 r[15]>0 -> r[15]-=0, goto 44 - 34 HashNext 2 16 36 payload=r[12]..r[13] 0 - 35 Goto 0 20 0 0 - 36 Next 1 15 0 0 - 37 HashGraceInit 2 0 49 0 hash_table_id=2 - 38 Integer 1 15 0 0 r[15]=1 - 39 HashGraceLoadPart 2 0 48 0 hash_table_id=2 - 40 HashGraceNextProbe 2 11 46 0 hash_table_id=2 keys=r[11]..r[11] probe_rowid_dest=r[14] - 41 SeekRowid 1 14 40 0 if (r[14]!=cursor 1 for table h_orders.rowid) goto 40 - 42 HashProbe 2 11 1 r[16]=40 payload=r[12]..r[13] 0 - 43 Goto 0 20 0 0 - 44 HashNext 2 16 40 payload=r[12]..r[13] 0 - 45 Goto 0 20 0 0 - 46 HashGraceAdvPart 2 0 48 0 hash_table_id=2 - 47 Goto 0 39 0 0 - 48 Integer 0 15 0 0 r[15]=0 - 49 OpenRead 1 2 0 k(3,B,B,B) 0 table=h_orders, root=2, iDb=0 - 50 OpenRead 2 3 0 k(4,B,B,B,B) 0 table=h_items, root=3, iDb=0 - 51 OpenRead 4 4 0 k(3,B,B,B) 0 table=h_returns, root=4, iDb=0 - 52 Once 66 0 0 0 goto 66 - 53 Rewind 0 65 0 0 Rewind ephemeral(hash_build_input_t4) - 54 Column 0 0 25 0 r[25]=ephemeral(hash_build_input_t4).key_0 - 55 Affinity 25 1 0 0 r[25..26] = A - 56 Column 0 1 26 0 r[26]=ephemeral(hash_build_input_t4).payload_0 - 57 Column 0 2 27 0 r[27]=ephemeral(hash_build_input_t4).payload_1 - 58 Column 0 3 28 0 r[28]=ephemeral(hash_build_input_t4).payload_2 - 59 Column 0 4 29 0 r[29]=ephemeral(hash_build_input_t4).payload_3 - 60 Column 0 5 30 0 r[30]=ephemeral(hash_build_input_t4).payload_4 - 61 Column 0 6 31 0 r[31]=ephemeral(hash_build_input_t4).payload_5 - 62 HashBuild 0 25 1 r=[1] budget=67108864 payload=r[26]..r[31] 0 - 63 FilterAdd 0 25 1 0 bloom_filter_add(r[25..26]) - 64 Next 0 54 0 0 - 65 HashBuildFinalize 1 0 0 0 - 66 Rewind 4 81 0 0 Rewind table h_returns - 67 Column 4 1 32 0 r[32]=h_returns.order_name - 68 Affinity 32 1 0 0 r[32..33] = A - 69 Filter 0 80 32 1 if !bloom_filter(r[32..33]) goto 80 - 70 RowId 4 39 0 0 r[39]=h_returns.rowid - 71 Integer 0 40 0 0 r[40]=0 - 72 HashProbe 1 32 1 r[41]=80 payload=r[33]..r[38] 0 - 73 Copy 33 22 0 0 r[22]=r[33] - 74 Copy 37 23 0 0 r[23]=r[37] - 75 Column 4 2 24 0 r[24]=h_returns.reason - 76 ResultRow 22 3 0 0 output=r[22..24] - 77 IfPos 40 88 0 0 r[40]>0 -> r[40]-=0, goto 88 - 78 HashNext 1 41 80 payload=r[33]..r[38] 0 - 79 Goto 0 73 0 0 - 80 Next 4 67 0 0 - 81 HashGraceInit 1 0 93 0 hash_table_id=1 - 82 Integer 1 40 0 0 r[40]=1 - 83 HashGraceLoadPart 1 0 92 0 hash_table_id=1 - 84 HashGraceNextProbe 1 32 90 0 hash_table_id=1 keys=r[32]..r[32] probe_rowid_dest=r[39] - 85 SeekRowid 4 39 84 0 if (r[39]!=cursor 4 for table h_returns.rowid) goto 84 - 86 HashProbe 1 32 1 r[41]=84 payload=r[33]..r[38] 0 - 87 Goto 0 73 0 0 - 88 HashNext 1 41 84 payload=r[33]..r[38] 0 - 89 Goto 0 73 0 0 - 90 HashGraceAdvPart 1 0 92 0 hash_table_id=1 - 91 Goto 0 83 0 0 - 92 Integer 0 40 0 0 r[40]=0 - 93 HashClose 1 0 0 0 - 94 Halt 0 0 0 0 - 95 Transaction 0 1 3 0 iDb=0 tx_mode=Read - 96 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 57 0 0 Start at 57 + 1 OpenRead 0 2 0 k(3,B,B,B) 0 table=h_orders, root=2, iDb=0 + 2 OpenRead 1 3 0 k(4,B,B,B,B) 0 table=h_items, root=3, iDb=0 + 3 OpenRead 2 4 0 k(3,B,B,B) 0 table=h_returns, root=4, iDb=0 + 4 Once 14 0 0 0 goto 14 + 5 OpenRead 4 2 0 k(3,B,B,B) 0 table=h_orders, root=2, iDb=0 + 6 Rewind 4 13 0 0 Rewind table h_orders + 7 Column 4 1 4 0 r[4]=h_orders.customer_name + 8 Affinity 4 1 0 0 r[4..5] = A + 9 RowId 4 5 0 0 r[5]=h_orders.rowid + 10 Column 4 1 6 0 r[6]=h_orders.customer_name + 11 HashBuild 4 4 1 r=[1] budget=67108864 payload=r[5]..r[6] 0 + 12 Next 4 7 0 0 + 13 HashBuildFinalize 1 0 0 0 + 14 Rewind 1 43 0 0 Rewind table h_items + 15 Column 1 1 7 0 r[7]=h_items.order_name + 16 Affinity 7 1 0 0 r[7..8] = A + 17 RowId 1 10 0 0 r[10]=h_items.rowid + 18 Integer 0 11 0 0 r[11]=0 + 19 HashProbe 1 7 1 r[12]=42 payload=r[8]..r[9] 0 + 20 Once 29 0 0 0 goto 29 + 21 OpenAutoindex 3 0 0 0 cursor=3 + 22 Rewind 2 23 0 0 Rewind table h_returns + 23 ColumnRange 2 1 13 2 0 r[13..14]=h_returns.order_name..reason + 24 RowId 2 15 0 0 r[15]=h_returns.rowid + 25 MakeRecord 13 3 16 0 r[16]=mkrec(r[13..15]); for ephemeral_h_returns_t3 + 26 FilterAdd 3 13 1 0 bloom_filter_add(r[13..14]) + 27 IdxInsert 3 16 13 0 key=r[16] + 28 Next 2 23 0 0 + 29 Copy 9 17 0 0 r[17]=r[9] + 30 Filter 3 39 17 1 if !bloom_filter(r[17..18]) goto 39 + 31 SeekGE 3 39 17 0 key=[17..17] + 32 IdxGT 3 39 17 0 key=[17..17] + 33 DeferredSeek 3 2 0 0 + 34 Copy 8 1 0 0 r[1]=r[8] + 35 Column 1 2 2 0 r[2]=h_items.product + 36 Column 3 1 3 0 r[3]=ephemeral(ephemeral_h_returns_t3).reason + 37 ResultRow 1 3 0 0 output=r[1..3] + 38 Next 3 32 0 0 + 39 IfPos 11 50 0 0 r[11]>0 -> r[11]-=0, goto 50 + 40 HashNext 1 12 42 payload=r[8]..r[9] 0 + 41 Goto 0 20 0 0 + 42 Next 1 15 0 0 + 43 HashGraceInit 1 0 55 0 hash_table_id=1 + 44 Integer 1 11 0 0 r[11]=1 + 45 HashGraceLoadPart 1 0 54 0 hash_table_id=1 + 46 HashGraceNextProbe 1 7 52 0 hash_table_id=1 keys=r[7]..r[7] probe_rowid_dest=r[10] + 47 SeekRowid 1 10 46 0 if (r[10]!=cursor 1 for table h_items.rowid) goto 46 + 48 HashProbe 1 7 1 r[12]=46 payload=r[8]..r[9] 0 + 49 Goto 0 20 0 0 + 50 HashNext 1 12 46 payload=r[8]..r[9] 0 + 51 Goto 0 20 0 0 + 52 HashGraceAdvPart 1 0 54 0 hash_table_id=1 + 53 Goto 0 45 0 0 + 54 Integer 0 11 0 0 r[11]=0 + 55 HashClose 1 0 0 0 + 56 Halt 0 0 0 0 + 57 Transaction 0 1 3 0 iDb=0 tx_mode=Read + 58 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-with-order-by.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-with-order-by.snap index bb8f73ea..1608e8be 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-with-order-by.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__hash-join-with-order-by.snap @@ -22,7 +22,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 51 0 0 Start at 51 + 0 Init 0 50 0 0 Start at 50 1 SorterOpen 0 2 0 k(2,B,B) 0 cursor=0 2 OpenRead 1 2 0 k(3,B,B,B) 0 table=h_orders, root=2, iDb=0 3 OpenRead 2 3 0 k(4,B,B,B,B) 0 table=h_items, root=3, iDb=0 @@ -65,13 +65,12 @@ addr opcode p1 p2 p3 p4 40 Integer 0 12 0 0 r[12]=0 41 HashClose 1 0 0 0 42 OpenPseudo 4 4 3 0 3 columns in r[4] - 43 SorterSort 0 50 0 0 + 43 SorterSort 0 49 0 0 44 SorterData 0 4 4 0 r[4]=data 45 Column 4 2 1 0 r[1]=pseudo.column 2 - 46 Column 4 0 2 0 r[2]=pseudo.column 0 - 47 Column 4 1 3 0 r[3]=pseudo.column 1 - 48 ResultRow 1 3 0 0 output=r[1..3] - 49 SorterNext 0 44 0 0 - 50 Halt 0 0 0 0 - 51 Transaction 0 1 3 0 iDb=0 tx_mode=Read - 52 Goto 0 1 0 0 + 46 ColumnRange 4 0 2 2 0 r[2..3]=pseudo.column 0..column 1 + 47 ResultRow 1 3 0 0 output=r[1..3] + 48 SorterNext 0 44 0 0 + 49 Halt 0 0 0 0 + 50 Transaction 0 1 3 0 iDb=0 tx_mode=Read + 51 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__index-assisted-join-range-scan.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__index-assisted-join-range-scan.snap index 6744523f..c2ea2000 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__index-assisted-join-range-scan.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__index-assisted-join-range-scan.snap @@ -23,5 +23,5 @@ info: --- QUERY PLAN |--SEARCH c USING INTEGER PRIMARY KEY (rowid=?) -|--SEARCH o USING INDEX idx_orders_customer_id (customer_id=?) +|--SEARCH o USING COVERING INDEX idx_orders_customer_id (customer_id=?) `--USE SORTER FOR GROUP BY diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__join-with-subquery.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__join-with-subquery.snap index 89ad63d1..2d75470f 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__join-with-subquery.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__join-with-subquery.snap @@ -33,7 +33,7 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN recent_orders +|--SCAN customers AS c +|--SEARCH recent_orders USING INDEX ephemeral_subquery_t3 (customer_id=?) | `--SCAN orders USING INDEX idx_orders_customer_id -|--SEARCH c USING INTEGER PRIMARY KEY (rowid=?) `--USE SORTER FOR ORDER BY diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-basic.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-basic.snap index d05dcd98..0a554265 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-basic.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-basic.snap @@ -46,7 +46,7 @@ addr opcode p1 p2 p3 p4 p5 comment 18 NullRow 1 0 0 0 Set cursor 1 to a (pseudo) NULL row 19 NullRow 2 0 0 0 Set cursor 2 to a (pseudo) NULL row 20 Goto 0 10 0 0 - 21 Next 0 5 0 0 + 21 Next 0 5 0 1 22 Halt 0 0 0 0 23 Transaction 0 1 10 0 iDb=0 tx_mode=Read 24 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-chained.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-chained.snap index 5a518915..292c9aa3 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-chained.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-chained.snap @@ -27,7 +27,7 @@ info: --- QUERY PLAN |--SCAN customers AS c -|--SEARCH o USING INDEX idx_orders_customer_id (customer_id=?) LEFT-JOIN +|--SEARCH o USING COVERING INDEX idx_orders_customer_id (customer_id=?) LEFT-JOIN |--SEARCH oi USING INDEX idx_order_items_order_id (order_id=?) LEFT-JOIN `--SEARCH p USING INTEGER PRIMARY KEY (rowid=?) LEFT-JOIN @@ -72,7 +72,7 @@ addr opcode p1 p2 p3 p4 p5 comment 36 IfPos 5 39 0 0 r[5]>0 -> r[5]-=0, goto 39 37 NullRow 1 0 0 0 Set cursor 1 to a (pseudo) NULL row 38 Goto 0 11 0 0 - 39 Next 0 7 0 0 + 39 Next 0 7 0 1 40 Halt 0 0 0 0 41 Transaction 0 1 10 0 iDb=0 tx_mode=Read 42 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-find-nulls.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-find-nulls.snap index f258be7a..dd361968 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-find-nulls.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-find-nulls.snap @@ -22,30 +22,29 @@ info: --- QUERY PLAN |--SCAN customers AS c -`--SEARCH o USING INDEX idx_orders_customer_id (customer_id=?) LEFT-JOIN +`--SEARCH o USING COVERING INDEX idx_orders_customer_id (customer_id=?) LEFT-JOIN BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 21 0 0 Start at 21 - 1 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=customers, root=2, iDb=0 - 2 OpenRead 1 8 0 k(2,B) 0 index=idx_orders_customer_id, root=8, iDb=0 - 3 Rewind 0 20 0 0 Rewind table customers - 4 Integer 0 4 0 0 r[4]=0 - 5 RowId 0 5 0 0 r[5]=customers.rowid - 6 SeekGE 1 16 5 0 key=[5..5] - 7 IdxGT 1 16 5 0 key=[5..5] - 8 Integer 1 4 0 0 r[4]=1 - 9 IdxRowId 1 6 0 0 r[6]=cursor 1 for index idx_orders_customer_id.rowid - 10 NotNull 6 15 0 0 r[6]!=NULL -> goto 15 - 11 RowId 0 1 0 0 r[1]=customers.rowid - 12 Column 0 1 2 0 r[2]=customers.name - 13 Column 0 2 3 0 r[3]=customers.email - 14 ResultRow 1 3 0 0 output=r[1..3] - 15 Next 1 7 0 0 - 16 IfPos 4 19 0 0 r[4]>0 -> r[4]-=0, goto 19 - 17 NullRow 1 0 0 0 Set cursor 1 to a (pseudo) NULL row - 18 Goto 0 8 0 0 - 19 Next 0 4 0 0 - 20 Halt 0 0 0 0 - 21 Transaction 0 1 10 0 iDb=0 tx_mode=Read - 22 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 20 0 0 Start at 20 + 1 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=customers, root=2, iDb=0 + 2 OpenRead 1 8 0 k(2,B) 0 index=idx_orders_customer_id, root=8, iDb=0 + 3 Rewind 0 19 0 0 Rewind table customers + 4 Integer 0 4 0 0 r[4]=0 + 5 RowId 0 5 0 0 r[5]=customers.rowid + 6 SeekGE 1 15 5 0 key=[5..5] + 7 IdxGT 1 15 5 0 key=[5..5] + 8 Integer 1 4 0 0 r[4]=1 + 9 IdxRowId 1 6 0 0 r[6]=cursor 1 for index idx_orders_customer_id.rowid + 10 NotNull 6 14 0 0 r[6]!=NULL -> goto 14 + 11 RowId 0 1 0 0 r[1]=customers.rowid + 12 ColumnRange 0 1 2 2 0 r[2..3]=customers.name..email + 13 ResultRow 1 3 0 0 output=r[1..3] + 14 Next 1 7 0 0 + 15 IfPos 4 18 0 0 r[4]>0 -> r[4]-=0, goto 18 + 16 NullRow 1 0 0 0 Set cursor 1 to a (pseudo) NULL row + 17 Goto 0 8 0 0 + 18 Next 0 4 0 1 + 19 Halt 0 0 0 0 + 20 Transaction 0 1 10 0 iDb=0 tx_mode=Read + 21 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-with-aggregation.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-with-aggregation.snap index ed294987..e7e85d8a 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-with-aggregation.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__left-outer-join-with-aggregation.snap @@ -31,13 +31,13 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 80 0 0 Start at 80 + 0 Init 0 75 0 0 Start at 75 1 SorterOpen 0 2 0 k(2,-B,Binary) 0 cursor=0 2 Null 0 13 14 0 r[13..14]=NULL 3 SorterOpen 1 4 0 k(2,B,B) 0 cursor=1 4 Integer 0 8 0 0 r[8]=0; clear group by abort flag 5 Null 0 9 10 0 r[9..10]=NULL; initialize group by comparison registers to NULL - 6 Gosub 20 67 0 0 ; go to clear accumulator subroutine + 6 Gosub 20 64 0 0 ; go to clear accumulator subroutine 7 OpenRead 3 2 0 k(5,B,B,B,B,B) 0 table=customers, root=2, iDb=0 8 OpenRead 4 4 0 k(5,B,B,B,B,B) 0 table=orders, root=4, iDb=0 9 OpenRead 5 8 0 k(2,B) 0 index=idx_orders_customer_id, root=8, iDb=0 @@ -59,57 +59,52 @@ addr opcode p1 p2 p3 p4 p5 comment 25 NullRow 4 0 0 0 Set cursor 4 to a (pseudo) NULL row 26 NullRow 5 0 0 0 Set cursor 5 to a (pseudo) NULL row 27 Goto 0 16 0 0 - 28 Next 3 11 0 0 + 28 Next 3 11 0 1 29 OpenPseudo 2 15 4 0 4 columns in r[15] - 30 SorterSort 1 49 0 0 + 30 SorterSort 1 46 0 0 31 SorterData 1 15 2 0 r[15]=data - 32 Column 2 0 23 0 r[23]=pseudo.column 0 - 33 Column 2 1 24 0 r[24]=pseudo.column 1 - 34 Compare 9 23 2 k(2, Binary, Binary) 0 r[9..10]==r[23..24] - 35 Jump 36 40 36 0 ; start new group if comparison is not equal - 36 Gosub 6 53 0 0 ; check if ended group had data, and output if so - 37 Move 23 9 2 0 r[9..10]=r[23..24] - 38 IfPos 8 70 0 0 r[8]>0 -> r[8]-=0, goto 70; check abort flag - 39 Gosub 20 67 0 0 ; goto clear accumulator subroutine - 40 Column 2 2 25 0 r[25]=pseudo.column 2 - 41 Column 2 3 26 0 r[26]=pseudo.column 3 - 42 AggStep 0 25 13 count 0 accum=r[13] step(r[25]) - 43 AggStep 0 26 14 sum 0 accum=r[14] step(r[26]) - 44 If 7 47 0 0 if r[7] goto 47; don't emit group columns if continuing existing group - 45 Column 2 0 11 0 r[11]=pseudo.column 0 - 46 Column 2 1 12 0 r[12]=pseudo.column 1 - 47 Integer 1 7 0 0 r[7]=1; indicate data in accumulator - 48 SorterNext 1 31 0 0 - 49 Gosub 6 53 0 0 ; emit row for final group - 50 Goto 0 70 0 0 ; group by finished - 51 Integer 1 8 0 0 r[8]=1 - 52 Return 6 0 0 0 - 53 IfPos 7 55 0 0 r[7]>0 -> r[7]-=0, goto 55; output group by row subroutine start - 54 Return 6 0 0 0 - 55 AggFinal 0 13 0 count 0 accum=r[13] - 56 AggFinal 0 14 0 sum 0 accum=r[14] - 57 Copy 14 27 0 0 r[27]=r[14] - 58 NotNull 27 60 0 0 r[27]!=NULL -> goto 60 - 59 Integer 0 27 0 0 r[27]=0 - 60 Sequence 0 28 0 0 - 61 Copy 11 29 0 0 r[29]=r[11] - 62 Copy 12 30 0 0 r[30]=r[12] - 63 Copy 13 31 0 0 r[31]=r[13] - 64 MakeRecord 27 5 5 0 r[5]=mkrec(r[27..31]) - 65 SorterInsert 0 5 0 0 0 key=r[5] - 66 Return 6 0 0 0 - 67 Null 0 11 14 0 r[11..14]=NULL; clear accumulator subroutine start - 68 Integer 0 7 0 0 r[7]=0 - 69 Return 20 0 0 0 - 70 OpenPseudo 6 5 5 0 5 columns in r[5] - 71 SorterSort 0 79 0 0 - 72 SorterData 0 5 6 0 r[5]=data - 73 Column 6 2 1 0 r[1]=pseudo.column 2 - 74 Column 6 3 2 0 r[2]=pseudo.column 3 - 75 Column 6 4 3 0 r[3]=pseudo.column 4 - 76 Column 6 0 4 0 r[4]=pseudo.column 0 - 77 ResultRow 1 4 0 0 output=r[1..4] - 78 SorterNext 0 72 0 0 - 79 Halt 0 0 0 0 - 80 Transaction 0 1 10 0 iDb=0 tx_mode=Read - 81 Goto 0 1 0 0 + 32 ColumnRange 2 0 23 2 0 r[23..24]=pseudo.column 0..column 1 + 33 Compare 9 23 2 k(2, Binary, Binary) 0 r[9..10]==r[23..24] + 34 Jump 35 39 35 0 ; start new group if comparison is not equal + 35 Gosub 6 50 0 0 ; check if ended group had data, and output if so + 36 Move 23 9 2 0 r[9..10]=r[23..24] + 37 IfPos 8 67 0 0 r[8]>0 -> r[8]-=0, goto 67; check abort flag + 38 Gosub 20 64 0 0 ; goto clear accumulator subroutine + 39 ColumnRange 2 2 25 2 0 r[25..26]=pseudo.column 2..column 3 + 40 AggStep 0 25 13 count 0 accum=r[13] step(r[25]) + 41 AggStep 0 26 14 sum 0 accum=r[14] step(r[26]) + 42 If 7 44 0 0 if r[7] goto 44; don't emit group columns if continuing existing group + 43 ColumnRange 2 0 11 2 0 r[11..12]=pseudo.column 0..column 1 + 44 Integer 1 7 0 0 r[7]=1; indicate data in accumulator + 45 SorterNext 1 31 0 0 + 46 Gosub 6 50 0 0 ; emit row for final group + 47 Goto 0 67 0 0 ; group by finished + 48 Integer 1 8 0 0 r[8]=1 + 49 Return 6 0 0 0 + 50 IfPos 7 52 0 0 r[7]>0 -> r[7]-=0, goto 52; output group by row subroutine start + 51 Return 6 0 0 0 + 52 AggFinal 0 13 0 count 0 accum=r[13] + 53 AggFinal 0 14 0 sum 0 accum=r[14] + 54 Copy 14 27 0 0 r[27]=r[14] + 55 NotNull 27 57 0 0 r[27]!=NULL -> goto 57 + 56 Integer 0 27 0 0 r[27]=0 + 57 Sequence 0 28 0 0 + 58 Copy 11 29 0 0 r[29]=r[11] + 59 Copy 12 30 0 0 r[30]=r[12] + 60 Copy 13 31 0 0 r[31]=r[13] + 61 MakeRecord 27 5 5 0 r[5]=mkrec(r[27..31]) + 62 SorterInsert 0 5 0 0 0 key=r[5] + 63 Return 6 0 0 0 + 64 Null 0 11 14 0 r[11..14]=NULL; clear accumulator subroutine start + 65 Integer 0 7 0 0 r[7]=0 + 66 Return 20 0 0 0 + 67 OpenPseudo 6 5 5 0 5 columns in r[5] + 68 SorterSort 0 74 0 0 + 69 SorterData 0 5 6 0 r[5]=data + 70 ColumnRange 6 2 1 3 0 r[1..3]=pseudo.column 2..column 4 + 71 Column 6 0 4 0 r[4]=pseudo.column 0 + 72 ResultRow 1 4 0 0 output=r[1..4] + 73 SorterNext 0 69 0 0 + 74 Halt 0 0 0 0 + 75 Transaction 0 1 10 0 iDb=0 tx_mode=Read + 76 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__self-join-category-hierarchy.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__self-join-category-hierarchy.snap index db41ed7a..f8509807 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__self-join-category-hierarchy.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__self-join-category-hierarchy.snap @@ -38,7 +38,7 @@ addr opcode p1 p2 p3 p4 p5 comment 12 IfPos 4 15 0 0 r[4]>0 -> r[4]-=0, goto 15 13 NullRow 1 0 0 0 Set cursor 1 to a (pseudo) NULL row 14 Goto 0 7 0 0 - 15 Next 0 4 0 0 + 15 Next 0 4 0 1 16 Halt 0 0 0 0 17 Transaction 0 1 10 0 iDb=0 tx_mode=Read 18 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__self-join-employee-manager.snap b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__self-join-employee-manager.snap index 0740333a..1b0bf955 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__self-join-employee-manager.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/joins/snapshots/joins__self-join-employee-manager.snap @@ -40,7 +40,7 @@ addr opcode p1 p2 p3 p4 p5 comment 13 IfPos 5 16 0 0 r[5]>0 -> r[5]-=0, goto 16 14 NullRow 1 0 0 0 Set cursor 1 to a (pseudo) NULL row 15 Goto 0 7 0 0 - 16 Next 0 4 0 0 + 16 Next 0 4 0 1 17 Halt 0 0 0 0 18 Transaction 0 1 10 0 iDb=0 tx_mode=Read 19 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__delete-with-subquery.snap b/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__delete-with-subquery.snap index 25ca646e..574ea0c4 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__delete-with-subquery.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__delete-with-subquery.snap @@ -17,7 +17,7 @@ info: QUERY PLAN |--LIST SUBQUERY 1 | `--SCAN users -`--SEARCH orders USING INDEX idx_orders_user_id (user_id=?) +`--SEARCH orders USING COVERING INDEX idx_orders_user_id (user_id=?) BYTECODE addr opcode p1 p2 p3 p4 p5 comment @@ -35,7 +35,7 @@ addr opcode p1 p2 p3 p4 p5 comment 11 RowId 4 2 0 0 r[2]=users.rowid 12 MakeRecord 2 1 6 0 r[6]=mkrec(r[2..2]); for ephemeral_index_where_sub_t2 13 IdxInsert 0 6 0 8 key=r[6] - 14 Next 4 9 0 0 + 14 Next 4 9 0 1 15 OpenRead 3 12 0 k(2,B) 0 index=idx_orders_user_id, root=12, iDb=0 16 NullRow 0 0 0 0 Set cursor 0 to a (pseudo) NULL row 17 Rewind 0 27 0 0 Rewind ephemeral(ephemeral_index_where_sub_t2) diff --git a/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__delete-with-where.snap b/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__delete-with-where.snap index c2c3caea..c20a5e50 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__delete-with-where.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__delete-with-where.snap @@ -35,7 +35,7 @@ addr opcode p1 p2 p3 p4 p5 comment 15 RowId 0 11 0 0 r[11]=orders.rowid 16 IdxDelete 2 10 2 1 17 Delete 0 0 0 orders 0 - 18 Next 0 5 0 0 + 18 Next 0 5 0 1 19 Halt 0 0 0 0 20 Transaction 0 2 10 0 iDb=0 tx_mode=Write 21 Real 0 3 0 10.0 0 r[3]=10 diff --git a/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__insert-select.snap b/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__insert-select.snap index 7cd4d9bf..4e50b23e 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__insert-select.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__insert-select.snap @@ -19,48 +19,45 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 42 0 0 Start at 42 - 1 InitCoroutine 2 15 2 0 + 0 Init 0 39 0 0 Start at 39 + 1 InitCoroutine 2 12 2 0 2 OpenRead 0 5 0 k(6,B,B,B,B,B,B) 0 table=orders, root=5, iDb=0 - 3 Rewind 0 14 0 0 Rewind table orders + 3 Rewind 0 11 0 0 Rewind table orders 4 Column 0 5 9 0 r[9]=orders.created_at - 5 Ge 9 10 13 Binary 0 if r[9]>=r[10] goto 13 + 5 Ge 9 10 10 Binary 0 if r[9]>=r[10] goto 10 6 RowId 0 3 0 0 r[3]=orders.rowid - 7 Column 0 1 4 0 r[4]=orders.user_id - 8 Column 0 2 5 0 r[5]=orders.product_id - 9 Column 0 3 6 0 r[6]=orders.quantity - 10 Column 0 4 7 0 r[7]=orders.total_price - 11 RealAffinity 7 0 0 0 - 12 Yield 2 0 0 0 - 13 Next 0 4 0 0 - 14 EndCoroutine 2 0 0 0 - 15 OpenWrite 1 7 0 0 root=7; iDb=0 - 16 Yield 2 40 0 0 - 17 Copy 3 11 0 0 r[11]=r[3] - 18 SoftNull 12 0 0 0 - 19 Copy 4 13 0 0 r[13]=r[4] - 20 Copy 5 14 0 0 r[14]=r[5] - 21 Copy 6 15 0 0 r[15]=r[6] - 22 Copy 7 16 0 0 r[16]=r[7] - 23 Null 0 17 0 0 r[17]=NULL - 24 Affinity 12 6 0 0 r[12..18] = D, D, D, D, E, B - 25 NotNull 11 27 0 0 r[11]!=NULL -> goto 27 - 26 Goto 0 29 0 0 - 27 MustBeInt 11 0 0 0 - 28 Goto 0 30 0 0 - 29 NewRowid 1 11 0 0 r[11]=rowid - 30 HaltIfNull 1299 0 13 order_archive.user_id 0 - 31 HaltIfNull 1299 0 14 order_archive.product_id 0 - 32 HaltIfNull 1299 0 15 order_archive.quantity 0 - 33 HaltIfNull 1299 0 16 order_archive.total_price 0 - 34 NotExists 1 36 11 0 - 35 Halt 1555 0 0 order_archive.id 0 - 36 MakeRecord 12 6 18 0 r[18]=mkrec(r[12..17]) - 37 Insert 1 18 11 order_archive 0 intkey=r[11] data=r[18] - 38 Goto 0 39 0 0 - 39 Goto 0 16 0 0 - 40 Goto 0 41 0 0 - 41 Halt 0 0 0 0 - 42 Transaction 0 2 10 0 iDb=0 tx_mode=Write - 43 String8 0 10 0 2024-01-01 0 r[10]='2024-01-01' - 44 Goto 0 1 0 0 + 7 ColumnRange 0 1 4 4 0 r[4..7]=orders.user_id..total_price + 8 RealAffinity 7 0 0 0 + 9 Yield 2 0 0 0 + 10 Next 0 4 0 1 + 11 EndCoroutine 2 0 0 0 + 12 OpenWrite 1 7 0 0 root=7; iDb=0 + 13 Yield 2 37 0 0 + 14 Copy 3 11 0 0 r[11]=r[3] + 15 SoftNull 12 0 0 0 + 16 Copy 4 13 0 0 r[13]=r[4] + 17 Copy 5 14 0 0 r[14]=r[5] + 18 Copy 6 15 0 0 r[15]=r[6] + 19 Copy 7 16 0 0 r[16]=r[7] + 20 Null 0 17 0 0 r[17]=NULL + 21 Affinity 12 6 0 0 r[12..18] = D, D, D, D, E, B + 22 NotNull 11 24 0 0 r[11]!=NULL -> goto 24 + 23 Goto 0 26 0 0 + 24 MustBeInt 11 0 0 0 + 25 Goto 0 27 0 0 + 26 NewRowid 1 11 0 0 r[11]=rowid + 27 HaltIfNull 1299 0 13 order_archive.user_id 0 + 28 HaltIfNull 1299 0 14 order_archive.product_id 0 + 29 HaltIfNull 1299 0 15 order_archive.quantity 0 + 30 HaltIfNull 1299 0 16 order_archive.total_price 0 + 31 NotExists 1 33 11 0 + 32 Halt 1555 0 0 order_archive.id 0 + 33 MakeRecord 12 6 18 0 r[18]=mkrec(r[12..17]) + 34 Insert 1 18 11 order_archive 0 intkey=r[11] data=r[18] + 35 Goto 0 36 0 0 + 36 Goto 0 13 0 0 + 37 Goto 0 38 0 0 + 38 Halt 0 0 0 0 + 39 Transaction 0 2 10 0 iDb=0 tx_mode=Write + 40 String8 0 10 0 2024-01-01 0 r[10]='2024-01-01' + 41 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__update-where-self-subquery.snap b/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__update-where-self-subquery.snap index 5a72f32b..ee006cf0 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__update-where-self-subquery.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__update-where-self-subquery.snap @@ -23,7 +23,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 36 0 0 Start at 36 + 0 Init 0 35 0 0 Start at 35 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true 2 OpenRead 1 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 3 Rewind 1 19 0 0 Rewind table products @@ -41,26 +41,25 @@ addr opcode p1 p2 p3 p4 p5 comment 15 RowId 1 2 0 0 r[2]=products.rowid 16 MakeRecord 2 1 11 0 r[11]=mkrec(r[2..2]); for ephemeral_scratch 17 Insert 0 11 2 ephemeral_scratch 6 intkey=r[2] data=r[11] - 18 Next 1 4 0 0 + 18 Next 1 4 0 1 19 OpenWrite 1 2 0 0 root=2; iDb=0 - 20 Rewind 0 35 0 0 Rewind table ephemeral(ephemeral_scratch) + 20 Rewind 0 34 0 0 Rewind table ephemeral(ephemeral_scratch) 21 RowId 0 12 0 0 r[12]=ephemeral(ephemeral_scratch).rowid - 22 NotExists 1 34 12 0 + 22 NotExists 1 33 12 0 23 Null 0 13 0 0 r[13]=NULL - 24 Column 1 1 14 0 r[14]=products.name - 25 Column 1 2 15 0 r[15]=products.price - 26 Column 1 3 18 0 0 r[18]=products.quantity - 27 Add 18 19 16 0 r[16]=r[18]+r[19] - 28 Column 1 4 17 0 r[17]=products.category - 29 Affinity 13 5 0 0 r[13..18] = D, B, E, D, B - 30 MakeRecord 13 5 20 0 r[20]=mkrec(r[13..17]) - 31 NotExists 1 34 12 0 - 32 Delete 1 0 0 products 0 - 33 Insert 1 20 12 products 11 intkey=r[12] data=r[20] - 34 Next 0 21 0 0 - 35 Halt 0 0 0 0 - 36 Transaction 0 2 10 0 iDb=0 tx_mode=Write - 37 Integer 1 7 0 0 r[7]=1 - 38 Integer 5 10 0 0 r[10]=5 - 39 Integer 1 19 0 0 r[19]=1 - 40 Goto 0 1 0 0 + 24 ColumnRange 1 1 14 2 0 r[14..15]=products.name..price + 25 Column 1 3 18 0 0 r[18]=products.quantity + 26 Add 18 19 16 0 r[16]=r[18]+r[19] + 27 Column 1 4 17 0 r[17]=products.category + 28 Affinity 13 5 0 0 r[13..18] = D, B, E, D, B + 29 MakeRecord 13 5 20 0 r[20]=mkrec(r[13..17]) + 30 NotExists 1 33 12 0 + 31 Delete 1 0 0 products 0 + 32 Insert 1 20 12 products 11 intkey=r[12] data=r[20] + 33 Next 0 21 0 1 + 34 Halt 0 0 0 0 + 35 Transaction 0 2 10 0 iDb=0 tx_mode=Write + 36 Integer 1 7 0 0 r[7]=1 + 37 Integer 5 10 0 0 r[10]=5 + 38 Integer 1 19 0 0 r[19]=1 + 39 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__update-where-unrelated-subquery.snap b/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__update-where-unrelated-subquery.snap index 994d8994..fc292917 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__update-where-unrelated-subquery.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/modifications/snapshots/modifications__update-where-unrelated-subquery.snap @@ -23,7 +23,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 33 0 0 Start at 33 + 0 Init 0 32 0 0 Start at 32 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true 2 OpenRead 1 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 3 Rewind 1 16 0 0 Rewind table products @@ -38,24 +38,23 @@ addr opcode p1 p2 p3 p4 p5 comment 12 RowId 1 2 0 0 r[2]=products.rowid 13 MakeRecord 2 1 6 0 r[6]=mkrec(r[2..2]); for ephemeral_scratch 14 Insert 0 6 2 ephemeral_scratch 6 intkey=r[2] data=r[6] - 15 Next 1 4 0 0 + 15 Next 1 4 0 1 16 OpenWrite 1 2 0 0 root=2; iDb=0 - 17 Rewind 0 32 0 0 Rewind table ephemeral(ephemeral_scratch) + 17 Rewind 0 31 0 0 Rewind table ephemeral(ephemeral_scratch) 18 RowId 0 7 0 0 r[7]=ephemeral(ephemeral_scratch).rowid - 19 NotExists 1 31 7 0 + 19 NotExists 1 30 7 0 20 Null 0 8 0 0 r[8]=NULL - 21 Column 1 1 9 0 r[9]=products.name - 22 Column 1 2 10 0 r[10]=products.price - 23 Column 1 3 13 0 0 r[13]=products.quantity - 24 Add 13 14 11 0 r[11]=r[13]+r[14] - 25 Column 1 4 12 0 r[12]=products.category - 26 Affinity 8 5 0 0 r[8..13] = D, B, E, D, B - 27 MakeRecord 8 5 15 0 r[15]=mkrec(r[8..12]) - 28 NotExists 1 31 7 0 - 29 Delete 1 0 0 products 0 - 30 Insert 1 15 7 products 11 intkey=r[7] data=r[15] - 31 Next 0 18 0 0 - 32 Halt 0 0 0 0 - 33 Transaction 0 2 10 0 iDb=0 tx_mode=Write - 34 Integer 1 14 0 0 r[14]=1 - 35 Goto 0 1 0 0 + 21 ColumnRange 1 1 9 2 0 r[9..10]=products.name..price + 22 Column 1 3 13 0 0 r[13]=products.quantity + 23 Add 13 14 11 0 r[11]=r[13]+r[14] + 24 Column 1 4 12 0 r[12]=products.category + 25 Affinity 8 5 0 0 r[8..13] = D, B, E, D, B + 26 MakeRecord 8 5 15 0 r[15]=mkrec(r[8..12]) + 27 NotExists 1 30 7 0 + 28 Delete 1 0 0 products 0 + 29 Insert 1 15 7 products 11 intkey=r[7] data=r[15] + 30 Next 0 18 0 1 + 31 Halt 0 0 0 0 + 32 Transaction 0 2 10 0 iDb=0 tx_mode=Write + 33 Integer 1 14 0 0 r[14]=1 + 34 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__delete-returning-expr.snap b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__delete-returning-expr.snap index 34e9de96..9de40bad 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__delete-returning-expr.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__delete-returning-expr.snap @@ -15,41 +15,39 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 35 0 0 Start at 35 + 0 Init 0 33 0 0 Start at 33 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true 2 OpenWrite 1 2 0 0 root=2; iDb=0 3 OpenWrite 2 3 0 0 root=3; iDb=0 - 4 Rewind 1 28 0 0 Rewind table t1 + 4 Rewind 1 27 0 0 Rewind table t1 5 Column 1 2 2 0 r[2]=t1.value 6 RealAffinity 2 0 0 0 - 7 Le 2 3 27 Binary 0 if r[2]<=r[3] goto 27 + 7 Le 2 3 26 Binary 0 if r[2]<=r[3] goto 26 8 RowId 1 4 0 0 r[4]=t1.rowid 9 RowId 1 5 0 0 r[5]=t1.rowid - 10 Column 1 1 6 0 r[6]=t1.name - 11 Column 1 2 7 0 r[7]=t1.value - 12 Column 1 1 8 0 r[8]=t1.name - 13 RowId 1 9 0 0 r[9]=t1.rowid - 14 IdxDelete 2 8 2 1 - 15 Delete 1 0 0 t1 0 - 16 Copy 4 10 0 0 r[10]=r[4] - 17 Copy 6 11 0 0 r[11]=r[6] - 18 Copy 7 12 0 0 r[12]=r[7] - 19 Copy 4 13 0 0 r[13]=r[4] - 20 Copy 6 14 0 0 r[14]=r[6] - 21 Copy 7 15 0 0 r[15]=r[7] - 22 Copy 13 16 0 0 r[16]=r[13] - 23 Copy 14 17 0 0 r[17]=r[14] - 24 MakeRecord 16 2 18 0 r[18]=mkrec(r[16..17]) - 25 NewRowid 0 19 0 0 r[19]=rowid - 26 Insert 0 18 19 4 intkey=r[19] data=r[18] - 27 Next 1 5 0 0 - 28 FkCheck 0 0 0 0 - 29 Rewind 0 34 0 0 Rewind ephemeral(t1) - 30 Column 0 0 20 0 r[20]=ephemeral(t1).id - 31 Column 0 1 21 0 r[21]=ephemeral(t1).name - 32 ResultRow 20 2 0 0 output=r[20..21] - 33 Next 0 30 0 0 - 34 Halt 0 0 0 0 - 35 Transaction 0 2 2 0 iDb=0 tx_mode=Write - 36 Real 0 3 0 15.0 0 r[3]=15 - 37 Goto 0 1 0 0 + 10 ColumnRange 1 1 6 2 0 r[6..7]=t1.name..value + 11 Column 1 1 8 0 r[8]=t1.name + 12 RowId 1 9 0 0 r[9]=t1.rowid + 13 IdxDelete 2 8 2 1 + 14 Delete 1 0 0 t1 0 + 15 Copy 4 10 0 0 r[10]=r[4] + 16 Copy 6 11 0 0 r[11]=r[6] + 17 Copy 7 12 0 0 r[12]=r[7] + 18 Copy 4 13 0 0 r[13]=r[4] + 19 Copy 6 14 0 0 r[14]=r[6] + 20 Copy 7 15 0 0 r[15]=r[7] + 21 Copy 13 16 0 0 r[16]=r[13] + 22 Copy 14 17 0 0 r[17]=r[14] + 23 MakeRecord 16 2 18 0 r[18]=mkrec(r[16..17]) + 24 NewRowid 0 19 0 0 r[19]=rowid + 25 Insert 0 18 19 4 intkey=r[19] data=r[18] + 26 Next 1 5 0 1 + 27 FkCheck 0 0 0 0 + 28 Rewind 0 32 0 0 Rewind ephemeral(t1) + 29 ColumnRange 0 0 20 2 0 r[20..21]=ephemeral(t1).id..name + 30 ResultRow 20 2 0 0 output=r[20..21] + 31 Next 0 29 0 0 + 32 Halt 0 0 0 0 + 33 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 34 Real 0 3 0 15.0 0 r[3]=15 + 35 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__delete-returning-star.snap b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__delete-returning-star.snap index 0dd055de..06d6d51b 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__delete-returning-star.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__delete-returning-star.snap @@ -15,39 +15,36 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 33 0 0 Start at 33 + 0 Init 0 30 0 0 Start at 30 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true 2 OpenWrite 1 2 0 0 root=2; iDb=0 3 OpenWrite 2 3 0 0 root=3; iDb=0 - 4 SeekRowid 1 1 25 0 if (r[1]!=cursor 1 for table t1.rowid) goto 25 + 4 SeekRowid 1 1 24 0 if (r[1]!=cursor 1 for table t1.rowid) goto 24 5 RowId 1 2 0 0 r[2]=t1.rowid 6 RowId 1 3 0 0 r[3]=t1.rowid - 7 Column 1 1 4 0 r[4]=t1.name - 8 Column 1 2 5 0 r[5]=t1.value - 9 Column 1 1 6 0 r[6]=t1.name - 10 RowId 1 7 0 0 r[7]=t1.rowid - 11 IdxDelete 2 6 2 1 - 12 Delete 1 0 0 t1 0 - 13 Copy 2 8 0 0 r[8]=r[2] - 14 Copy 4 9 0 0 r[9]=r[4] - 15 Copy 5 10 0 0 r[10]=r[5] - 16 Copy 2 11 0 0 r[11]=r[2] - 17 Copy 4 12 0 0 r[12]=r[4] - 18 Copy 5 13 0 0 r[13]=r[5] - 19 Copy 11 14 0 0 r[14]=r[11] - 20 Copy 12 15 0 0 r[15]=r[12] - 21 Copy 13 16 0 0 r[16]=r[13] - 22 MakeRecord 14 3 17 0 r[17]=mkrec(r[14..16]) - 23 NewRowid 0 18 0 0 r[18]=rowid - 24 Insert 0 17 18 4 intkey=r[18] data=r[17] - 25 FkCheck 0 0 0 0 - 26 Rewind 0 32 0 0 Rewind ephemeral(t1) - 27 Column 0 0 19 0 r[19]=ephemeral(t1).id - 28 Column 0 1 20 0 r[20]=ephemeral(t1).name - 29 Column 0 2 21 0 r[21]=ephemeral(t1).value - 30 ResultRow 19 3 0 0 output=r[19..21] - 31 Next 0 27 0 0 - 32 Halt 0 0 0 0 - 33 Transaction 0 2 2 0 iDb=0 tx_mode=Write - 34 Integer 2 1 0 0 r[1]=2 - 35 Goto 0 1 0 0 + 7 ColumnRange 1 1 4 2 0 r[4..5]=t1.name..value + 8 Column 1 1 6 0 r[6]=t1.name + 9 RowId 1 7 0 0 r[7]=t1.rowid + 10 IdxDelete 2 6 2 1 + 11 Delete 1 0 0 t1 0 + 12 Copy 2 8 0 0 r[8]=r[2] + 13 Copy 4 9 0 0 r[9]=r[4] + 14 Copy 5 10 0 0 r[10]=r[5] + 15 Copy 2 11 0 0 r[11]=r[2] + 16 Copy 4 12 0 0 r[12]=r[4] + 17 Copy 5 13 0 0 r[13]=r[5] + 18 Copy 11 14 0 0 r[14]=r[11] + 19 Copy 12 15 0 0 r[15]=r[12] + 20 Copy 13 16 0 0 r[16]=r[13] + 21 MakeRecord 14 3 17 0 r[17]=mkrec(r[14..16]) + 22 NewRowid 0 18 0 0 r[18]=rowid + 23 Insert 0 17 18 4 intkey=r[18] data=r[17] + 24 FkCheck 0 0 0 0 + 25 Rewind 0 29 0 0 Rewind ephemeral(t1) + 26 ColumnRange 0 0 19 3 0 r[19..21]=ephemeral(t1).id..value + 27 ResultRow 19 3 0 0 output=r[19..21] + 28 Next 0 26 0 0 + 29 Halt 0 0 0 0 + 30 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 31 Integer 2 1 0 0 r[1]=2 + 32 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__insert-returning-expr.snap b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__insert-returning-expr.snap index 0ae31bca..e291565d 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__insert-returning-expr.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__insert-returning-expr.snap @@ -14,7 +14,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 46 0 0 Start at 46 + 0 Init 0 44 0 0 Start at 44 1 OpenEphemeral 1 1 0 0 cursor=1 is_table=true 2 OpenWrite 2 2 0 0 root=2; iDb=0 3 Integer 1 2 0 0 r[2]=1 @@ -53,12 +53,10 @@ addr opcode p1 p2 p3 p4 p5 comment 36 Goto 0 37 0 0 37 Goto 0 38 0 0 38 FkCheck 0 0 0 0 - 39 Rewind 1 45 0 0 Rewind ephemeral(t1) - 40 Column 1 0 23 0 r[23]=ephemeral(t1).id - 41 Column 1 1 24 0 r[24]=ephemeral(t1).name - 42 Column 1 2 25 0 r[25]=ephemeral(t1).value - 43 ResultRow 23 3 0 0 output=r[23..25] - 44 Next 1 40 0 0 - 45 Halt 0 0 0 0 - 46 Transaction 0 2 2 0 iDb=0 tx_mode=Write - 47 Goto 0 1 0 0 + 39 Rewind 1 43 0 0 Rewind ephemeral(t1) + 40 ColumnRange 1 0 23 3 0 r[23..25]=ephemeral(t1).id..value + 41 ResultRow 23 3 0 0 output=r[23..25] + 42 Next 1 40 0 0 + 43 Halt 0 0 0 0 + 44 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 45 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__insert-returning-star.snap b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__insert-returning-star.snap index 93d5dc65..464b2ebf 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__insert-returning-star.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__insert-returning-star.snap @@ -14,7 +14,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 55 0 0 Start at 55 + 0 Init 0 53 0 0 Start at 53 1 OpenEphemeral 1 1 0 0 cursor=1 is_table=true 2 InitCoroutine 2 12 3 0 3 Integer 1 3 0 0 r[3]=1 @@ -62,12 +62,10 @@ addr opcode p1 p2 p3 p4 p5 comment 45 Goto 0 13 0 0 46 Goto 0 47 0 0 47 FkCheck 0 0 0 0 - 48 Rewind 1 54 0 0 Rewind ephemeral(t1) - 49 Column 1 0 24 0 r[24]=ephemeral(t1).id - 50 Column 1 1 25 0 r[25]=ephemeral(t1).name - 51 Column 1 2 26 0 r[26]=ephemeral(t1).value - 52 ResultRow 24 3 0 0 output=r[24..26] - 53 Next 1 49 0 0 - 54 Halt 0 0 0 0 - 55 Transaction 0 2 2 0 iDb=0 tx_mode=Write - 56 Goto 0 1 0 0 + 48 Rewind 1 52 0 0 Rewind ephemeral(t1) + 49 ColumnRange 1 0 24 3 0 r[24..26]=ephemeral(t1).id..value + 50 ResultRow 24 3 0 0 output=r[24..26] + 51 Next 1 49 0 0 + 52 Halt 0 0 0 0 + 53 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 54 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__update-returning-expr.snap b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__update-returning-expr.snap index 720b01b5..113b3f27 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__update-returning-expr.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__update-returning-expr.snap @@ -15,7 +15,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 34 0 0 Start at 34 + 0 Init 0 32 0 0 Start at 32 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true 2 OpenWrite 1 2 0 0 root=2; iDb=0 3 Rewind 1 26 0 0 Rewind table t1 @@ -40,15 +40,13 @@ addr opcode p1 p2 p3 p4 p5 comment 22 MakeRecord 11 3 16 0 r[16]=mkrec(r[11..13]) 23 NewRowid 0 17 0 0 r[17]=rowid 24 Insert 0 16 17 4 intkey=r[17] data=r[16] - 25 Next 1 4 0 0 + 25 Next 1 4 0 1 26 FkCheck 0 0 0 0 - 27 Rewind 0 33 0 0 Rewind ephemeral(t1) - 28 Column 0 0 18 0 r[18]=ephemeral(t1).id - 29 Column 0 1 19 0 r[19]=ephemeral(t1).name - 30 Column 0 2 20 0 r[20]=ephemeral(t1).value - 31 ResultRow 18 3 0 0 output=r[18..20] - 32 Next 0 28 0 0 - 33 Halt 0 0 0 0 - 34 Transaction 0 2 2 0 iDb=0 tx_mode=Write - 35 Integer 2 6 0 0 r[6]=2 - 36 Goto 0 1 0 0 + 27 Rewind 0 31 0 0 Rewind ephemeral(t1) + 28 ColumnRange 0 0 18 3 0 r[18..20]=ephemeral(t1).id..value + 29 ResultRow 18 3 0 0 output=r[18..20] + 30 Next 0 28 0 0 + 31 Halt 0 0 0 0 + 32 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 33 Integer 2 6 0 0 r[6]=2 + 34 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__update-returning-star.snap b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__update-returning-star.snap index 83d469e7..0552e0ad 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__update-returning-star.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__update-returning-star.snap @@ -15,7 +15,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 36 0 0 Start at 36 + 0 Init 0 34 0 0 Start at 34 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true 2 OpenWrite 1 2 0 0 root=2; iDb=0 3 OpenWrite 2 3 0 0 root=3; iDb=0 @@ -44,13 +44,11 @@ addr opcode p1 p2 p3 p4 p5 comment 26 Insert 0 15 16 4 intkey=r[16] data=r[15] 27 Next 2 6 0 0 28 FkCheck 0 0 0 0 - 29 Rewind 0 35 0 0 Rewind ephemeral(t1) - 30 Column 0 0 17 0 r[17]=ephemeral(t1).id - 31 Column 0 1 18 0 r[18]=ephemeral(t1).name - 32 Column 0 2 19 0 r[19]=ephemeral(t1).value - 33 ResultRow 17 3 0 0 output=r[17..19] - 34 Next 0 30 0 0 - 35 Halt 0 0 0 0 - 36 Transaction 0 2 2 0 iDb=0 tx_mode=Write - 37 Integer 1 7 0 0 r[7]=1 - 38 Goto 0 1 0 0 + 29 Rewind 0 33 0 0 Rewind ephemeral(t1) + 30 ColumnRange 0 0 17 3 0 r[17..19]=ephemeral(t1).id..value + 31 ResultRow 17 3 0 0 output=r[17..19] + 32 Next 0 30 0 0 + 33 Halt 0 0 0 0 + 34 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 35 Integer 1 7 0 0 r[7]=1 + 36 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__upsert-returning.snap b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__upsert-returning.snap index 24a914e3..3b84a8a8 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__upsert-returning.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/returning/snapshots/returning__upsert-returning.snap @@ -16,7 +16,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 58 0 0 Start at 58 + 0 Init 0 55 0 0 Start at 55 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true 2 OpenWrite 1 2 0 0 root=2; iDb=0 3 Integer 1 2 0 0 r[2]=1 @@ -43,36 +43,33 @@ addr opcode p1 p2 p3 p4 p5 comment 24 MakeRecord 10 3 13 0 r[13]=mkrec(r[10..12]) 25 NewRowid 0 14 0 0 r[14]=rowid 26 Insert 0 13 14 4 intkey=r[14] data=r[13] - 27 Goto 0 49 0 0 - 28 SeekRowid 1 1 49 0 if (r[1]!=cursor 1 for table t2.rowid) goto 49 + 27 Goto 0 48 0 0 + 28 SeekRowid 1 1 48 0 if (r[1]!=cursor 1 for table t2.rowid) goto 48 29 RowId 1 15 0 0 r[15]=t2.rowid - 30 Column 1 1 16 0 r[16]=t2.name - 31 Column 1 2 17 0 0 r[17]=t2.counter - 32 Copy 15 18 2 0 r[18]=r[15] - 33 Copy 17 21 0 0 r[21]=r[17] - 34 Integer 1 22 0 0 r[22]=1 - 35 Add 21 22 20 0 r[20]=r[21]+r[22] - 36 Affinity 18 3 0 0 r[18..21] = D, B, D - 37 MakeRecord 18 3 23 0 r[23]=mkrec(r[18..20]) - 38 Insert 1 23 1 t2 8 intkey=r[1] data=r[23] - 39 Copy 1 24 0 0 r[24]=r[1] - 40 Copy 19 25 0 0 r[25]=r[19] - 41 Copy 20 26 0 0 r[26]=r[20] - 42 Copy 24 27 0 0 r[27]=r[24] - 43 Copy 25 28 0 0 r[28]=r[25] - 44 Copy 26 29 0 0 r[29]=r[26] - 45 MakeRecord 27 3 30 0 r[30]=mkrec(r[27..29]) - 46 NewRowid 0 31 0 0 r[31]=rowid - 47 Insert 0 30 31 4 intkey=r[31] data=r[30] + 30 ColumnRange 1 1 16 2 0 r[16..17]=t2.name..counter + 31 Copy 15 18 2 0 r[18]=r[15] + 32 Copy 17 21 0 0 r[21]=r[17] + 33 Integer 1 22 0 0 r[22]=1 + 34 Add 21 22 20 0 r[20]=r[21]+r[22] + 35 Affinity 18 3 0 0 r[18..21] = D, B, D + 36 MakeRecord 18 3 23 0 r[23]=mkrec(r[18..20]) + 37 Insert 1 23 1 t2 8 intkey=r[1] data=r[23] + 38 Copy 1 24 0 0 r[24]=r[1] + 39 Copy 19 25 0 0 r[25]=r[19] + 40 Copy 20 26 0 0 r[26]=r[20] + 41 Copy 24 27 0 0 r[27]=r[24] + 42 Copy 25 28 0 0 r[28]=r[25] + 43 Copy 26 29 0 0 r[29]=r[26] + 44 MakeRecord 27 3 30 0 r[30]=mkrec(r[27..29]) + 45 NewRowid 0 31 0 0 r[31]=rowid + 46 Insert 0 30 31 4 intkey=r[31] data=r[30] + 47 Goto 0 48 0 0 48 Goto 0 49 0 0 - 49 Goto 0 50 0 0 - 50 FkCheck 0 0 0 0 - 51 Rewind 0 57 0 0 Rewind ephemeral(t2) - 52 Column 0 0 32 0 r[32]=ephemeral(t2).id - 53 Column 0 1 33 0 r[33]=ephemeral(t2).name - 54 Column 0 2 34 0 r[34]=ephemeral(t2).counter - 55 ResultRow 32 3 0 0 output=r[32..34] - 56 Next 0 52 0 0 - 57 Halt 0 0 0 0 - 58 Transaction 0 2 1 0 iDb=0 tx_mode=Write - 59 Goto 0 1 0 0 + 49 FkCheck 0 0 0 0 + 50 Rewind 0 54 0 0 Rewind ephemeral(t2) + 51 ColumnRange 0 0 32 3 0 r[32..34]=ephemeral(t2).id..counter + 52 ResultRow 32 3 0 0 output=r[32..34] + 53 Next 0 51 0 0 + 54 Halt 0 0 0 0 + 55 Transaction 0 2 1 0 iDb=0 tx_mode=Write + 56 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/schema/snapshots/schema__explain-drop-temp-trigger-opens-temp-transaction.snap b/conformance/sqlite-sqltests/snapshot_tests/schema/snapshots/schema__explain-drop-temp-trigger-opens-temp-transaction.snap index eb63c5ff..5e7b135f 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/schema/snapshots/schema__explain-drop-temp-trigger-opens-temp-transaction.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/schema/snapshots/schema__explain-drop-temp-trigger-opens-temp-transaction.snap @@ -11,22 +11,21 @@ QUERY PLAN BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 15 0 0 Start at 15 - 1 OpenWrite 0 1 1 0 root=1; iDb=1 - 2 Rewind 0 12 0 0 Rewind sqlite_schema - 3 Column 0 0 1 0 r[1]=sqlite_schema.type - 4 Column 0 1 2 0 r[2]=sqlite_schema.name - 5 String8 0 3 0 trigger 0 r[3]='trigger' - 6 Ne 1 3 11 0 if r[1]!=r[3] goto 11 - 7 String8 0 4 0 tr 0 r[4]='tr' - 8 Ne 2 4 11 0 if r[2]!=r[4] goto 11 - 9 Delete 0 0 0 sqlite_schema 0 - 10 Goto 0 12 0 0 - 11 Next 0 3 0 0 - 12 SetCookie 1 1 3 0 - 13 DropTrigger 1 0 0 tr 0 DROP TRIGGER tr - 14 Halt 0 0 0 0 - 15 Transaction 0 2 0 0 iDb=0 tx_mode=Write - 16 Transaction 1 2 2 0 iDb=1 tx_mode=Write - 17 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 14 0 0 Start at 14 + 1 OpenWrite 0 1 1 0 root=1; iDb=1 + 2 Rewind 0 11 0 0 Rewind sqlite_schema + 3 ColumnRange 0 0 1 2 0 r[1..2]=sqlite_schema.type..name + 4 String8 0 3 0 trigger 0 r[3]='trigger' + 5 Ne 1 3 10 0 if r[1]!=r[3] goto 10 + 6 String8 0 4 0 tr 0 r[4]='tr' + 7 Ne 2 4 10 0 if r[2]!=r[4] goto 10 + 8 Delete 0 0 0 sqlite_schema 0 + 9 Goto 0 11 0 0 + 10 Next 0 3 0 0 + 11 SetCookie 1 1 3 0 + 12 DropTrigger 1 0 0 tr 0 DROP TRIGGER tr + 13 Halt 0 0 0 0 + 14 Transaction 0 2 0 0 iDb=0 tx_mode=Write + 15 Transaction 1 2 2 0 iDb=1 tx_mode=Write + 16 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-mixed-union.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-mixed-union.snap index 936b57e9..7df6d008 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-mixed-union.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-mixed-union.snap @@ -27,34 +27,30 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 29 0 0 Start at 29 + 0 Init 0 25 0 0 Start at 25 1 OpenEphemeral 0 0 0 0 cursor=0 is_table=false 2 OpenRead 1 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 - 3 Rewind 1 9 0 0 Rewind table current_employees - 4 Column 1 1 1 0 r[1]=current_employees.name - 5 Column 1 2 2 0 r[2]=current_employees.department - 6 MakeRecord 1 2 3 0 r[3]=mkrec(r[1..2]); for compound_dedupe - 7 IdxInsert 0 3 0 8 key=r[3] - 8 Next 1 4 0 0 - 9 OpenRead 2 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 - 10 Rewind 2 16 0 0 Rewind table former_employees - 11 Column 2 1 4 0 r[4]=former_employees.name - 12 Column 2 2 5 0 r[5]=former_employees.department - 13 MakeRecord 4 2 6 0 r[6]=mkrec(r[4..5]); for compound_dedupe - 14 IdxInsert 0 6 0 8 key=r[6] - 15 Next 2 11 0 0 - 16 Rewind 0 20 0 0 Rewind ephemeral(compound_dedupe) - 17 Column 0 0 7 0 r[7]=ephemeral(compound_dedupe).name - 18 Column 0 1 8 0 r[8]=ephemeral(compound_dedupe).department - 19 ResultRow 7 2 0 0 output=r[7..8] - 20 Next 0 17 0 0 - 21 Close 0 0 0 0 - 22 OpenRead 3 4 0 k(4,B,B,B,B) 0 table=contractors, root=4, iDb=0 - 23 Rewind 3 28 0 0 Rewind table contractors - 24 Column 3 1 9 0 r[9]=contractors.name - 25 Column 3 2 10 0 r[10]=contractors.department - 26 ResultRow 9 2 0 0 output=r[9..10] - 27 Next 3 24 0 0 - 28 Halt 0 0 0 0 - 29 Transaction 0 1 6 0 iDb=0 tx_mode=Read - 30 Goto 0 1 0 0 + 3 Rewind 1 8 0 0 Rewind table current_employees + 4 ColumnRange 1 1 1 2 0 r[1..2]=current_employees.name..department + 5 MakeRecord 1 2 3 0 r[3]=mkrec(r[1..2]); for compound_dedupe + 6 IdxInsert 0 3 0 8 key=r[3] + 7 Next 1 4 0 1 + 8 OpenRead 2 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 + 9 Rewind 2 14 0 0 Rewind table former_employees + 10 ColumnRange 2 1 4 2 0 r[4..5]=former_employees.name..department + 11 MakeRecord 4 2 6 0 r[6]=mkrec(r[4..5]); for compound_dedupe + 12 IdxInsert 0 6 0 8 key=r[6] + 13 Next 2 10 0 1 + 14 Rewind 0 17 0 0 Rewind ephemeral(compound_dedupe) + 15 ColumnRange 0 0 7 2 0 r[7..8]=ephemeral(compound_dedupe).name..department + 16 ResultRow 7 2 0 0 output=r[7..8] + 17 Next 0 15 0 0 + 18 Close 0 0 0 0 + 19 OpenRead 3 4 0 k(4,B,B,B,B) 0 table=contractors, root=4, iDb=0 + 20 Rewind 3 24 0 0 Rewind table contractors + 21 ColumnRange 3 1 9 2 0 r[9..10]=contractors.name..department + 22 ResultRow 9 2 0 0 output=r[9..10] + 23 Next 3 21 0 1 + 24 Halt 0 0 0 0 + 25 Transaction 0 1 6 0 iDb=0 tx_mode=Read + 26 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-all-three-way.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-all-three-way.snap index da201862..2021cf18 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-all-three-way.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-all-three-way.snap @@ -26,26 +26,23 @@ QUERY PLAN `--SCAN contractors BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 20 0 0 Start at 20 - 1 OpenRead 0 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 - 2 Rewind 0 7 0 0 Rewind table current_employees - 3 Column 0 1 1 0 r[1]=current_employees.name - 4 Column 0 2 2 0 r[2]=current_employees.department - 5 ResultRow 1 2 0 0 output=r[1..2] - 6 Next 0 3 0 0 - 7 OpenRead 1 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 - 8 Rewind 1 13 0 0 Rewind table former_employees - 9 Column 1 1 3 0 r[3]=former_employees.name - 10 Column 1 2 4 0 r[4]=former_employees.department - 11 ResultRow 3 2 0 0 output=r[3..4] - 12 Next 1 9 0 0 - 13 OpenRead 2 4 0 k(4,B,B,B,B) 0 table=contractors, root=4, iDb=0 - 14 Rewind 2 19 0 0 Rewind table contractors - 15 Column 2 1 5 0 r[5]=contractors.name - 16 Column 2 2 6 0 r[6]=contractors.department - 17 ResultRow 5 2 0 0 output=r[5..6] - 18 Next 2 15 0 0 - 19 Halt 0 0 0 0 - 20 Transaction 0 1 6 0 iDb=0 tx_mode=Read - 21 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 17 0 0 Start at 17 + 1 OpenRead 0 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 + 2 Rewind 0 6 0 0 Rewind table current_employees + 3 ColumnRange 0 1 1 2 0 r[1..2]=current_employees.name..department + 4 ResultRow 1 2 0 0 output=r[1..2] + 5 Next 0 3 0 1 + 6 OpenRead 1 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 + 7 Rewind 1 11 0 0 Rewind table former_employees + 8 ColumnRange 1 1 3 2 0 r[3..4]=former_employees.name..department + 9 ResultRow 3 2 0 0 output=r[3..4] + 10 Next 1 8 0 1 + 11 OpenRead 2 4 0 k(4,B,B,B,B) 0 table=contractors, root=4, iDb=0 + 12 Rewind 2 16 0 0 Rewind table contractors + 13 ColumnRange 2 1 5 2 0 r[5..6]=contractors.name..department + 14 ResultRow 5 2 0 0 output=r[5..6] + 15 Next 2 13 0 1 + 16 Halt 0 0 0 0 + 17 Transaction 0 1 6 0 iDb=0 tx_mode=Read + 18 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-except.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-except.snap index 7a5dd249..3c4037af 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-except.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-except.snap @@ -34,18 +34,18 @@ addr opcode p1 p2 p3 p4 p5 comment 4 Column 1 0 1 0 r[1]=idx_current_dept.department 5 MakeRecord 1 1 2 0 r[2]=mkrec(r[1..1]); for compound_dedupe 6 IdxInsert 0 2 0 8 key=r[2] - 7 Next 1 4 0 0 + 7 Next 1 4 0 1 8 OpenRead 2 7 0 k(2,B) 0 index=idx_contractors_dept, root=7, iDb=0 9 Rewind 2 14 0 0 Rewind index idx_contractors_dept 10 Column 2 0 3 0 r[3]=idx_contractors_dept.department 11 MakeRecord 3 1 4 0 r[4]=mkrec(r[3..3]); for compound_dedupe 12 IdxInsert 0 4 0 8 key=r[4] - 13 Next 2 10 0 0 + 13 Next 2 10 0 1 14 OpenRead 3 6 0 k(2,B) 0 index=idx_former_dept, root=6, iDb=0 15 Rewind 3 19 0 0 Rewind index idx_former_dept 16 Column 3 0 5 0 r[5]=idx_former_dept.department 17 IdxDelete 0 5 1 0 - 18 Next 3 16 0 0 + 18 Next 3 16 0 1 19 Rewind 0 22 0 0 Rewind ephemeral(compound_dedupe) 20 Column 0 0 6 0 r[6]=ephemeral(compound_dedupe).department 21 ResultRow 6 1 0 0 output=r[6] diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-intersect.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-intersect.snap index 36dc9b27..c2c84d80 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-intersect.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__compound-union-intersect.snap @@ -35,19 +35,19 @@ addr opcode p1 p2 p3 p4 p5 comment 5 Column 2 0 1 0 r[1]=idx_current_dept.department 6 MakeRecord 1 1 2 0 r[2]=mkrec(r[1..1]); for compound_dedupe 7 IdxInsert 0 2 0 8 key=r[2] - 8 Next 2 5 0 0 + 8 Next 2 5 0 1 9 OpenRead 3 7 0 k(2,B) 0 index=idx_contractors_dept, root=7, iDb=0 10 Rewind 3 15 0 0 Rewind index idx_contractors_dept 11 Column 3 0 3 0 r[3]=idx_contractors_dept.department 12 MakeRecord 3 1 4 0 r[4]=mkrec(r[3..3]); for compound_dedupe 13 IdxInsert 0 4 0 8 key=r[4] - 14 Next 3 11 0 0 + 14 Next 3 11 0 1 15 OpenRead 4 6 0 k(2,B) 0 index=idx_former_dept, root=6, iDb=0 16 Rewind 4 21 0 0 Rewind index idx_former_dept 17 Column 4 0 5 0 r[5]=idx_former_dept.department 18 MakeRecord 5 1 6 0 r[6]=mkrec(r[5..5]); for compound_dedupe 19 IdxInsert 1 6 0 8 key=r[6] - 20 Next 4 17 0 0 + 20 Next 4 17 0 1 21 Rewind 0 27 0 0 Rewind ephemeral(compound_dedupe) 22 RowData 0 7 0 0 r[7] = data 23 NotFound 1 26 7 0 if not found goto 26 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__except-basic.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__except-basic.snap index 7e1a62d2..3db1a95b 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__except-basic.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__except-basic.snap @@ -29,12 +29,12 @@ addr opcode p1 p2 p3 p4 p5 comment 4 Column 1 0 1 0 r[1]=idx_current_dept.department 5 MakeRecord 1 1 2 0 r[2]=mkrec(r[1..1]); for compound_dedupe 6 IdxInsert 0 2 0 8 key=r[2] - 7 Next 1 4 0 0 + 7 Next 1 4 0 1 8 OpenRead 2 6 0 k(2,B) 0 index=idx_former_dept, root=6, iDb=0 9 Rewind 2 13 0 0 Rewind index idx_former_dept 10 Column 2 0 3 0 r[3]=idx_former_dept.department 11 IdxDelete 0 3 1 0 - 12 Next 2 10 0 0 + 12 Next 2 10 0 1 13 Rewind 0 16 0 0 Rewind ephemeral(compound_dedupe) 14 Column 0 0 4 0 r[4]=ephemeral(compound_dedupe).department 15 ResultRow 4 1 0 0 output=r[4] diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__except-multiple-columns.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__except-multiple-columns.snap index 5faa07a5..f497d28d 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__except-multiple-columns.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__except-multiple-columns.snap @@ -22,27 +22,24 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 22 0 0 Start at 22 + 0 Init 0 19 0 0 Start at 19 1 OpenEphemeral 0 0 0 0 cursor=0 is_table=false 2 OpenRead 1 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 - 3 Rewind 1 9 0 0 Rewind table current_employees - 4 Column 1 1 1 0 r[1]=current_employees.name - 5 Column 1 2 2 0 r[2]=current_employees.department - 6 MakeRecord 1 2 3 0 r[3]=mkrec(r[1..2]); for compound_dedupe - 7 IdxInsert 0 3 0 8 key=r[3] - 8 Next 1 4 0 0 - 9 OpenRead 2 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 - 10 Rewind 2 15 0 0 Rewind table former_employees - 11 Column 2 1 4 0 r[4]=former_employees.name - 12 Column 2 2 5 0 r[5]=former_employees.department - 13 IdxDelete 0 4 2 0 - 14 Next 2 11 0 0 - 15 Rewind 0 19 0 0 Rewind ephemeral(compound_dedupe) - 16 Column 0 0 6 0 r[6]=ephemeral(compound_dedupe).name - 17 Column 0 1 7 0 r[7]=ephemeral(compound_dedupe).department - 18 ResultRow 6 2 0 0 output=r[6..7] - 19 Next 0 16 0 0 - 20 Close 0 0 0 0 - 21 Halt 0 0 0 0 - 22 Transaction 0 1 6 0 iDb=0 tx_mode=Read - 23 Goto 0 1 0 0 + 3 Rewind 1 8 0 0 Rewind table current_employees + 4 ColumnRange 1 1 1 2 0 r[1..2]=current_employees.name..department + 5 MakeRecord 1 2 3 0 r[3]=mkrec(r[1..2]); for compound_dedupe + 6 IdxInsert 0 3 0 8 key=r[3] + 7 Next 1 4 0 1 + 8 OpenRead 2 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 + 9 Rewind 2 13 0 0 Rewind table former_employees + 10 ColumnRange 2 1 4 2 0 r[4..5]=former_employees.name..department + 11 IdxDelete 0 4 2 0 + 12 Next 2 10 0 1 + 13 Rewind 0 16 0 0 Rewind ephemeral(compound_dedupe) + 14 ColumnRange 0 0 6 2 0 r[6..7]=ephemeral(compound_dedupe).name..department + 15 ResultRow 6 2 0 0 output=r[6..7] + 16 Next 0 14 0 0 + 17 Close 0 0 0 0 + 18 Halt 0 0 0 0 + 19 Transaction 0 1 6 0 iDb=0 tx_mode=Read + 20 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__intersect-basic.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__intersect-basic.snap index 2c0026d2..2f460d73 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__intersect-basic.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__intersect-basic.snap @@ -30,13 +30,13 @@ addr opcode p1 p2 p3 p4 p5 comment 5 Column 2 0 1 0 r[1]=idx_current_dept.department 6 MakeRecord 1 1 2 0 r[2]=mkrec(r[1..1]); for compound_dedupe 7 IdxInsert 0 2 0 8 key=r[2] - 8 Next 2 5 0 0 + 8 Next 2 5 0 1 9 OpenRead 3 6 0 k(2,B) 0 index=idx_former_dept, root=6, iDb=0 10 Rewind 3 15 0 0 Rewind index idx_former_dept 11 Column 3 0 3 0 r[3]=idx_former_dept.department 12 MakeRecord 3 1 4 0 r[4]=mkrec(r[3..3]); for compound_dedupe 13 IdxInsert 1 4 0 8 key=r[4] - 14 Next 3 11 0 0 + 14 Next 3 11 0 1 15 Rewind 0 21 0 0 Rewind ephemeral(compound_dedupe) 16 RowData 0 5 0 0 r[5] = data 17 NotFound 1 20 5 0 if not found goto 20 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__intersect-multiple-columns.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__intersect-multiple-columns.snap index 515d4ec8..1a94c54f 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__intersect-multiple-columns.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__intersect-multiple-columns.snap @@ -22,32 +22,29 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 27 0 0 Start at 27 + 0 Init 0 24 0 0 Start at 24 1 OpenEphemeral 0 0 0 0 cursor=0 is_table=false 2 OpenEphemeral 1 0 0 0 cursor=1 is_table=false 3 OpenRead 2 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 - 4 Rewind 2 10 0 0 Rewind table current_employees - 5 Column 2 1 1 0 r[1]=current_employees.name - 6 Column 2 2 2 0 r[2]=current_employees.department - 7 MakeRecord 1 2 3 0 r[3]=mkrec(r[1..2]); for compound_dedupe - 8 IdxInsert 0 3 0 8 key=r[3] - 9 Next 2 5 0 0 - 10 OpenRead 3 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 - 11 Rewind 3 17 0 0 Rewind table former_employees - 12 Column 3 1 4 0 r[4]=former_employees.name - 13 Column 3 2 5 0 r[5]=former_employees.department - 14 MakeRecord 4 2 6 0 r[6]=mkrec(r[4..5]); for compound_dedupe - 15 IdxInsert 1 6 0 8 key=r[6] - 16 Next 3 12 0 0 - 17 Rewind 0 24 0 0 Rewind ephemeral(compound_dedupe) - 18 RowData 0 7 0 0 r[7] = data - 19 NotFound 1 23 7 0 if not found goto 23 - 20 Column 0 0 8 0 r[8]=ephemeral(compound_dedupe).name - 21 Column 0 1 9 0 r[9]=ephemeral(compound_dedupe).department - 22 ResultRow 8 2 0 0 output=r[8..9] - 23 Next 0 18 0 0 - 24 Close 1 0 0 0 - 25 Close 0 0 0 0 - 26 Halt 0 0 0 0 - 27 Transaction 0 1 6 0 iDb=0 tx_mode=Read - 28 Goto 0 1 0 0 + 4 Rewind 2 9 0 0 Rewind table current_employees + 5 ColumnRange 2 1 1 2 0 r[1..2]=current_employees.name..department + 6 MakeRecord 1 2 3 0 r[3]=mkrec(r[1..2]); for compound_dedupe + 7 IdxInsert 0 3 0 8 key=r[3] + 8 Next 2 5 0 1 + 9 OpenRead 3 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 + 10 Rewind 3 15 0 0 Rewind table former_employees + 11 ColumnRange 3 1 4 2 0 r[4..5]=former_employees.name..department + 12 MakeRecord 4 2 6 0 r[6]=mkrec(r[4..5]); for compound_dedupe + 13 IdxInsert 1 6 0 8 key=r[6] + 14 Next 3 11 0 1 + 15 Rewind 0 21 0 0 Rewind ephemeral(compound_dedupe) + 16 RowData 0 7 0 0 r[7] = data + 17 NotFound 1 20 7 0 if not found goto 20 + 18 ColumnRange 0 0 8 2 0 r[8..9]=ephemeral(compound_dedupe).name..department + 19 ResultRow 8 2 0 0 output=r[8..9] + 20 Next 0 16 0 0 + 21 Close 1 0 0 0 + 22 Close 0 0 0 0 + 23 Halt 0 0 0 0 + 24 Transaction 0 1 6 0 iDb=0 tx_mode=Read + 25 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__subquery-with-union.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__subquery-with-union.snap index f1f73d43..eb414218 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__subquery-with-union.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__subquery-with-union.snap @@ -26,39 +26,36 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 33 0 0 Start at 33 - 1 InitCoroutine 1 24 2 0 + 0 Init 0 30 0 0 Start at 30 + 1 InitCoroutine 1 21 2 0 2 OpenEphemeral 0 0 0 0 cursor=0 is_table=false 3 OpenRead 1 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 - 4 Rewind 1 10 0 0 Rewind table current_employees - 5 Column 1 1 2 0 r[2]=current_employees.name - 6 Column 1 2 3 0 r[3]=current_employees.department - 7 MakeRecord 2 2 4 0 r[4]=mkrec(r[2..3]); for compound_dedupe - 8 IdxInsert 0 4 0 8 key=r[4] - 9 Next 1 5 0 0 - 10 OpenRead 2 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 - 11 Rewind 2 17 0 0 Rewind table former_employees - 12 Column 2 1 2 0 r[2]=former_employees.name - 13 Column 2 2 3 0 r[3]=former_employees.department - 14 MakeRecord 2 2 5 0 r[5]=mkrec(r[2..3]); for compound_dedupe - 15 IdxInsert 0 5 0 8 key=r[5] - 16 Next 2 12 0 0 - 17 Rewind 0 21 0 0 Rewind ephemeral(compound_dedupe) - 18 Column 0 0 2 0 r[2]=ephemeral(compound_dedupe).name - 19 Column 0 1 3 0 r[3]=ephemeral(compound_dedupe).department - 20 Yield 1 0 0 0 - 21 Next 0 18 0 0 - 22 Close 0 0 0 0 - 23 EndCoroutine 1 0 0 0 - 24 InitCoroutine 1 0 2 0 - 25 Yield 1 32 0 0 - 26 Copy 3 9 0 0 r[9]=r[3] - 27 Ne 9 10 31 Binary 0 if r[9]!=r[10] goto 31 - 28 Copy 2 6 0 0 r[6]=r[2] - 29 Copy 3 7 0 0 r[7]=r[3] - 30 ResultRow 6 2 0 0 output=r[6..7] - 31 Goto 0 25 0 0 - 32 Halt 0 0 0 0 - 33 Transaction 0 1 6 0 iDb=0 tx_mode=Read - 34 String8 0 10 0 Engineering 0 r[10]='Engineering' - 35 Goto 0 1 0 0 + 4 Rewind 1 9 0 0 Rewind table current_employees + 5 ColumnRange 1 1 2 2 0 r[2..3]=current_employees.name..department + 6 MakeRecord 2 2 4 0 r[4]=mkrec(r[2..3]); for compound_dedupe + 7 IdxInsert 0 4 0 8 key=r[4] + 8 Next 1 5 0 1 + 9 OpenRead 2 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 + 10 Rewind 2 15 0 0 Rewind table former_employees + 11 ColumnRange 2 1 2 2 0 r[2..3]=former_employees.name..department + 12 MakeRecord 2 2 5 0 r[5]=mkrec(r[2..3]); for compound_dedupe + 13 IdxInsert 0 5 0 8 key=r[5] + 14 Next 2 11 0 1 + 15 Rewind 0 18 0 0 Rewind ephemeral(compound_dedupe) + 16 ColumnRange 0 0 2 2 0 r[2..3]=ephemeral(compound_dedupe).name..department + 17 Yield 1 0 0 0 + 18 Next 0 16 0 0 + 19 Close 0 0 0 0 + 20 EndCoroutine 1 0 0 0 + 21 InitCoroutine 1 0 2 0 + 22 Yield 1 29 0 0 + 23 Copy 3 9 0 0 r[9]=r[3] + 24 Ne 9 10 28 Binary 0 if r[9]!=r[10] goto 28 + 25 Copy 2 6 0 0 r[6]=r[2] + 26 Copy 3 7 0 0 r[7]=r[3] + 27 ResultRow 6 2 0 0 output=r[6..7] + 28 Goto 0 22 0 0 + 29 Halt 0 0 0 0 + 30 Transaction 0 1 6 0 iDb=0 tx_mode=Read + 31 String8 0 10 0 Engineering 0 r[10]='Engineering' + 32 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-all-basic.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-all-basic.snap index 46902ed7..936fc0ba 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-all-basic.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-all-basic.snap @@ -21,20 +21,18 @@ QUERY PLAN `--SCAN former_employees BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 14 0 0 Start at 14 - 1 OpenRead 0 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 - 2 Rewind 0 7 0 0 Rewind table current_employees - 3 Column 0 1 1 0 r[1]=current_employees.name - 4 Column 0 2 2 0 r[2]=current_employees.department - 5 ResultRow 1 2 0 0 output=r[1..2] - 6 Next 0 3 0 0 - 7 OpenRead 1 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 - 8 Rewind 1 13 0 0 Rewind table former_employees - 9 Column 1 1 3 0 r[3]=former_employees.name - 10 Column 1 2 4 0 r[4]=former_employees.department - 11 ResultRow 3 2 0 0 output=r[3..4] - 12 Next 1 9 0 0 - 13 Halt 0 0 0 0 - 14 Transaction 0 1 6 0 iDb=0 tx_mode=Read - 15 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 12 0 0 Start at 12 + 1 OpenRead 0 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 + 2 Rewind 0 6 0 0 Rewind table current_employees + 3 ColumnRange 0 1 1 2 0 r[1..2]=current_employees.name..department + 4 ResultRow 1 2 0 0 output=r[1..2] + 5 Next 0 3 0 1 + 6 OpenRead 1 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 + 7 Rewind 1 11 0 0 Rewind table former_employees + 8 ColumnRange 1 1 3 2 0 r[3..4]=former_employees.name..department + 9 ResultRow 3 2 0 0 output=r[3..4] + 10 Next 1 8 0 1 + 11 Halt 0 0 0 0 + 12 Transaction 0 1 6 0 iDb=0 tx_mode=Read + 13 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-distinct.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-distinct.snap index bff1be1b..28fa3e11 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-distinct.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-distinct.snap @@ -22,28 +22,25 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 23 0 0 Start at 23 + 0 Init 0 20 0 0 Start at 20 1 OpenEphemeral 0 0 0 0 cursor=0 is_table=false 2 OpenRead 1 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 - 3 Rewind 1 9 0 0 Rewind table current_employees - 4 Column 1 1 1 0 r[1]=current_employees.name - 5 Column 1 2 2 0 r[2]=current_employees.department - 6 MakeRecord 1 2 3 0 r[3]=mkrec(r[1..2]); for compound_dedupe - 7 IdxInsert 0 3 0 8 key=r[3] - 8 Next 1 4 0 0 - 9 OpenRead 2 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 - 10 Rewind 2 16 0 0 Rewind table former_employees - 11 Column 2 1 4 0 r[4]=former_employees.name - 12 Column 2 2 5 0 r[5]=former_employees.department - 13 MakeRecord 4 2 6 0 r[6]=mkrec(r[4..5]); for compound_dedupe - 14 IdxInsert 0 6 0 8 key=r[6] - 15 Next 2 11 0 0 - 16 Rewind 0 20 0 0 Rewind ephemeral(compound_dedupe) - 17 Column 0 0 7 0 r[7]=ephemeral(compound_dedupe).name - 18 Column 0 1 8 0 r[8]=ephemeral(compound_dedupe).department - 19 ResultRow 7 2 0 0 output=r[7..8] - 20 Next 0 17 0 0 - 21 Close 0 0 0 0 - 22 Halt 0 0 0 0 - 23 Transaction 0 1 6 0 iDb=0 tx_mode=Read - 24 Goto 0 1 0 0 + 3 Rewind 1 8 0 0 Rewind table current_employees + 4 ColumnRange 1 1 1 2 0 r[1..2]=current_employees.name..department + 5 MakeRecord 1 2 3 0 r[3]=mkrec(r[1..2]); for compound_dedupe + 6 IdxInsert 0 3 0 8 key=r[3] + 7 Next 1 4 0 1 + 8 OpenRead 2 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 + 9 Rewind 2 14 0 0 Rewind table former_employees + 10 ColumnRange 2 1 4 2 0 r[4..5]=former_employees.name..department + 11 MakeRecord 4 2 6 0 r[6]=mkrec(r[4..5]); for compound_dedupe + 12 IdxInsert 0 6 0 8 key=r[6] + 13 Next 2 10 0 1 + 14 Rewind 0 17 0 0 Rewind ephemeral(compound_dedupe) + 15 ColumnRange 0 0 7 2 0 r[7..8]=ephemeral(compound_dedupe).name..department + 16 ResultRow 7 2 0 0 output=r[7..8] + 17 Next 0 15 0 0 + 18 Close 0 0 0 0 + 19 Halt 0 0 0 0 + 20 Transaction 0 1 6 0 iDb=0 tx_mode=Read + 21 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-explicit-distinct.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-explicit-distinct.snap index 126f43be..df604ad5 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-explicit-distinct.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-explicit-distinct.snap @@ -33,7 +33,7 @@ addr opcode p1 p2 p3 p4 p5 comment 6 HashDistinct 1073741824 1 1 jmp=9 0 7 MakeRecord 1 1 2 0 r[2]=mkrec(r[1..1]); for compound_dedupe 8 IdxInsert 0 2 0 8 key=r[2] - 9 Next 1 5 0 0 + 9 Next 1 5 0 1 10 HashClear 1073741825 0 0 0 11 OpenRead 2 6 0 k(2,B) 0 index=idx_former_dept, root=6, iDb=0 12 Rewind 2 18 0 0 Rewind index idx_former_dept @@ -41,7 +41,7 @@ addr opcode p1 p2 p3 p4 p5 comment 14 HashDistinct 1073741825 3 1 jmp=17 0 15 MakeRecord 3 1 4 0 r[4]=mkrec(r[3..3]); for compound_dedupe 16 IdxInsert 0 4 0 8 key=r[4] - 17 Next 2 13 0 0 + 17 Next 2 13 0 1 18 Rewind 0 21 0 0 Rewind ephemeral(compound_dedupe) 19 Column 0 0 5 0 r[5]=ephemeral(compound_dedupe).department 20 ResultRow 5 1 0 0 output=r[5] diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-order-by-literals.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-order-by-literals.snap index bcecc935..d815f6f4 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-order-by-literals.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-order-by-literals.snap @@ -15,7 +15,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 34 0 0 Start at 34 + 0 Init 0 33 0 0 Start at 33 1 OpenEphemeral 0 0 0 0 cursor=0 is_table=false 2 OpenEphemeral 1 0 0 0 cursor=1 is_table=false 3 Integer 1 1 0 0 r[1]=1 @@ -33,20 +33,19 @@ addr opcode p1 p2 p3 p4 p5 comment 15 Next 1 10 0 0 16 Close 1 0 0 0 17 SorterOpen 2 2 0 k(2,B,B) 0 cursor=2 - 18 Rewind 0 26 0 0 Rewind ephemeral(compound_collection) - 19 Column 0 0 7 0 r[7]=ephemeral(compound_collection).2 - 20 Column 0 1 8 0 r[8]=ephemeral(compound_collection).column 1 - 21 Copy 7 9 0 0 r[9]=r[7] - 22 Copy 8 10 0 0 r[10]=r[8] - 23 MakeRecord 9 2 11 0 r[11]=mkrec(r[9..10]) - 24 SorterInsert 2 11 0 0 0 key=r[11] - 25 Next 0 19 0 0 - 26 Close 0 0 0 0 - 27 OpenPseudo 3 11 2 0 2 columns in r[11] - 28 SorterSort 2 33 0 0 - 29 SorterData 2 11 3 0 r[11]=data - 30 Column 3 0 12 0 r[12]=pseudo.column 0 - 31 ResultRow 12 1 0 0 output=r[12] - 32 SorterNext 2 29 0 0 - 33 Halt 0 0 0 0 - 34 Goto 0 1 0 0 + 18 Rewind 0 25 0 0 Rewind ephemeral(compound_collection) + 19 ColumnRange 0 0 7 2 0 r[7..8]=ephemeral(compound_collection).2..column 1 + 20 Copy 7 9 0 0 r[9]=r[7] + 21 Copy 8 10 0 0 r[10]=r[8] + 22 MakeRecord 9 2 11 0 r[11]=mkrec(r[9..10]) + 23 SorterInsert 2 11 0 0 0 key=r[11] + 24 Next 0 19 0 0 + 25 Close 0 0 0 0 + 26 OpenPseudo 3 11 2 0 2 columns in r[11] + 27 SorterSort 2 32 0 0 + 28 SorterData 2 11 3 0 r[11]=data + 29 Column 3 0 12 0 r[12]=pseudo.column 0 + 30 ResultRow 12 1 0 0 output=r[12] + 31 SorterNext 2 28 0 0 + 32 Halt 0 0 0 0 + 33 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-subquery-aggregation.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-subquery-aggregation.snap index e19d4d6b..7083fac3 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-subquery-aggregation.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-subquery-aggregation.snap @@ -35,77 +35,74 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 71 0 0 Start at 71 - 1 InitCoroutine 1 21 2 0 + 0 Init 0 68 0 0 Start at 68 + 1 InitCoroutine 1 18 2 0 2 OpenRead 0 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 - 3 Rewind 0 8 0 0 Rewind table current_employees - 4 Column 0 1 2 0 r[2]=current_employees.name - 5 Column 0 2 3 0 r[3]=current_employees.department - 6 Yield 1 0 0 0 - 7 Next 0 4 0 0 - 8 OpenRead 1 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 - 9 Rewind 1 14 0 0 Rewind table former_employees - 10 Column 1 1 2 0 r[2]=former_employees.name - 11 Column 1 2 3 0 r[3]=former_employees.department - 12 Yield 1 0 0 0 - 13 Next 1 10 0 0 - 14 OpenRead 2 4 0 k(4,B,B,B,B) 0 table=contractors, root=4, iDb=0 - 15 Rewind 2 20 0 0 Rewind table contractors - 16 Column 2 1 2 0 r[2]=contractors.name - 17 Column 2 2 3 0 r[3]=contractors.department - 18 Yield 1 0 0 0 - 19 Next 2 16 0 0 - 20 EndCoroutine 1 0 0 0 - 21 SorterOpen 3 1 0 k(1,-B) 0 cursor=3 - 22 Null 0 12 0 0 r[12]=NULL - 23 SorterOpen 4 1 0 k(1,-B) 0 cursor=4 - 24 Integer 0 9 0 0 r[9]=0; clear group by abort flag - 25 Null 0 10 0 0 r[10]=NULL; initialize group by comparison registers to NULL - 26 Gosub 15 60 0 0 ; go to clear accumulator subroutine - 27 InitCoroutine 1 0 2 0 - 28 Yield 1 33 0 0 - 29 Copy 3 14 0 0 r[14]=r[3] - 30 MakeRecord 14 1 13 0 r[13]=mkrec(r[14..14]) - 31 SorterInsert 4 13 0 0 0 key=r[13] - 32 Goto 0 28 0 0 - 33 OpenPseudo 5 13 1 0 1 columns in r[13] - 34 SorterSort 4 48 0 0 - 35 SorterData 4 13 5 0 r[13]=data - 36 Column 5 0 16 0 r[16]=pseudo.column 0 - 37 Compare 10 16 1 k(1, Binary) 0 r[10..10]==r[16..16] - 38 Jump 39 43 39 0 ; start new group if comparison is not equal - 39 Gosub 7 52 0 0 ; check if ended group had data, and output if so - 40 Move 16 10 1 0 r[10..10]=r[16..16] - 41 IfPos 9 63 0 0 r[9]>0 -> r[9]-=0, goto 63; check abort flag - 42 Gosub 15 60 0 0 ; goto clear accumulator subroutine - 43 AggStep 0 17 12 count 0 accum=r[12] step(r[17]) - 44 If 8 46 0 0 if r[8] goto 46; don't emit group columns if continuing existing group - 45 Column 5 0 11 0 r[11]=pseudo.column 0 - 46 Integer 1 8 0 0 r[8]=1; indicate data in accumulator - 47 SorterNext 4 35 0 0 - 48 Gosub 7 52 0 0 ; emit row for final group - 49 Goto 0 63 0 0 ; group by finished - 50 Integer 1 9 0 0 r[9]=1 - 51 Return 7 0 0 0 - 52 IfPos 8 54 0 0 r[8]>0 -> r[8]-=0, goto 54; output group by row subroutine start - 53 Return 7 0 0 0 - 54 AggFinal 0 12 0 count 0 accum=r[12] - 55 Copy 12 18 0 0 r[18]=r[12] - 56 Copy 11 19 0 0 r[19]=r[11] - 57 MakeRecord 18 2 6 0 r[6]=mkrec(r[18..19]) - 58 SorterInsert 3 6 0 0 0 key=r[6] - 59 Return 7 0 0 0 - 60 Null 0 11 12 0 r[11..12]=NULL; clear accumulator subroutine start - 61 Integer 0 8 0 0 r[8]=0 - 62 Return 15 0 0 0 - 63 OpenPseudo 6 6 2 0 2 columns in r[6] - 64 SorterSort 3 70 0 0 - 65 SorterData 3 6 6 0 r[6]=data - 66 Column 6 1 4 0 r[4]=pseudo.column 1 - 67 Column 6 0 5 0 r[5]=pseudo.column 0 - 68 ResultRow 4 2 0 0 output=r[4..5] - 69 SorterNext 3 65 0 0 - 70 Halt 0 0 0 0 - 71 Transaction 0 1 6 0 iDb=0 tx_mode=Read - 72 Integer 1 17 0 0 r[17]=1 - 73 Goto 0 1 0 0 + 3 Rewind 0 7 0 0 Rewind table current_employees + 4 ColumnRange 0 1 2 2 0 r[2..3]=current_employees.name..department + 5 Yield 1 0 0 0 + 6 Next 0 4 0 1 + 7 OpenRead 1 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 + 8 Rewind 1 12 0 0 Rewind table former_employees + 9 ColumnRange 1 1 2 2 0 r[2..3]=former_employees.name..department + 10 Yield 1 0 0 0 + 11 Next 1 9 0 1 + 12 OpenRead 2 4 0 k(4,B,B,B,B) 0 table=contractors, root=4, iDb=0 + 13 Rewind 2 17 0 0 Rewind table contractors + 14 ColumnRange 2 1 2 2 0 r[2..3]=contractors.name..department + 15 Yield 1 0 0 0 + 16 Next 2 14 0 1 + 17 EndCoroutine 1 0 0 0 + 18 SorterOpen 3 1 0 k(1,-B) 0 cursor=3 + 19 Null 0 12 0 0 r[12]=NULL + 20 SorterOpen 4 1 0 k(1,-B) 0 cursor=4 + 21 Integer 0 9 0 0 r[9]=0; clear group by abort flag + 22 Null 0 10 0 0 r[10]=NULL; initialize group by comparison registers to NULL + 23 Gosub 15 57 0 0 ; go to clear accumulator subroutine + 24 InitCoroutine 1 0 2 0 + 25 Yield 1 30 0 0 + 26 Copy 3 14 0 0 r[14]=r[3] + 27 MakeRecord 14 1 13 0 r[13]=mkrec(r[14..14]) + 28 SorterInsert 4 13 0 0 0 key=r[13] + 29 Goto 0 25 0 0 + 30 OpenPseudo 5 13 1 0 1 columns in r[13] + 31 SorterSort 4 45 0 0 + 32 SorterData 4 13 5 0 r[13]=data + 33 Column 5 0 16 0 r[16]=pseudo.column 0 + 34 Compare 10 16 1 k(1, Binary) 0 r[10..10]==r[16..16] + 35 Jump 36 40 36 0 ; start new group if comparison is not equal + 36 Gosub 7 49 0 0 ; check if ended group had data, and output if so + 37 Move 16 10 1 0 r[10..10]=r[16..16] + 38 IfPos 9 60 0 0 r[9]>0 -> r[9]-=0, goto 60; check abort flag + 39 Gosub 15 57 0 0 ; goto clear accumulator subroutine + 40 AggStep 0 17 12 count 0 accum=r[12] step(r[17]) + 41 If 8 43 0 0 if r[8] goto 43; don't emit group columns if continuing existing group + 42 Column 5 0 11 0 r[11]=pseudo.column 0 + 43 Integer 1 8 0 0 r[8]=1; indicate data in accumulator + 44 SorterNext 4 32 0 0 + 45 Gosub 7 49 0 0 ; emit row for final group + 46 Goto 0 60 0 0 ; group by finished + 47 Integer 1 9 0 0 r[9]=1 + 48 Return 7 0 0 0 + 49 IfPos 8 51 0 0 r[8]>0 -> r[8]-=0, goto 51; output group by row subroutine start + 50 Return 7 0 0 0 + 51 AggFinal 0 12 0 count 0 accum=r[12] + 52 Copy 12 18 0 0 r[18]=r[12] + 53 Copy 11 19 0 0 r[19]=r[11] + 54 MakeRecord 18 2 6 0 r[6]=mkrec(r[18..19]) + 55 SorterInsert 3 6 0 0 0 key=r[6] + 56 Return 7 0 0 0 + 57 Null 0 11 12 0 r[11..12]=NULL; clear accumulator subroutine start + 58 Integer 0 8 0 0 r[8]=0 + 59 Return 15 0 0 0 + 60 OpenPseudo 6 6 2 0 2 columns in r[6] + 61 SorterSort 3 67 0 0 + 62 SorterData 3 6 6 0 r[6]=data + 63 Column 6 1 4 0 r[4]=pseudo.column 1 + 64 Column 6 0 5 0 r[5]=pseudo.column 0 + 65 ResultRow 4 2 0 0 output=r[4..5] + 66 SorterNext 3 62 0 0 + 67 Halt 0 0 0 0 + 68 Transaction 0 1 6 0 iDb=0 tx_mode=Read + 69 Integer 1 17 0 0 r[17]=1 + 70 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-with-limit.snap b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-with-limit.snap index 3eafb9ba..f1b9b0a1 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-with-limit.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/setops/snapshots/setops__union-with-limit.snap @@ -23,31 +23,28 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 26 0 0 Start at 26 + 0 Init 0 23 0 0 Start at 23 1 Integer 10 1 0 0 r[1]=10; LIMIT counter - 2 IfNot 1 25 0 0 if !r[1] goto 25 + 2 IfNot 1 22 0 0 if !r[1] goto 22 3 OpenEphemeral 0 0 0 0 cursor=0 is_table=false 4 OpenRead 1 2 0 k(4,B,B,B,B) 0 table=current_employees, root=2, iDb=0 - 5 Rewind 1 11 0 0 Rewind table current_employees - 6 Column 1 1 2 0 r[2]=current_employees.name - 7 Column 1 2 3 0 r[3]=current_employees.department - 8 MakeRecord 2 2 4 0 r[4]=mkrec(r[2..3]); for compound_dedupe - 9 IdxInsert 0 4 0 8 key=r[4] - 10 Next 1 6 0 0 - 11 OpenRead 2 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 - 12 Rewind 2 18 0 0 Rewind table former_employees - 13 Column 2 1 5 0 r[5]=former_employees.name - 14 Column 2 2 6 0 r[6]=former_employees.department - 15 MakeRecord 5 2 7 0 r[7]=mkrec(r[5..6]); for compound_dedupe - 16 IdxInsert 0 7 0 8 key=r[7] - 17 Next 2 13 0 0 - 18 Rewind 0 23 0 0 Rewind ephemeral(compound_dedupe) - 19 Column 0 0 8 0 r[8]=ephemeral(compound_dedupe).name - 20 Column 0 1 9 0 r[9]=ephemeral(compound_dedupe).department - 21 ResultRow 8 2 0 0 output=r[8..9] - 22 DecrJumpZero 1 24 0 0 if (--r[1]==0) goto 24 - 23 Next 0 19 0 0 - 24 Close 0 0 0 0 - 25 Halt 0 0 0 0 - 26 Transaction 0 1 6 0 iDb=0 tx_mode=Read - 27 Goto 0 1 0 0 + 5 Rewind 1 10 0 0 Rewind table current_employees + 6 ColumnRange 1 1 2 2 0 r[2..3]=current_employees.name..department + 7 MakeRecord 2 2 4 0 r[4]=mkrec(r[2..3]); for compound_dedupe + 8 IdxInsert 0 4 0 8 key=r[4] + 9 Next 1 6 0 1 + 10 OpenRead 2 3 0 k(5,B,B,B,B,B) 0 table=former_employees, root=3, iDb=0 + 11 Rewind 2 16 0 0 Rewind table former_employees + 12 ColumnRange 2 1 5 2 0 r[5..6]=former_employees.name..department + 13 MakeRecord 5 2 7 0 r[7]=mkrec(r[5..6]); for compound_dedupe + 14 IdxInsert 0 7 0 8 key=r[7] + 15 Next 2 12 0 1 + 16 Rewind 0 20 0 0 Rewind ephemeral(compound_dedupe) + 17 ColumnRange 0 0 8 2 0 r[8..9]=ephemeral(compound_dedupe).name..department + 18 ResultRow 8 2 0 0 output=r[8..9] + 19 DecrJumpZero 1 21 0 0 if (--r[1]==0) goto 21 + 20 Next 0 17 0 0 + 21 Close 0 0 0 0 + 22 Halt 0 0 0 0 + 23 Transaction 0 1 6 0 iDb=0 tx_mode=Read + 24 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__large-offset-small-limit.snap b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__large-offset-small-limit.snap index ff0435cd..10d92f8d 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__large-offset-small-limit.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__large-offset-small-limit.snap @@ -35,7 +35,7 @@ addr opcode p1 p2 p3 p4 p5 comment 14 RealAffinity 3 0 0 0 15 ResultRow 1 3 0 0 output=r[1..3] 16 DecrJumpZero 4 18 0 0 if (--r[4]==0) goto 18 - 17 Next 1 9 0 0 + 17 Next 1 9 0 1 18 Halt 0 0 0 0 19 Transaction 0 1 7 0 iDb=0 tx_mode=Read 20 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__limit-with-offset.snap b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__limit-with-offset.snap index c0d95877..ed4b67dd 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__limit-with-offset.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__limit-with-offset.snap @@ -18,22 +18,21 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 17 0 0 Start at 17 + 0 Init 0 16 0 0 Start at 16 1 Integer 10 4 0 0 r[4]=10; LIMIT counter 2 Integer 20 5 0 0 r[5]=20 3 MustBeInt 5 0 0 0 ; OFFSET counter 4 OffsetLimit 4 6 5 0 if r[4]>0 then r[6]=r[4]+max(0,r[5]) else r[6]=(-1); OFFSET + LIMIT - 5 IfNot 4 16 0 0 if !r[4] goto 16 + 5 IfNot 4 15 0 0 if !r[4] goto 15 6 OpenRead 0 2 0 k(6,B,B,B,B,B,B) 0 table=products, root=2, iDb=0 - 7 Rewind 0 16 0 0 Rewind table products - 8 IfPos 5 15 1 0 r[5]>0 -> r[5]-=1, goto 15 + 7 Rewind 0 15 0 0 Rewind table products + 8 IfPos 5 14 1 0 r[5]>0 -> r[5]-=1, goto 14 9 RowId 0 1 0 0 r[1]=products.rowid - 10 Column 0 1 2 0 r[2]=products.name - 11 Column 0 2 3 0 r[3]=products.price - 12 RealAffinity 3 0 0 0 - 13 ResultRow 1 3 0 0 output=r[1..3] - 14 DecrJumpZero 4 16 0 0 if (--r[4]==0) goto 16 - 15 Next 0 8 0 0 - 16 Halt 0 0 0 0 - 17 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 18 Goto 0 1 0 0 + 10 ColumnRange 0 1 2 2 0 r[2..3]=products.name..price + 11 RealAffinity 3 0 0 0 + 12 ResultRow 1 3 0 0 output=r[1..3] + 13 DecrJumpZero 4 15 0 0 if (--r[4]==0) goto 15 + 14 Next 0 8 0 1 + 15 Halt 0 0 0 0 + 16 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 17 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__limit-without-order.snap b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__limit-without-order.snap index 3f50ef20..1c24ff03 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__limit-without-order.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__limit-without-order.snap @@ -17,18 +17,17 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 13 0 0 Start at 13 + 0 Init 0 12 0 0 Start at 12 1 Integer 5 4 0 0 r[4]=5; LIMIT counter - 2 IfNot 4 12 0 0 if !r[4] goto 12 + 2 IfNot 4 11 0 0 if !r[4] goto 11 3 OpenRead 0 2 0 k(6,B,B,B,B,B,B) 0 table=products, root=2, iDb=0 - 4 Rewind 0 12 0 0 Rewind table products + 4 Rewind 0 11 0 0 Rewind table products 5 RowId 0 1 0 0 r[1]=products.rowid - 6 Column 0 1 2 0 r[2]=products.name - 7 Column 0 2 3 0 r[3]=products.price - 8 RealAffinity 3 0 0 0 - 9 ResultRow 1 3 0 0 output=r[1..3] - 10 DecrJumpZero 4 12 0 0 if (--r[4]==0) goto 12 - 11 Next 0 5 0 0 - 12 Halt 0 0 0 0 - 13 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 14 Goto 0 1 0 0 + 6 ColumnRange 0 1 2 2 0 r[2..3]=products.name..price + 7 RealAffinity 3 0 0 0 + 8 ResultRow 1 3 0 0 output=r[1..3] + 9 DecrJumpZero 4 11 0 0 if (--r[4]==0) goto 11 + 10 Next 0 5 0 1 + 11 Halt 0 0 0 0 + 12 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 13 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-expression.snap b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-expression.snap index bcde46f3..4c77840f 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-expression.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-expression.snap @@ -18,32 +18,28 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 27 0 0 Start at 27 + 0 Init 0 23 0 0 Start at 23 1 SorterOpen 0 1 0 k(1,-B) 0 cursor=0 2 OpenRead 1 2 0 k(6,B,B,B,B,B,B) 0 table=products, root=2, iDb=0 - 3 Rewind 1 16 0 0 Rewind table products + 3 Rewind 1 15 0 0 Rewind table products 4 Column 1 2 12 0 r[12]=products.price 5 RealAffinity 12 0 0 0 6 Column 1 4 13 0 r[13]=products.stock 7 Multiply 12 13 7 0 r[7]=r[12]*r[13] 8 RowId 1 8 0 0 r[8]=products.rowid - 9 Column 1 1 9 0 r[9]=products.name - 10 Column 1 2 10 0 r[10]=products.price - 11 RealAffinity 10 0 0 0 - 12 Column 1 4 11 0 r[11]=products.stock - 13 MakeRecord 7 5 6 0 r[6]=mkrec(r[7..11]) - 14 SorterInsert 0 6 0 0 0 key=r[6] - 15 Next 1 4 0 0 - 16 OpenPseudo 2 6 5 0 5 columns in r[6] - 17 SorterSort 0 26 0 0 - 18 SorterData 0 6 2 0 r[6]=data - 19 Column 2 1 1 0 r[1]=pseudo.column 1 - 20 Column 2 2 2 0 r[2]=pseudo.column 2 - 21 Column 2 3 3 0 r[3]=pseudo.column 3 - 22 Column 2 4 4 0 r[4]=pseudo.column 4 - 23 Column 2 0 5 0 r[5]=pseudo.column 0 - 24 ResultRow 1 5 0 0 output=r[1..5] - 25 SorterNext 0 18 0 0 - 26 Halt 0 0 0 0 - 27 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 28 Goto 0 1 0 0 + 9 ColumnRange 1 1 9 2 0 r[9..10]=products.name..price + 10 RealAffinity 10 0 0 0 + 11 Column 1 4 11 0 r[11]=products.stock + 12 MakeRecord 7 5 6 0 r[6]=mkrec(r[7..11]) + 13 SorterInsert 0 6 0 0 0 key=r[6] + 14 Next 1 4 0 1 + 15 OpenPseudo 2 6 5 0 5 columns in r[6] + 16 SorterSort 0 22 0 0 + 17 SorterData 0 6 2 0 r[6]=data + 18 ColumnRange 2 1 1 4 0 r[1..4]=pseudo.column 1..column 4 + 19 Column 2 0 5 0 r[5]=pseudo.column 0 + 20 ResultRow 1 5 0 0 output=r[1..5] + 21 SorterNext 0 17 0 0 + 22 Halt 0 0 0 0 + 23 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 24 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-limit-one.snap b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-limit-one.snap index 12f7c177..3c4e0f44 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-limit-one.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-limit-one.snap @@ -31,7 +31,7 @@ addr opcode p1 p2 p3 p4 p5 comment 10 RealAffinity 3 0 0 0 11 ResultRow 1 3 0 0 output=r[1..3] 12 DecrJumpZero 4 14 0 0 if (--r[4]==0) goto 14 - 13 Next 1 6 0 0 + 13 Next 1 6 0 1 14 Halt 0 0 0 0 15 Transaction 0 1 7 0 iDb=0 tx_mode=Read 16 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-mixed-direction.snap b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-mixed-direction.snap index 694ea0af..f082fd24 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-mixed-direction.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-mixed-direction.snap @@ -18,7 +18,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 22 0 0 Start at 22 + 0 Init 0 20 0 0 Start at 20 1 SorterOpen 0 2 0 k(2,B,-B) 0 cursor=0 2 OpenRead 1 2 0 k(6,B,B,B,B,B,B) 0 table=products, root=2, iDb=0 3 Rewind 1 12 0 0 Rewind table products @@ -29,16 +29,14 @@ addr opcode p1 p2 p3 p4 p5 comment 8 Column 1 1 9 0 r[9]=products.name 9 MakeRecord 6 4 5 0 r[5]=mkrec(r[6..9]) 10 SorterInsert 0 5 0 0 0 key=r[5] - 11 Next 1 4 0 0 + 11 Next 1 4 0 1 12 OpenPseudo 2 5 4 0 4 columns in r[5] - 13 SorterSort 0 21 0 0 + 13 SorterSort 0 19 0 0 14 SorterData 0 5 2 0 r[5]=data - 15 Column 2 2 1 0 r[1]=pseudo.column 2 - 16 Column 2 3 2 0 r[2]=pseudo.column 3 - 17 Column 2 0 3 0 r[3]=pseudo.column 0 - 18 Column 2 1 4 0 r[4]=pseudo.column 1 - 19 ResultRow 1 4 0 0 output=r[1..4] - 20 SorterNext 0 14 0 0 - 21 Halt 0 0 0 0 - 22 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 23 Goto 0 1 0 0 + 15 ColumnRange 2 2 1 2 0 r[1..2]=pseudo.column 2..column 3 + 16 ColumnRange 2 0 3 2 0 r[3..4]=pseudo.column 0..column 1 + 17 ResultRow 1 4 0 0 output=r[1..4] + 18 SorterNext 0 14 0 0 + 19 Halt 0 0 0 0 + 20 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 21 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-single-column.snap b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-single-column.snap index 13ebafbd..cd79c092 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-single-column.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__order-by-single-column.snap @@ -28,7 +28,7 @@ addr opcode p1 p2 p3 p4 p5 comment 7 RealAffinity 7 0 0 0 8 MakeRecord 5 3 4 0 r[4]=mkrec(r[5..7]) 9 SorterInsert 0 4 0 0 0 key=r[4] - 10 Next 1 4 0 0 + 10 Next 1 4 0 1 11 OpenPseudo 2 4 3 0 3 columns in r[4] 12 SorterSort 0 19 0 0 13 SorterData 0 4 2 0 r[4]=data diff --git a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__sort-elim-left-join-outer-rowid-suffix.snap b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__sort-elim-left-join-outer-rowid-suffix.snap index 7f9b0d87..520ea2ca 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__sort-elim-left-join-outer-rowid-suffix.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/sorting/snapshots/sorting__sort-elim-left-join-outer-rowid-suffix.snap @@ -18,5 +18,5 @@ info: database: ':memory:' --- QUERY PLAN -|--SEARCH o USING INDEX idx_lhs_customer (customer_id=?) +|--SEARCH o USING COVERING INDEX idx_lhs_customer (customer_id=?) `--SEARCH i USING INDEX idx_rhs_product_qty (product=?) LEFT-JOIN diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-count.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-count.snap index e9f16b43..c3d78f27 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-count.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-count.snap @@ -19,39 +19,72 @@ info: --- QUERY PLAN |--SCAN orders AS o -`--CORRELATED SCALAR SUBQUERY 1 - `--SEARCH oi USING INDEX idx_order_items_order (order_id=?) +`--SEARCH scalar_subquery_t2 USING INDEX ephemeral_subquery_t2 (correlation_key_0=?) LEFT-JOIN + `--SCAN order_items AS oi USING COVERING INDEX idx_order_items_order BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 27 0 0 Start at 27 - 1 OpenRead 0 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 - 2 Rewind 0 26 0 0 Rewind table orders - 3 BeginSubrtn 5 0 0 0 r[5]=NULL - 4 Null 0 1 0 0 r[1]=NULL - 5 Null 0 7 0 0 r[7]=NULL - 6 Integer 1 8 0 0 r[8]=1; LIMIT counter - 7 IfNot 8 14 0 0 if !r[8] goto 14 - 8 OpenRead 1 10 0 k(2,B) 0 index=idx_order_items_order, root=10, iDb=0 - 9 RowId 0 9 0 0 r[9]=orders.rowid - 10 SeekGE 1 14 9 0 key=[9..9] - 11 IdxGT 1 14 9 0 key=[9..9] - 12 AggStep 0 10 7 count 0 accum=r[7] step(r[10]) - 13 Next 1 11 0 0 - 14 AggFinal 0 7 0 count 0 accum=r[7] - 15 Copy 7 6 0 0 r[6]=r[7] - 16 Copy 6 1 0 0 r[1]=r[6] - 17 Return 5 0 1 0 - 18 Copy 1 12 0 0 r[12]=r[1] - 19 Lt 12 13 25 0 if r[12]0 -> r[5]-=0, goto 39; check abort flag + 15 Gosub 11 36 0 0 ; goto clear accumulator subroutine + 16 AggStep 0 12 8 count 0 accum=r[8] step(r[12]) + 17 If 4 19 0 0 if r[4] goto 19; don't emit group columns if continuing existing group + 18 Column 1 0 7 0 r[7]=idx_order_items_order.order_id + 19 Integer 1 4 0 0 r[4]=1; indicate data in accumulator + 20 Next 1 9 0 1 + 21 Gosub 3 25 0 0 ; emit row for final group + 22 Goto 0 39 0 0 ; group by finished + 23 Integer 1 5 0 0 r[5]=1 + 24 Return 3 0 0 0 + 25 IfPos 4 27 0 0 r[4]>0 -> r[4]-=0, goto 27; output group by row subroutine start + 26 Return 3 0 0 0 + 27 AggFinal 0 8 0 count 0 accum=r[8] + 28 Copy 8 1 0 0 r[1]=r[8] + 29 Copy 7 2 0 0 r[2]=r[7] + 30 Copy 2 14 0 0 r[14]=r[2] + 31 Copy 1 15 0 0 r[15]=r[1] + 32 Sequence 0 16 0 0 + 33 MakeRecord 14 3 13 0 r[13]=mkrec(r[14..16]); for ephemeral_subquery_t2 + 34 IdxInsert 0 13 0 8 key=r[13] + 35 Return 3 0 0 0 + 36 Null 0 7 8 0 r[7..8]=NULL; clear accumulator subroutine start + 37 Integer 0 4 0 0 r[4]=0 + 38 Return 11 0 0 0 + 39 OpenRead 2 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 + 40 Rewind 2 59 0 0 Rewind table orders + 41 Integer 0 20 0 0 r[20]=0 + 42 RowId 2 21 0 0 r[21]=orders.rowid + 43 SeekGE 0 55 21 0 key=[21..21] + 44 IdxGT 0 55 21 0 key=[21..21] + 45 Integer 1 20 0 0 r[20]=1 + 46 Column 0 1 23 0 r[23]=ephemeral(ephemeral_subquery_t2).count(*) + 47 NotNull 23 49 0 0 r[23]!=NULL -> goto 49 + 48 Integer 0 23 0 0 r[23]=0 + 49 Lt 23 24 54 Binary 0 if r[23]0 -> r[20]-=0, goto 58 + 56 NullRow 0 0 0 0 Set cursor 0 to a (pseudo) NULL row + 57 Goto 0 45 0 0 + 58 Next 2 41 0 1 + 59 Halt 0 0 0 0 + 60 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 61 Integer 1 12 0 0 r[12]=1 + 62 Integer 3 24 0 0 r[24]=3 + 63 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-salary.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-salary.snap index a79a1e1f..86dac27e 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-salary.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-salary.snap @@ -18,43 +18,78 @@ info: --- QUERY PLAN |--SCAN employees AS e -`--CORRELATED SCALAR SUBQUERY 1 - `--SCAN employees AS e2 +`--SEARCH scalar_subquery_t2 USING INDEX ephemeral_subquery_t2 (correlation_key_0=?) + |--SCAN employees AS e2 + `--USE SORTER FOR GROUP BY BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 33 0 0 Start at 33 - 1 OpenRead 0 7 0 k(5,B,B,B,B,B) 0 table=employees, root=7, iDb=0 - 2 Rewind 0 32 0 0 Rewind table employees - 3 BeginSubrtn 6 0 0 0 r[6]=NULL - 4 Null 0 1 0 0 r[1]=NULL - 5 Null 0 8 0 0 r[8]=NULL - 6 Integer 1 9 0 0 r[9]=1; LIMIT counter - 7 IfNot 9 17 0 0 if !r[9] goto 17 - 8 OpenRead 1 7 0 k(5,B,B,B,B,B) 0 table=employees, root=7, iDb=0 - 9 Rewind 1 17 0 0 Rewind table employees - 10 Column 1 3 11 0 r[11]=employees.department - 11 Column 0 3 12 0 r[12]=employees.department - 12 Ne 11 12 16 Binary 0 if r[11]!=r[12] goto 16 - 13 Column 1 4 13 0 r[13]=employees.salary - 14 RealAffinity 13 0 0 0 - 15 AggStep 0 13 8 avg 0 accum=r[8] step(r[13]) - 16 Next 1 10 0 0 - 17 AggFinal 0 8 0 avg 0 accum=r[8] - 18 Copy 8 7 0 0 r[7]=r[8] - 19 Copy 7 1 0 0 r[1]=r[7] - 20 Return 6 0 1 0 - 21 Column 0 4 15 0 r[15]=employees.salary - 22 RealAffinity 15 0 0 0 - 23 Copy 1 16 0 0 r[16]=r[1] - 24 Le 15 16 31 Binary 0 if r[15]<=r[16] goto 31 - 25 RowId 0 2 0 0 r[2]=employees.rowid - 26 Column 0 1 3 0 r[3]=employees.name - 27 Column 0 3 4 0 r[4]=employees.department - 28 Column 0 4 5 0 r[5]=employees.salary - 29 RealAffinity 5 0 0 0 - 30 ResultRow 2 4 0 0 output=r[2..5] - 31 Next 0 3 0 0 - 32 Halt 0 0 0 0 - 33 Transaction 0 1 11 0 iDb=0 tx_mode=Read - 34 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 67 0 0 Start at 67 + 1 Once 49 0 0 0 goto 49 + 2 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 3 Null 0 8 0 0 r[8]=NULL + 4 SorterOpen 1 2 0 k(1,B) 0 cursor=1 + 5 Integer 0 5 0 0 r[5]=0; clear group by abort flag + 6 Null 0 6 0 0 r[6]=NULL; initialize group by comparison registers to NULL + 7 Gosub 12 46 0 0 ; go to clear accumulator subroutine + 8 OpenRead 3 7 0 k(5,B,B,B,B,B) 0 table=employees, root=7, iDb=0 + 9 Rewind 3 15 0 0 Rewind table employees + 10 ColumnRange 3 3 10 2 0 r[10..11]=employees.department..salary + 11 RealAffinity 11 0 0 0 + 12 MakeRecord 10 2 9 0 r[9]=mkrec(r[10..11]) + 13 SorterInsert 1 9 0 0 0 key=r[9] + 14 Next 3 10 0 1 + 15 OpenPseudo 2 9 2 0 2 columns in r[9] + 16 SorterSort 1 31 0 0 + 17 SorterData 1 9 2 0 r[9]=data + 18 Column 2 0 13 0 r[13]=pseudo.column 0 + 19 Compare 6 13 1 k(1, Binary) 0 r[6..6]==r[13..13] + 20 Jump 21 25 21 0 ; start new group if comparison is not equal + 21 Gosub 3 35 0 0 ; check if ended group had data, and output if so + 22 Move 13 6 1 0 r[6..6]=r[13..13] + 23 IfPos 5 49 0 0 r[5]>0 -> r[5]-=0, goto 49; check abort flag + 24 Gosub 12 46 0 0 ; goto clear accumulator subroutine + 25 Column 2 1 14 0 r[14]=pseudo.column 1 + 26 AggStep 0 14 8 avg 0 accum=r[8] step(r[14]) + 27 If 4 29 0 0 if r[4] goto 29; don't emit group columns if continuing existing group + 28 Column 2 0 7 0 r[7]=pseudo.column 0 + 29 Integer 1 4 0 0 r[4]=1; indicate data in accumulator + 30 SorterNext 1 17 0 0 + 31 Gosub 3 35 0 0 ; emit row for final group + 32 Goto 0 49 0 0 ; group by finished + 33 Integer 1 5 0 0 r[5]=1 + 34 Return 3 0 0 0 + 35 IfPos 4 37 0 0 r[4]>0 -> r[4]-=0, goto 37; output group by row subroutine start + 36 Return 3 0 0 0 + 37 AggFinal 0 8 0 avg 0 accum=r[8] + 38 Copy 8 1 0 0 r[1]=r[8] + 39 Copy 7 2 0 0 r[2]=r[7] + 40 Copy 2 16 0 0 r[16]=r[2] + 41 Copy 1 17 0 0 r[17]=r[1] + 42 Sequence 0 18 0 0 + 43 MakeRecord 16 3 15 0 r[15]=mkrec(r[16..18]); for ephemeral_subquery_t2 + 44 IdxInsert 0 15 0 8 key=r[15] + 45 Return 3 0 0 0 + 46 Null 0 7 8 0 r[7..8]=NULL; clear accumulator subroutine start + 47 Integer 0 4 0 0 r[4]=0 + 48 Return 12 0 0 0 + 49 OpenRead 4 7 0 k(5,B,B,B,B,B) 0 table=employees, root=7, iDb=0 + 50 Rewind 4 66 0 0 Rewind table employees + 51 Column 4 3 23 0 r[23]=employees.department + 52 IsNull 23 65 0 0 if (r[23]==NULL) goto 65 + 53 SeekGE 0 65 23 0 key=[23..23] + 54 IdxGT 0 65 23 0 key=[23..23] + 55 Column 4 4 25 0 r[25]=employees.salary + 56 RealAffinity 25 0 0 0 + 57 Column 0 1 26 0 r[26]=ephemeral(ephemeral_subquery_t2).avg(e2.salary) + 58 Le 25 26 64 Binary 0 if r[25]<=r[26] goto 64 + 59 RowId 4 19 0 0 r[19]=employees.rowid + 60 Column 4 1 20 0 r[20]=employees.name + 61 ColumnRange 4 3 21 2 0 r[21..22]=employees.department..salary + 62 RealAffinity 22 0 0 0 + 63 ResultRow 19 4 0 0 output=r[19..22] + 64 Next 0 54 0 0 + 65 Next 4 51 0 1 + 66 Halt 0 0 0 0 + 67 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 68 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-where.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-where.snap index fcd4397b..4bb47a78 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-where.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__correlated-subquery-where.snap @@ -18,45 +18,72 @@ info: --- QUERY PLAN |--SCAN products AS p -`--CORRELATED SCALAR SUBQUERY 1 - `--SEARCH p2 USING INDEX idx_products_category (category_id=?) +`--SEARCH scalar_subquery_t2 USING INDEX ephemeral_subquery_t2 (correlation_key_0=?) + `--SCAN products AS p2 USING INDEX idx_products_category BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 35 0 0 Start at 35 - 1 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 - 2 Rewind 0 34 0 0 Rewind table products - 3 BeginSubrtn 5 0 0 0 r[5]=NULL - 4 Null 0 1 0 0 r[1]=NULL - 5 Null 0 7 0 0 r[7]=NULL - 6 Integer 1 8 0 0 r[8]=1; LIMIT counter - 7 IfNot 8 20 0 0 if !r[8] goto 20 - 8 OpenRead 1 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 - 9 OpenRead 2 8 0 k(2,B) 0 index=idx_products_category, root=8, iDb=0 - 10 Column 0 2 9 0 r[9]=products.category_id - 11 IsNull 9 20 0 0 if (r[9]==NULL) goto 20 - 12 Affinity 9 1 0 0 r[9..10] = C - 13 SeekGE 2 20 9 0 key=[9..9] - 14 IdxGT 2 20 9 0 key=[9..9] - 15 DeferredSeek 2 1 0 0 - 16 Column 1 3 10 0 r[10]=products.price - 17 RealAffinity 10 0 0 0 - 18 AggStep 0 10 7 avg 0 accum=r[7] step(r[10]) - 19 Next 2 14 0 0 - 20 AggFinal 0 7 0 avg 0 accum=r[7] - 21 Copy 7 6 0 0 r[6]=r[7] - 22 Copy 6 1 0 0 r[1]=r[6] - 23 Return 5 0 1 0 - 24 Column 0 3 12 0 r[12]=products.price - 25 RealAffinity 12 0 0 0 - 26 Copy 1 13 0 0 r[13]=r[1] - 27 Le 12 13 33 Binary 0 if r[12]<=r[13] goto 33 - 28 RowId 0 2 0 0 r[2]=products.rowid - 29 Column 0 1 3 0 r[3]=products.name - 30 Column 0 3 4 0 r[4]=products.price - 31 RealAffinity 4 0 0 0 - 32 ResultRow 2 3 0 0 output=r[2..4] - 33 Next 0 3 0 0 - 34 Halt 0 0 0 0 - 35 Transaction 0 1 11 0 iDb=0 tx_mode=Read - 36 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 62 0 0 Start at 62 + 1 Once 43 0 0 0 goto 43 + 2 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 3 Null 0 8 0 0 r[8]=NULL + 4 Integer 0 5 0 0 r[5]=0; clear group by abort flag + 5 Null 0 6 0 0 r[6]=NULL; initialize group by comparison registers to NULL + 6 Gosub 12 40 0 0 ; go to clear accumulator subroutine + 7 OpenRead 1 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 + 8 OpenRead 2 8 0 k(2,B) 0 index=idx_products_category, root=8, iDb=0 + 9 Rewind 2 25 0 0 Rewind index idx_products_category + 10 DeferredSeek 2 1 0 0 + 11 Column 2 0 10 0 r[10]=idx_products_category.category_id + 12 Column 1 3 11 0 r[11]=products.price + 13 RealAffinity 11 0 0 0 + 14 Compare 6 10 1 k(1, Binary) 0 r[6..6]==r[10..10] + 15 Jump 16 20 16 0 ; start new group if comparison is not equal + 16 Gosub 3 29 0 0 ; check if ended group had data, and output if so + 17 Move 10 6 1 0 r[6..6]=r[10..10] + 18 IfPos 5 43 0 0 r[5]>0 -> r[5]-=0, goto 43; check abort flag + 19 Gosub 12 40 0 0 ; goto clear accumulator subroutine + 20 AggStep 0 11 8 avg 0 accum=r[8] step(r[11]) + 21 If 4 23 0 0 if r[4] goto 23; don't emit group columns if continuing existing group + 22 Column 2 0 7 0 r[7]=idx_products_category.category_id + 23 Integer 1 4 0 0 r[4]=1; indicate data in accumulator + 24 Next 2 10 0 1 + 25 Gosub 3 29 0 0 ; emit row for final group + 26 Goto 0 43 0 0 ; group by finished + 27 Integer 1 5 0 0 r[5]=1 + 28 Return 3 0 0 0 + 29 IfPos 4 31 0 0 r[4]>0 -> r[4]-=0, goto 31; output group by row subroutine start + 30 Return 3 0 0 0 + 31 AggFinal 0 8 0 avg 0 accum=r[8] + 32 Copy 8 1 0 0 r[1]=r[8] + 33 Copy 7 2 0 0 r[2]=r[7] + 34 Copy 2 14 0 0 r[14]=r[2] + 35 Copy 1 15 0 0 r[15]=r[1] + 36 Sequence 0 16 0 0 + 37 MakeRecord 14 3 13 0 r[13]=mkrec(r[14..16]); for ephemeral_subquery_t2 + 38 IdxInsert 0 13 0 8 key=r[13] + 39 Return 3 0 0 0 + 40 Null 0 7 8 0 r[7..8]=NULL; clear accumulator subroutine start + 41 Integer 0 4 0 0 r[4]=0 + 42 Return 12 0 0 0 + 43 OpenRead 3 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 + 44 Rewind 3 61 0 0 Rewind table products + 45 Column 3 2 20 0 r[20]=products.category_id + 46 IsNull 20 60 0 0 if (r[20]==NULL) goto 60 + 47 Affinity 20 1 0 0 r[20..21] = C + 48 SeekGE 0 60 20 0 key=[20..20] + 49 IdxGT 0 60 20 0 key=[20..20] + 50 Column 3 3 22 0 r[22]=products.price + 51 RealAffinity 22 0 0 0 + 52 Column 0 1 23 0 r[23]=ephemeral(ephemeral_subquery_t2).avg(p2.price) + 53 Le 22 23 59 Binary 0 if r[22]<=r[23] goto 59 + 54 RowId 3 17 0 0 r[17]=products.rowid + 55 Column 3 1 18 0 r[18]=products.name + 56 Column 3 3 19 0 r[19]=products.price + 57 RealAffinity 19 0 0 0 + 58 ResultRow 17 3 0 0 output=r[17..19] + 59 Next 0 49 0 0 + 60 Next 3 45 0 1 + 61 Halt 0 0 0 0 + 62 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 63 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-aggregation.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-aggregation.snap index 567da84a..b661bc8c 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-aggregation.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-aggregation.snap @@ -26,6 +26,6 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN category_stats AS cs -| `--SCAN products USING INDEX idx_products_category -`--SEARCH c USING INTEGER PRIMARY KEY (rowid=?) +|--SCAN categories AS c +`--SEARCH cs USING INDEX ephemeral_subquery_t2 (category_id=? AND product_count>?) + `--SCAN products USING INDEX idx_products_category diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-in-subquery-and-main.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-in-subquery-and-main.snap index c9cb9e0b..04675479 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-in-subquery-and-main.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-in-subquery-and-main.snap @@ -27,128 +27,137 @@ QUERY PLAN | `--SCAN category_avg | `--SCAN products USING INDEX idx_products_category |--SCAN products USING INDEX idx_products_category -|--SCAN category_avg AS ca -`--SEARCH p USING INDEX idx_products_category (category_id=?) +|--SCAN products AS p +`--SEARCH ca USING INDEX ephemeral_subquery_t5 (category_id=?) BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 117 0 0 Start at 117 - 1 Once 54 0 0 0 goto 54 - 2 BeginSubrtn 2 0 0 0 r[2]=NULL - 3 Null 0 1 0 0 r[1]=NULL - 4 InitCoroutine 3 42 5 0 - 5 Null 0 11 0 0 r[11]=NULL - 6 Integer 0 8 0 0 r[8]=0; clear group by abort flag - 7 Null 0 9 0 0 r[9]=NULL; initialize group by comparison registers to NULL - 8 Gosub 15 38 0 0 ; go to clear accumulator subroutine - 9 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 - 10 OpenRead 1 8 0 k(2,B) 0 index=idx_products_category, root=8, iDb=0 - 11 Rewind 1 27 0 0 Rewind index idx_products_category - 12 DeferredSeek 1 0 0 0 - 13 Column 1 0 13 0 r[13]=idx_products_category.category_id - 14 Column 0 3 14 0 r[14]=products.price - 15 RealAffinity 14 0 0 0 - 16 Compare 9 13 1 k(1, Binary) 0 r[9..9]==r[13..13] - 17 Jump 18 22 18 0 ; start new group if comparison is not equal - 18 Gosub 6 31 0 0 ; check if ended group had data, and output if so - 19 Move 13 9 1 0 r[9..9]=r[13..13] - 20 IfPos 8 41 0 0 r[8]>0 -> r[8]-=0, goto 41; check abort flag - 21 Gosub 15 38 0 0 ; goto clear accumulator subroutine - 22 AggStep 0 14 11 avg 0 accum=r[11] step(r[14]) - 23 If 7 25 0 0 if r[7] goto 25; don't emit group columns if continuing existing group - 24 Column 1 0 10 0 r[10]=idx_products_category.category_id - 25 Integer 1 7 0 0 r[7]=1; indicate data in accumulator - 26 Next 1 12 0 0 - 27 Gosub 6 31 0 0 ; emit row for final group - 28 Goto 0 41 0 0 ; group by finished - 29 Integer 1 8 0 0 r[8]=1 - 30 Return 6 0 0 0 - 31 IfPos 7 33 0 0 r[7]>0 -> r[7]-=0, goto 33; output group by row subroutine start - 32 Return 6 0 0 0 - 33 AggFinal 0 11 0 avg 0 accum=r[11] - 34 Copy 10 4 0 0 r[4]=r[10] - 35 Copy 11 5 0 0 r[5]=r[11] - 36 Yield 3 0 0 0 - 37 Return 6 0 0 0 - 38 Null 0 10 11 0 r[10..11]=NULL; clear accumulator subroutine start - 39 Integer 0 7 0 0 r[7]=0 - 40 Return 15 0 0 0 - 41 EndCoroutine 3 0 0 0 - 42 Null 0 17 0 0 r[17]=NULL - 43 Integer 1 18 0 0 r[18]=1; LIMIT counter - 44 IfNot 18 50 0 0 if !r[18] goto 50 - 45 InitCoroutine 3 0 5 0 - 46 Yield 3 50 0 0 - 47 Copy 5 19 0 0 r[19]=r[5] - 48 AggStep 0 19 17 max(Binary) 0 accum=r[17] step(r[19]) - 49 Goto 0 46 0 0 - 50 AggFinal 0 17 0 max 0 accum=r[17] - 51 Copy 17 16 0 0 r[16]=r[17] - 52 Copy 16 1 0 0 r[1]=r[16] - 53 Return 2 0 1 0 - 54 OpenEphemeral 2 1 0 0 cursor=2 is_table=true - 55 Null 0 29 0 0 r[29]=NULL - 56 Integer 0 26 0 0 r[26]=0; clear group by abort flag - 57 Null 0 27 0 0 r[27]=NULL; initialize group by comparison registers to NULL - 58 Gosub 33 90 0 0 ; go to clear accumulator subroutine - 59 OpenRead 3 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 - 60 OpenRead 4 8 0 k(2,B) 0 index=idx_products_category, root=8, iDb=0 - 61 Rewind 4 77 0 0 Rewind index idx_products_category - 62 DeferredSeek 4 3 0 0 - 63 Column 4 0 31 0 r[31]=idx_products_category.category_id - 64 Column 3 3 32 0 r[32]=products.price - 65 RealAffinity 32 0 0 0 - 66 Compare 27 31 1 k(1, Binary) 0 r[27..27]==r[31..31] - 67 Jump 68 72 68 0 ; start new group if comparison is not equal - 68 Gosub 24 81 0 0 ; check if ended group had data, and output if so - 69 Move 31 27 1 0 r[27..27]=r[31..31] - 70 IfPos 26 93 0 0 r[26]>0 -> r[26]-=0, goto 93; check abort flag - 71 Gosub 33 90 0 0 ; goto clear accumulator subroutine - 72 AggStep 0 32 29 avg 0 accum=r[29] step(r[32]) - 73 If 25 75 0 0 if r[25] goto 75; don't emit group columns if continuing existing group - 74 Column 4 0 28 0 r[28]=idx_products_category.category_id - 75 Integer 1 25 0 0 r[25]=1; indicate data in accumulator - 76 Next 4 62 0 0 - 77 Gosub 24 81 0 0 ; emit row for final group - 78 Goto 0 93 0 0 ; group by finished - 79 Integer 1 26 0 0 r[26]=1 - 80 Return 24 0 0 0 - 81 IfPos 25 83 0 0 r[25]>0 -> r[25]-=0, goto 83; output group by row subroutine start - 82 Return 24 0 0 0 - 83 AggFinal 0 29 0 avg 0 accum=r[29] - 84 Copy 28 22 0 0 r[22]=r[28] - 85 Copy 29 23 0 0 r[23]=r[29] - 86 MakeRecord 22 2 34 0 r[34]=mkrec(r[22..23]); for - 87 NewRowid 2 35 0 0 r[35]=rowid - 88 Insert 2 34 35 4 intkey=r[35] data=r[34] - 89 Return 24 0 0 0 - 90 Null 0 28 29 0 r[28..29]=NULL; clear accumulator subroutine start - 91 Integer 0 25 0 0 r[25]=0 - 92 Return 33 0 0 0 - 93 OpenRead 5 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 - 94 OpenRead 6 8 0 k(2,B) 0 index=idx_products_category, root=8, iDb=0 - 95 Rewind 2 116 0 0 Rewind ephemeral() - 96 Column 2 0 20 0 r[20]=ephemeral().category_id - 97 Column 2 1 21 0 r[21]=ephemeral().avg_price - 98 Copy 20 39 0 0 r[39]=r[20] - 99 IsNull 39 115 0 0 if (r[39]==NULL) goto 115 - 100 Affinity 39 1 0 0 r[39..40] = C - 101 SeekGE 6 115 39 0 key=[39..39] - 102 IdxGT 6 115 39 0 key=[39..39] - 103 DeferredSeek 6 5 0 0 - 104 Column 5 3 41 0 r[41]=products.price - 105 RealAffinity 41 0 0 0 - 106 Copy 1 43 0 0 r[43]=r[1] - 107 Multiply 43 44 42 0 r[42]=r[43]*r[44] - 108 Le 41 42 114 Binary 0 if r[41]<=r[42] goto 114 - 109 Column 5 1 36 0 r[36]=products.name - 110 Column 5 3 37 0 r[37]=products.price - 111 RealAffinity 37 0 0 0 - 112 Copy 21 38 0 0 r[38]=r[21] - 113 ResultRow 36 3 0 0 output=r[36..38] - 114 Next 6 102 0 0 - 115 Next 2 96 0 0 - 116 Halt 0 0 0 0 - 117 Transaction 0 1 11 0 iDb=0 tx_mode=Read - 118 Real 0 44 0 0.8 0 r[44]=0.8 - 119 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 126 0 0 Start at 126 + 1 Once 54 0 0 0 goto 54 + 2 BeginSubrtn 2 0 0 0 r[2]=NULL + 3 Null 0 1 0 0 r[1]=NULL + 4 InitCoroutine 3 42 5 0 + 5 Null 0 11 0 0 r[11]=NULL + 6 Integer 0 8 0 0 r[8]=0; clear group by abort flag + 7 Null 0 9 0 0 r[9]=NULL; initialize group by comparison registers to NULL + 8 Gosub 15 38 0 0 ; go to clear accumulator subroutine + 9 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 + 10 OpenRead 1 8 0 k(2,B) 0 index=idx_products_category, root=8, iDb=0 + 11 Rewind 1 27 0 0 Rewind index idx_products_category + 12 DeferredSeek 1 0 0 0 + 13 Column 1 0 13 0 r[13]=idx_products_category.category_id + 14 Column 0 3 14 0 r[14]=products.price + 15 RealAffinity 14 0 0 0 + 16 Compare 9 13 1 k(1, Binary) 0 r[9..9]==r[13..13] + 17 Jump 18 22 18 0 ; start new group if comparison is not equal + 18 Gosub 6 31 0 0 ; check if ended group had data, and output if so + 19 Move 13 9 1 0 r[9..9]=r[13..13] + 20 IfPos 8 41 0 0 r[8]>0 -> r[8]-=0, goto 41; check abort flag + 21 Gosub 15 38 0 0 ; goto clear accumulator subroutine + 22 AggStep 0 14 11 avg 0 accum=r[11] step(r[14]) + 23 If 7 25 0 0 if r[7] goto 25; don't emit group columns if continuing existing group + 24 Column 1 0 10 0 r[10]=idx_products_category.category_id + 25 Integer 1 7 0 0 r[7]=1; indicate data in accumulator + 26 Next 1 12 0 1 + 27 Gosub 6 31 0 0 ; emit row for final group + 28 Goto 0 41 0 0 ; group by finished + 29 Integer 1 8 0 0 r[8]=1 + 30 Return 6 0 0 0 + 31 IfPos 7 33 0 0 r[7]>0 -> r[7]-=0, goto 33; output group by row subroutine start + 32 Return 6 0 0 0 + 33 AggFinal 0 11 0 avg 0 accum=r[11] + 34 Copy 10 4 0 0 r[4]=r[10] + 35 Copy 11 5 0 0 r[5]=r[11] + 36 Yield 3 0 0 0 + 37 Return 6 0 0 0 + 38 Null 0 10 11 0 r[10..11]=NULL; clear accumulator subroutine start + 39 Integer 0 7 0 0 r[7]=0 + 40 Return 15 0 0 0 + 41 EndCoroutine 3 0 0 0 + 42 Null 0 17 0 0 r[17]=NULL + 43 Integer 1 18 0 0 r[18]=1; LIMIT counter + 44 IfNot 18 50 0 0 if !r[18] goto 50 + 45 InitCoroutine 3 0 5 0 + 46 Yield 3 50 0 0 + 47 Copy 5 19 0 0 r[19]=r[5] + 48 AggStep 0 19 17 max(Binary) 0 accum=r[17] step(r[19]) + 49 Goto 0 46 0 0 + 50 AggFinal 0 17 0 max 0 accum=r[17] + 51 Copy 17 16 0 0 r[16]=r[17] + 52 Copy 16 1 0 0 r[1]=r[16] + 53 Return 2 0 1 0 + 54 Once 94 0 0 0 goto 94 + 55 OpenEphemeral 2 1 0 0 cursor=2 is_table=true + 56 Null 0 29 0 0 r[29]=NULL + 57 Integer 0 26 0 0 r[26]=0; clear group by abort flag + 58 Null 0 27 0 0 r[27]=NULL; initialize group by comparison registers to NULL + 59 Gosub 33 91 0 0 ; go to clear accumulator subroutine + 60 OpenRead 3 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 + 61 OpenRead 4 8 0 k(2,B) 0 index=idx_products_category, root=8, iDb=0 + 62 Rewind 4 78 0 0 Rewind index idx_products_category + 63 DeferredSeek 4 3 0 0 + 64 Column 4 0 31 0 r[31]=idx_products_category.category_id + 65 Column 3 3 32 0 r[32]=products.price + 66 RealAffinity 32 0 0 0 + 67 Compare 27 31 1 k(1, Binary) 0 r[27..27]==r[31..31] + 68 Jump 69 73 69 0 ; start new group if comparison is not equal + 69 Gosub 24 82 0 0 ; check if ended group had data, and output if so + 70 Move 31 27 1 0 r[27..27]=r[31..31] + 71 IfPos 26 94 0 0 r[26]>0 -> r[26]-=0, goto 94; check abort flag + 72 Gosub 33 91 0 0 ; goto clear accumulator subroutine + 73 AggStep 0 32 29 avg 0 accum=r[29] step(r[32]) + 74 If 25 76 0 0 if r[25] goto 76; don't emit group columns if continuing existing group + 75 Column 4 0 28 0 r[28]=idx_products_category.category_id + 76 Integer 1 25 0 0 r[25]=1; indicate data in accumulator + 77 Next 4 63 0 1 + 78 Gosub 24 82 0 0 ; emit row for final group + 79 Goto 0 94 0 0 ; group by finished + 80 Integer 1 26 0 0 r[26]=1 + 81 Return 24 0 0 0 + 82 IfPos 25 84 0 0 r[25]>0 -> r[25]-=0, goto 84; output group by row subroutine start + 83 Return 24 0 0 0 + 84 AggFinal 0 29 0 avg 0 accum=r[29] + 85 Copy 28 22 0 0 r[22]=r[28] + 86 Copy 29 23 0 0 r[23]=r[29] + 87 MakeRecord 22 2 34 0 r[34]=mkrec(r[22..23]); for + 88 NewRowid 2 35 0 0 r[35]=rowid + 89 Insert 2 34 35 4 intkey=r[35] data=r[34] + 90 Return 24 0 0 0 + 91 Null 0 28 29 0 r[28..29]=NULL; clear accumulator subroutine start + 92 Integer 0 25 0 0 r[25]=0 + 93 Return 33 0 0 0 + 94 OpenRead 5 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 + 95 Rewind 5 125 0 0 Rewind table products + 96 Column 5 3 40 0 r[40]=products.price + 97 RealAffinity 40 0 0 0 + 98 Copy 1 42 0 0 r[42]=r[1] + 99 Multiply 42 43 41 0 r[41]=r[42]*r[43] + 100 Le 40 41 124 Binary 0 if r[40]<=r[41] goto 124 + 101 Once 110 0 0 0 goto 110 + 102 OpenAutoindex 6 0 0 0 cursor=6 + 103 Rewind 2 104 0 0 Rewind ephemeral() + 104 ColumnRange 2 0 44 2 0 r[44..45]=ephemeral().category_id..avg_price + 105 RowId 2 46 0 0 r[46]=ephemeral().rowid + 106 MakeRecord 44 3 47 0 r[47]=mkrec(r[44..46]); for ephemeral_subquery_t5 + 107 FilterAdd 6 44 1 0 bloom_filter_add(r[44..45]) + 108 IdxInsert 6 47 44 0 key=r[47] + 109 Next 2 104 0 0 + 110 Column 5 2 48 0 r[48]=products.category_id + 111 IsNull 48 124 0 0 if (r[48]==NULL) goto 124 + 112 Affinity 48 1 0 0 r[48..49] = C + 113 Filter 6 124 48 1 if !bloom_filter(r[48..49]) goto 124 + 114 SeekGE 6 124 48 0 key=[48..48] + 115 IdxGT 6 124 48 0 key=[48..48] + 116 DeferredSeek 6 2 0 0 + 117 ColumnRange 2 0 20 2 0 r[20..21]=ephemeral().category_id..avg_price + 118 Column 5 1 36 0 r[36]=products.name + 119 Column 5 3 37 0 r[37]=products.price + 120 RealAffinity 37 0 0 0 + 121 Column 6 1 38 0 r[38]=ephemeral(ephemeral_subquery_t5).avg_price + 122 ResultRow 36 3 0 0 output=r[36..38] + 123 Next 6 115 0 0 + 124 Next 5 96 0 1 + 125 Halt 0 0 0 0 + 126 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 127 Real 0 43 0 0.8 0 r[43]=0.8 + 128 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-multiple-chained.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-multiple-chained.snap index ad321989..c2cd821c 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-multiple-chained.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-multiple-chained.snap @@ -29,7 +29,7 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN customer_orders AS co -| `--SCAN orders USING INDEX idx_orders_customer -`--SEARCH ac USING INDEX ephemeral_subquery_t2 (id=?) - `--SCAN customers +|--SCAN active_customers AS ac +| `--SCAN customers +`--SEARCH co USING INDEX ephemeral_subquery_t4 (customer_id=? AND order_count>?) + `--SCAN orders USING INDEX idx_orders_customer diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-multiple-independent.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-multiple-independent.snap index bb562330..f9e30b7b 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-multiple-independent.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-multiple-independent.snap @@ -34,85 +34,80 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN vip_customers AS v -| |--LIST SUBQUERY 1 -| | `--SCAN large_orders -| | `--SCAN orders -| `--SEARCH customers USING INTEGER PRIMARY KEY (rowid=?) -`--SEARCH l USING INDEX ephemeral_subquery_t8 (customer_id=?) - `--SCAN orders +|--SCAN large_orders AS l +| `--SCAN orders +`--SEARCH v USING INDEX ephemeral_subquery_t6 (id=?) + |--LIST SUBQUERY 1 + | `--SCAN large_orders + | `--SCAN orders + `--SEARCH customers USING INTEGER PRIMARY KEY (rowid=?) BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 68 0 0 Start at 68 - 1 InitCoroutine 1 34 2 0 - 2 Once 23 0 0 0 goto 23 - 3 OpenEphemeral 0 0 0 0 cursor=0 is_table=false - 4 InitCoroutine 2 17 5 0 - 5 OpenRead 1 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 - 6 Rewind 1 16 0 0 Rewind table orders - 7 Column 1 3 7 0 r[7]=orders.total_amount - 8 RealAffinity 7 0 0 0 - 9 Le 7 8 15 Binary 0 if r[7]<=r[8] goto 15 - 10 RowId 1 3 0 0 r[3]=orders.rowid - 11 Column 1 1 4 0 r[4]=orders.customer_id - 12 Column 1 3 5 0 r[5]=orders.total_amount - 13 RealAffinity 5 0 0 0 - 14 Yield 2 0 0 0 - 15 Next 1 7 0 0 - 16 EndCoroutine 2 0 0 0 - 17 InitCoroutine 2 0 5 0 - 18 Yield 2 23 0 0 - 19 Copy 4 9 0 0 r[9]=r[4] - 20 MakeRecord 9 1 10 0 r[10]=mkrec(r[9..9]); for ephemeral_index_where_sub_t4 - 21 IdxInsert 0 10 0 8 key=r[10] - 22 Goto 0 18 0 0 - 23 OpenRead 2 6 0 k(4,B,B,B,B) 0 table=customers, root=6, iDb=0 - 24 NullRow 0 0 0 0 Set cursor 0 to a (pseudo) NULL row - 25 Rewind 0 33 0 0 Rewind ephemeral(ephemeral_index_where_sub_t4) - 26 Column 0 0 13 0 r[13]=ephemeral(ephemeral_index_where_sub_t4).customer_id - 27 IsNull 13 32 0 0 if (r[13]==NULL) goto 32 - 28 SeekRowid 2 13 32 0 if (r[13]!=cursor 2 for table customers.rowid) goto 32 - 29 RowId 2 11 0 0 r[11]=customers.rowid - 30 Column 2 1 12 0 r[12]=customers.name - 31 Yield 1 0 0 0 - 32 Next 0 26 0 0 - 33 EndCoroutine 1 0 0 0 - 34 OpenEphemeral 3 0 0 0 cursor=3 is_table=false - 35 OpenRead 4 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 - 36 Rewind 4 51 0 0 Rewind table orders - 37 Column 4 3 21 0 r[21]=orders.total_amount - 38 RealAffinity 21 0 0 0 - 39 Le 21 22 50 Binary 0 if r[21]<=r[22] goto 50 - 40 RowId 4 17 0 0 r[17]=orders.rowid - 41 Column 4 1 18 0 r[18]=orders.customer_id - 42 Column 4 3 19 0 r[19]=orders.total_amount - 43 RealAffinity 19 0 0 0 - 44 Copy 18 24 0 0 r[24]=r[18] - 45 Copy 17 25 0 0 r[25]=r[17] - 46 Copy 19 26 0 0 r[26]=r[19] - 47 Sequence 3 27 0 0 - 48 MakeRecord 24 4 23 0 r[23]=mkrec(r[24..27]); for ephemeral_subquery_t8 - 49 IdxInsert 3 23 0 8 key=r[23] - 50 Next 4 37 0 0 - 51 InitCoroutine 1 0 2 0 - 52 Yield 1 67 0 0 - 53 Copy 11 30 0 0 r[30]=r[11] - 54 IsNull 30 66 0 0 if (r[30]==NULL) goto 66 - 55 Affinity 30 1 0 0 r[30..31] = C - 56 SeekGE 3 66 30 0 key=[30..30] - 57 IdxGT 3 66 30 0 key=[30..30] - 58 Column 3 1 14 0 r[14]=ephemeral(ephemeral_subquery_t8).id - 59 Column 3 0 15 0 r[15]=ephemeral(ephemeral_subquery_t8).customer_id - 60 Column 3 2 16 0 r[16]=ephemeral(ephemeral_subquery_t8).total_amount - 61 Copy 12 28 0 0 r[28]=r[12] - 62 Column 3 2 29 0 r[29]=ephemeral(ephemeral_subquery_t8).total_amount - 63 RealAffinity 29 0 0 0 - 64 ResultRow 28 2 0 0 output=r[28..29] - 65 Next 3 57 0 0 - 66 Goto 0 52 0 0 - 67 Halt 0 0 0 0 - 68 Transaction 0 1 11 0 iDb=0 tx_mode=Read - 69 Integer 1000 8 0 0 r[8]=1000 - 70 Integer 1000 22 0 0 r[22]=1000 - 71 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 63 0 0 Start at 63 + 1 InitCoroutine 1 14 2 0 + 2 OpenRead 1 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 + 3 Rewind 1 13 0 0 Rewind table orders + 4 Column 1 3 6 0 r[6]=orders.total_amount + 5 RealAffinity 6 0 0 0 + 6 Le 6 7 12 Binary 0 if r[6]<=r[7] goto 12 + 7 RowId 1 2 0 0 r[2]=orders.rowid + 8 Column 1 1 3 0 r[3]=orders.customer_id + 9 Column 1 3 4 0 r[4]=orders.total_amount + 10 RealAffinity 4 0 0 0 + 11 Yield 1 0 0 0 + 12 Next 1 4 0 1 + 13 EndCoroutine 1 0 0 0 + 14 Once 50 0 0 0 goto 50 + 15 OpenEphemeral 2 0 0 0 cursor=2 is_table=false + 16 Once 37 0 0 0 goto 37 + 17 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 18 InitCoroutine 8 31 19 0 + 19 OpenRead 3 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 + 20 Rewind 3 30 0 0 Rewind table orders + 21 Column 3 3 13 0 r[13]=orders.total_amount + 22 RealAffinity 13 0 0 0 + 23 Le 13 14 29 Binary 0 if r[13]<=r[14] goto 29 + 24 RowId 3 9 0 0 r[9]=orders.rowid + 25 Column 3 1 10 0 r[10]=orders.customer_id + 26 Column 3 3 11 0 r[11]=orders.total_amount + 27 RealAffinity 11 0 0 0 + 28 Yield 8 0 0 0 + 29 Next 3 21 0 1 + 30 EndCoroutine 8 0 0 0 + 31 InitCoroutine 8 0 19 0 + 32 Yield 8 37 0 0 + 33 Copy 10 15 0 0 r[15]=r[10] + 34 MakeRecord 15 1 16 0 r[16]=mkrec(r[15..15]); for ephemeral_index_where_sub_t4 + 35 IdxInsert 0 16 0 8 key=r[16] + 36 Goto 0 32 0 0 + 37 OpenRead 4 6 0 k(4,B,B,B,B) 0 table=customers, root=6, iDb=0 + 38 NullRow 0 0 0 0 Set cursor 0 to a (pseudo) NULL row + 39 Rewind 0 50 0 0 Rewind ephemeral(ephemeral_index_where_sub_t4) + 40 Column 0 0 19 0 r[19]=ephemeral(ephemeral_index_where_sub_t4).customer_id + 41 IsNull 19 49 0 0 if (r[19]==NULL) goto 49 + 42 SeekRowid 4 19 49 0 if (r[19]!=cursor 4 for table customers.rowid) goto 49 + 43 RowId 4 17 0 0 r[17]=customers.rowid + 44 Column 4 1 18 0 r[18]=customers.name + 45 Copy 17 21 1 0 r[21]=r[17] + 46 Sequence 2 23 0 0 + 47 MakeRecord 21 3 20 0 r[20]=mkrec(r[21..23]); for ephemeral_subquery_t6 + 48 IdxInsert 2 20 0 8 key=r[20] + 49 Next 0 40 0 0 + 50 InitCoroutine 1 0 2 0 + 51 Yield 1 62 0 0 + 52 Copy 3 26 0 0 r[26]=r[3] + 53 IsNull 26 61 0 0 if (r[26]==NULL) goto 61 + 54 Affinity 26 1 0 0 r[26..27] = C + 55 SeekGE 2 61 26 0 key=[26..26] + 56 IdxGT 2 61 26 0 key=[26..26] + 57 Column 2 1 24 0 r[24]=ephemeral(ephemeral_subquery_t6).name + 58 Copy 4 25 0 0 r[25]=r[4] + 59 ResultRow 24 2 0 0 output=r[24..25] + 60 Next 2 56 0 0 + 61 Goto 0 51 0 0 + 62 Halt 0 0 0 0 + 63 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 64 Integer 1000 7 0 0 r[7]=1000 + 65 Integer 1000 14 0 0 r[14]=1000 + 66 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-referenced-multiple-times.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-referenced-multiple-times.snap index c6c3cba2..74ee093b 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-referenced-multiple-times.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-referenced-multiple-times.snap @@ -32,12 +32,12 @@ QUERY PLAN | `--SCAN order_totals | `--SCAN orders USING INDEX idx_orders_customer |--SCAN orders USING INDEX idx_orders_customer -|--SCAN order_totals AS ot -`--SEARCH c USING INTEGER PRIMARY KEY (rowid=?) +|--SCAN customers AS c +`--SEARCH ot USING INDEX ephemeral_subquery_t5 (customer_id=? AND total>?) BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 161 0 0 Start at 161 + 0 Init 0 171 0 0 Start at 171 1 Once 54 0 0 0 goto 54 2 BeginSubrtn 3 0 0 0 r[3]=NULL 3 Null 0 1 0 0 r[1]=NULL @@ -63,7 +63,7 @@ addr opcode p1 p2 p3 p4 p 23 If 8 25 0 0 if r[8] goto 25; don't emit group columns if continuing existing group 24 Column 1 0 11 0 r[11]=idx_orders_customer.customer_id 25 Integer 1 8 0 0 r[8]=1; indicate data in accumulator - 26 Next 1 12 0 0 + 26 Next 1 12 0 1 27 Gosub 7 31 0 0 ; emit row for final group 28 Goto 0 41 0 0 ; group by finished 29 Integer 1 9 0 0 r[9]=1 @@ -116,7 +116,7 @@ addr opcode p1 p2 p3 p4 p 76 If 26 78 0 0 if r[26] goto 78; don't emit group columns if continuing existing group 77 Column 3 0 29 0 r[29]=idx_orders_customer.customer_id 78 Integer 1 26 0 0 r[26]=1; indicate data in accumulator - 79 Next 3 65 0 0 + 79 Next 3 65 0 1 80 Gosub 25 84 0 0 ; emit row for final group 81 Goto 0 94 0 0 ; group by finished 82 Integer 1 27 0 0 r[27]=1 @@ -144,59 +144,69 @@ addr opcode p1 p2 p3 p4 p 104 Copy 36 35 0 0 r[35]=r[36] 105 Copy 35 2 0 0 r[2]=r[35] 106 Return 21 0 1 0 - 107 OpenEphemeral 4 1 0 0 cursor=4 is_table=true - 108 Null 0 48 0 0 r[48]=NULL - 109 Integer 0 45 0 0 r[45]=0; clear group by abort flag - 110 Null 0 46 0 0 r[46]=NULL; initialize group by comparison registers to NULL - 111 Gosub 52 143 0 0 ; go to clear accumulator subroutine - 112 OpenRead 5 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 - 113 OpenRead 6 9 0 k(2,B) 0 index=idx_orders_customer, root=9, iDb=0 - 114 Rewind 6 130 0 0 Rewind index idx_orders_customer - 115 DeferredSeek 6 5 0 0 - 116 Column 6 0 50 0 r[50]=idx_orders_customer.customer_id - 117 Column 5 3 51 0 r[51]=orders.total_amount - 118 RealAffinity 51 0 0 0 - 119 Compare 46 50 1 k(1, Binary) 0 r[46..46]==r[50..50] - 120 Jump 121 125 121 0 ; start new group if comparison is not equal - 121 Gosub 43 134 0 0 ; check if ended group had data, and output if so - 122 Move 50 46 1 0 r[46..46]=r[50..50] - 123 IfPos 45 146 0 0 r[45]>0 -> r[45]-=0, goto 146; check abort flag - 124 Gosub 52 143 0 0 ; goto clear accumulator subroutine - 125 AggStep 0 51 48 sum 0 accum=r[48] step(r[51]) - 126 If 44 128 0 0 if r[44] goto 128; don't emit group columns if continuing existing group - 127 Column 6 0 47 0 r[47]=idx_orders_customer.customer_id - 128 Integer 1 44 0 0 r[44]=1; indicate data in accumulator - 129 Next 6 115 0 0 - 130 Gosub 43 134 0 0 ; emit row for final group - 131 Goto 0 146 0 0 ; group by finished - 132 Integer 1 45 0 0 r[45]=1 - 133 Return 43 0 0 0 - 134 IfPos 44 136 0 0 r[44]>0 -> r[44]-=0, goto 136; output group by row subroutine start - 135 Return 43 0 0 0 - 136 AggFinal 0 48 0 sum 0 accum=r[48] - 137 Copy 47 41 0 0 r[41]=r[47] - 138 Copy 48 42 0 0 r[42]=r[48] - 139 MakeRecord 41 2 53 0 r[53]=mkrec(r[41..42]); for - 140 NewRowid 4 54 0 0 r[54]=rowid - 141 Insert 4 53 54 4 intkey=r[54] data=r[53] - 142 Return 43 0 0 0 - 143 Null 0 47 48 0 r[47..48]=NULL; clear accumulator subroutine start - 144 Integer 0 44 0 0 r[44]=0 - 145 Return 52 0 0 0 - 146 OpenRead 7 6 0 k(4,B,B,B,B) 0 table=customers, root=6, iDb=0 - 147 Rewind 4 160 0 0 Rewind ephemeral() - 148 Column 4 0 39 0 r[39]=ephemeral().customer_id - 149 Column 4 1 40 0 r[40]=ephemeral().total - 150 Copy 40 59 0 0 r[59]=r[40] - 151 Copy 1 60 0 0 r[60]=r[1] - 152 Le 59 60 159 Binary 0 if r[59]<=r[60] goto 159 - 153 Copy 39 61 0 0 r[61]=r[39] - 154 SeekRowid 7 61 159 0 if (r[61]!=cursor 7 for table customers.rowid) goto 159 - 155 Column 7 1 55 0 r[55]=customers.name - 156 Copy 40 56 0 0 r[56]=r[40] - 157 Copy 2 57 0 0 r[57]=r[2] - 158 ResultRow 55 3 0 0 output=r[55..57] - 159 Next 4 148 0 0 - 160 Halt 0 0 0 0 - 161 Transaction 0 1 11 0 iDb=0 tx_mode=Read - 162 Goto 0 1 0 0 + 107 Once 147 0 0 0 goto 147 + 108 OpenEphemeral 4 1 0 0 cursor=4 is_table=true + 109 Null 0 48 0 0 r[48]=NULL + 110 Integer 0 45 0 0 r[45]=0; clear group by abort flag + 111 Null 0 46 0 0 r[46]=NULL; initialize group by comparison registers to NULL + 112 Gosub 52 144 0 0 ; go to clear accumulator subroutine + 113 OpenRead 5 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 + 114 OpenRead 6 9 0 k(2,B) 0 index=idx_orders_customer, root=9, iDb=0 + 115 Rewind 6 131 0 0 Rewind index idx_orders_customer + 116 DeferredSeek 6 5 0 0 + 117 Column 6 0 50 0 r[50]=idx_orders_customer.customer_id + 118 Column 5 3 51 0 r[51]=orders.total_amount + 119 RealAffinity 51 0 0 0 + 120 Compare 46 50 1 k(1, Binary) 0 r[46..46]==r[50..50] + 121 Jump 122 126 122 0 ; start new group if comparison is not equal + 122 Gosub 43 135 0 0 ; check if ended group had data, and output if so + 123 Move 50 46 1 0 r[46..46]=r[50..50] + 124 IfPos 45 147 0 0 r[45]>0 -> r[45]-=0, goto 147; check abort flag + 125 Gosub 52 144 0 0 ; goto clear accumulator subroutine + 126 AggStep 0 51 48 sum 0 accum=r[48] step(r[51]) + 127 If 44 129 0 0 if r[44] goto 129; don't emit group columns if continuing existing group + 128 Column 6 0 47 0 r[47]=idx_orders_customer.customer_id + 129 Integer 1 44 0 0 r[44]=1; indicate data in accumulator + 130 Next 6 116 0 1 + 131 Gosub 43 135 0 0 ; emit row for final group + 132 Goto 0 147 0 0 ; group by finished + 133 Integer 1 45 0 0 r[45]=1 + 134 Return 43 0 0 0 + 135 IfPos 44 137 0 0 r[44]>0 -> r[44]-=0, goto 137; output group by row subroutine start + 136 Return 43 0 0 0 + 137 AggFinal 0 48 0 sum 0 accum=r[48] + 138 Copy 47 41 0 0 r[41]=r[47] + 139 Copy 48 42 0 0 r[42]=r[48] + 140 MakeRecord 41 2 53 0 r[53]=mkrec(r[41..42]); for + 141 NewRowid 4 54 0 0 r[54]=rowid + 142 Insert 4 53 54 4 intkey=r[54] data=r[53] + 143 Return 43 0 0 0 + 144 Null 0 47 48 0 r[47..48]=NULL; clear accumulator subroutine start + 145 Integer 0 44 0 0 r[44]=0 + 146 Return 52 0 0 0 + 147 OpenRead 7 6 0 k(4,B,B,B,B) 0 table=customers, root=6, iDb=0 + 148 Rewind 7 170 0 0 Rewind table customers + 149 Once 157 0 0 0 goto 157 + 150 OpenAutoindex 8 0 0 0 cursor=8 + 151 Rewind 4 152 0 0 Rewind ephemeral() + 152 ColumnRange 4 0 58 2 0 r[58..59]=ephemeral().customer_id..total + 153 RowId 4 60 0 0 r[60]=ephemeral().rowid + 154 MakeRecord 58 3 61 0 r[61]=mkrec(r[58..60]); for ephemeral_subquery_t5 + 155 IdxInsert 8 61 58 0 key=r[61] + 156 Next 4 152 0 0 + 157 RowId 7 62 0 0 r[62]=customers.rowid + 158 Copy 1 63 0 0 r[63]=r[1] + 159 IsNull 63 169 0 0 if (r[63]==NULL) goto 169 + 160 SeekGT 8 169 62 0 key=[62..63] + 161 IdxGT 8 169 62 0 key=[62..62] + 162 DeferredSeek 8 4 0 0 + 163 ColumnRange 4 0 39 2 0 r[39..40]=ephemeral().customer_id..total + 164 Column 7 1 55 0 r[55]=customers.name + 165 Column 8 1 56 0 r[56]=ephemeral(ephemeral_subquery_t5).total + 166 Copy 2 57 0 0 r[57]=r[2] + 167 ResultRow 55 3 0 0 output=r[55..57] + 168 Next 8 161 0 0 + 169 Next 7 149 0 1 + 170 Halt 0 0 0 0 + 171 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 172 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-referencing-previous.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-referencing-previous.snap index 582bae6a..7b831b81 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-referencing-previous.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-referencing-previous.snap @@ -50,96 +50,93 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 90 0 0 Start at 90 - 1 InitCoroutine 1 69 2 0 - 2 InitCoroutine 2 57 3 0 - 3 InitCoroutine 3 47 4 0 + 0 Init 0 87 0 0 Start at 87 + 1 InitCoroutine 1 67 2 0 + 2 InitCoroutine 2 56 3 0 + 3 InitCoroutine 3 46 4 0 4 Null 0 12 13 0 r[12..13]=NULL 5 Integer 0 9 0 0 r[9]=0; clear group by abort flag 6 Null 0 10 0 0 r[10]=NULL; initialize group by comparison registers to NULL - 7 Gosub 18 43 0 0 ; go to clear accumulator subroutine + 7 Gosub 18 42 0 0 ; go to clear accumulator subroutine 8 OpenRead 0 5 0 k(5,B,B,B,B,B) 0 table=order_items, root=5, iDb=0 9 OpenRead 1 11 0 k(2,B) 0 index=idx_order_items_product, root=11, iDb=0 - 10 Rewind 1 30 0 0 Rewind index idx_order_items_product + 10 Rewind 1 29 0 0 Rewind index idx_order_items_product 11 DeferredSeek 1 0 0 0 12 Column 1 0 15 0 r[15]=idx_order_items_product.product_id 13 Column 0 3 16 0 r[16]=order_items.quantity - 14 Column 0 3 19 0 r[19]=order_items.quantity - 15 Column 0 4 20 0 r[20]=order_items.unit_price - 16 RealAffinity 20 0 0 0 - 17 Multiply 19 20 17 0 r[17]=r[19]*r[20] - 18 Compare 10 15 1 k(1, Binary) 0 r[10..10]==r[15..15] - 19 Jump 20 24 20 0 ; start new group if comparison is not equal - 20 Gosub 7 34 0 0 ; check if ended group had data, and output if so - 21 Move 15 10 1 0 r[10..10]=r[15..15] - 22 IfPos 9 46 0 0 r[9]>0 -> r[9]-=0, goto 46; check abort flag - 23 Gosub 18 43 0 0 ; goto clear accumulator subroutine - 24 AggStep 0 16 12 sum 0 accum=r[12] step(r[16]) - 25 AggStep 0 17 13 sum 0 accum=r[13] step(r[17]) - 26 If 8 28 0 0 if r[8] goto 28; don't emit group columns if continuing existing group - 27 Column 1 0 11 0 r[11]=idx_order_items_product.product_id - 28 Integer 1 8 0 0 r[8]=1; indicate data in accumulator - 29 Next 1 11 0 0 - 30 Gosub 7 34 0 0 ; emit row for final group - 31 Goto 0 46 0 0 ; group by finished - 32 Integer 1 9 0 0 r[9]=1 - 33 Return 7 0 0 0 - 34 IfPos 8 36 0 0 r[8]>0 -> r[8]-=0, goto 36; output group by row subroutine start - 35 Return 7 0 0 0 - 36 AggFinal 0 12 0 sum 0 accum=r[12] - 37 AggFinal 0 13 0 sum 0 accum=r[13] - 38 Copy 11 4 0 0 r[4]=r[11] - 39 Copy 12 5 0 0 r[5]=r[12] - 40 Copy 13 6 0 0 r[6]=r[13] - 41 Yield 3 0 0 0 - 42 Return 7 0 0 0 - 43 Null 0 11 13 0 r[11..13]=NULL; clear accumulator subroutine start - 44 Integer 0 8 0 0 r[8]=0 - 45 Return 18 0 0 0 - 46 EndCoroutine 3 0 0 0 - 47 InitCoroutine 3 0 4 0 - 48 Yield 3 56 0 0 - 49 Copy 6 25 0 0 r[25]=r[6] - 50 Le 25 26 55 Binary 0 if r[25]<=r[26] goto 55 - 51 Copy 4 21 0 0 r[21]=r[4] - 52 Copy 5 22 0 0 r[22]=r[5] - 53 Copy 6 23 0 0 r[23]=r[6] - 54 Yield 2 0 0 0 - 55 Goto 0 48 0 0 - 56 EndCoroutine 2 0 0 0 - 57 OpenRead 2 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 - 58 InitCoroutine 2 0 3 0 - 59 Yield 2 68 0 0 - 60 Copy 21 31 0 0 r[31]=r[21] - 61 SeekRowid 2 31 67 0 if (r[31]!=cursor 2 for table products.rowid) goto 67 - 62 Column 2 1 27 0 r[27]=products.name - 63 Column 2 2 28 0 r[28]=products.category_id - 64 Copy 22 29 0 0 r[29]=r[22] - 65 Copy 23 30 0 0 r[30]=r[23] - 66 Yield 1 0 0 0 - 67 Goto 0 59 0 0 - 68 EndCoroutine 1 0 0 0 - 69 SorterOpen 3 1 0 k(1,-B) 0 cursor=3 - 70 OpenRead 4 3 0 k(3,B,B,B) 0 table=categories, root=3, iDb=0 - 71 InitCoroutine 1 0 2 0 - 72 Yield 1 81 0 0 - 73 Copy 28 36 0 0 r[36]=r[28] - 74 SeekRowid 4 36 80 0 if (r[36]!=cursor 4 for table categories.rowid) goto 80 - 75 Copy 30 37 0 0 r[37]=r[30] - 76 Copy 27 38 0 0 r[38]=r[27] - 77 Column 4 1 39 0 r[39]=categories.name - 78 MakeRecord 37 3 35 0 r[35]=mkrec(r[37..39]) - 79 SorterInsert 3 35 0 0 0 key=r[35] - 80 Goto 0 72 0 0 - 81 OpenPseudo 5 35 3 0 3 columns in r[35] - 82 SorterSort 3 89 0 0 - 83 SorterData 3 35 5 0 r[35]=data - 84 Column 5 1 32 0 r[32]=pseudo.column 1 - 85 Column 5 2 33 0 r[33]=pseudo.column 2 - 86 Column 5 0 34 0 r[34]=pseudo.column 0 - 87 ResultRow 32 3 0 0 output=r[32..34] - 88 SorterNext 3 83 0 0 - 89 Halt 0 0 0 0 - 90 Transaction 0 1 11 0 iDb=0 tx_mode=Read - 91 Integer 1000 26 0 0 r[26]=1000 - 92 Goto 0 1 0 0 + 14 ColumnRange 0 3 19 2 0 r[19..20]=order_items.quantity..unit_price + 15 RealAffinity 20 0 0 0 + 16 Multiply 19 20 17 0 r[17]=r[19]*r[20] + 17 Compare 10 15 1 k(1, Binary) 0 r[10..10]==r[15..15] + 18 Jump 19 23 19 0 ; start new group if comparison is not equal + 19 Gosub 7 33 0 0 ; check if ended group had data, and output if so + 20 Move 15 10 1 0 r[10..10]=r[15..15] + 21 IfPos 9 45 0 0 r[9]>0 -> r[9]-=0, goto 45; check abort flag + 22 Gosub 18 42 0 0 ; goto clear accumulator subroutine + 23 AggStep 0 16 12 sum 0 accum=r[12] step(r[16]) + 24 AggStep 0 17 13 sum 0 accum=r[13] step(r[17]) + 25 If 8 27 0 0 if r[8] goto 27; don't emit group columns if continuing existing group + 26 Column 1 0 11 0 r[11]=idx_order_items_product.product_id + 27 Integer 1 8 0 0 r[8]=1; indicate data in accumulator + 28 Next 1 11 0 1 + 29 Gosub 7 33 0 0 ; emit row for final group + 30 Goto 0 45 0 0 ; group by finished + 31 Integer 1 9 0 0 r[9]=1 + 32 Return 7 0 0 0 + 33 IfPos 8 35 0 0 r[8]>0 -> r[8]-=0, goto 35; output group by row subroutine start + 34 Return 7 0 0 0 + 35 AggFinal 0 12 0 sum 0 accum=r[12] + 36 AggFinal 0 13 0 sum 0 accum=r[13] + 37 Copy 11 4 0 0 r[4]=r[11] + 38 Copy 12 5 0 0 r[5]=r[12] + 39 Copy 13 6 0 0 r[6]=r[13] + 40 Yield 3 0 0 0 + 41 Return 7 0 0 0 + 42 Null 0 11 13 0 r[11..13]=NULL; clear accumulator subroutine start + 43 Integer 0 8 0 0 r[8]=0 + 44 Return 18 0 0 0 + 45 EndCoroutine 3 0 0 0 + 46 InitCoroutine 3 0 4 0 + 47 Yield 3 55 0 0 + 48 Copy 6 25 0 0 r[25]=r[6] + 49 Le 25 26 54 Binary 0 if r[25]<=r[26] goto 54 + 50 Copy 4 21 0 0 r[21]=r[4] + 51 Copy 5 22 0 0 r[22]=r[5] + 52 Copy 6 23 0 0 r[23]=r[6] + 53 Yield 2 0 0 0 + 54 Goto 0 47 0 0 + 55 EndCoroutine 2 0 0 0 + 56 OpenRead 2 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 + 57 InitCoroutine 2 0 3 0 + 58 Yield 2 66 0 0 + 59 Copy 21 31 0 0 r[31]=r[21] + 60 SeekRowid 2 31 65 0 if (r[31]!=cursor 2 for table products.rowid) goto 65 + 61 ColumnRange 2 1 27 2 0 r[27..28]=products.name..category_id + 62 Copy 22 29 0 0 r[29]=r[22] + 63 Copy 23 30 0 0 r[30]=r[23] + 64 Yield 1 0 0 0 + 65 Goto 0 58 0 0 + 66 EndCoroutine 1 0 0 0 + 67 SorterOpen 3 1 0 k(1,-B) 0 cursor=3 + 68 OpenRead 4 3 0 k(3,B,B,B) 0 table=categories, root=3, iDb=0 + 69 InitCoroutine 1 0 2 0 + 70 Yield 1 79 0 0 + 71 Copy 28 36 0 0 r[36]=r[28] + 72 SeekRowid 4 36 78 0 if (r[36]!=cursor 4 for table categories.rowid) goto 78 + 73 Copy 30 37 0 0 r[37]=r[30] + 74 Copy 27 38 0 0 r[38]=r[27] + 75 Column 4 1 39 0 r[39]=categories.name + 76 MakeRecord 37 3 35 0 r[35]=mkrec(r[37..39]) + 77 SorterInsert 3 35 0 0 0 key=r[35] + 78 Goto 0 70 0 0 + 79 OpenPseudo 5 35 3 0 3 columns in r[35] + 80 SorterSort 3 86 0 0 + 81 SorterData 3 35 5 0 r[35]=data + 82 ColumnRange 5 1 32 2 0 r[32..33]=pseudo.column 1..column 2 + 83 Column 5 0 34 0 r[34]=pseudo.column 0 + 84 ResultRow 32 3 0 0 output=r[32..34] + 85 SorterNext 3 81 0 0 + 86 Halt 0 0 0 0 + 87 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 88 Integer 1000 26 0 0 r[26]=1000 + 89 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-with-correlated-subquery.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-with-correlated-subquery.snap index 233c7c21..5ca2dd86 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-with-correlated-subquery.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__cte-with-correlated-subquery.snap @@ -31,81 +31,78 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 76 0 0 Start at 76 + 0 Init 0 73 0 0 Start at 73 1 OpenRead 0 7 0 k(5,B,B,B,B,B) 0 table=employees, root=7, iDb=0 - 2 Rewind 0 75 0 0 Rewind table employees + 2 Rewind 0 72 0 0 Rewind table employees 3 BeginSubrtn 5 0 0 0 r[5]=NULL 4 Null 0 1 0 0 r[1]=NULL - 5 OpenEphemeral 1 0 0 0 cursor=1 is_table=false - 6 Null 0 15 0 0 r[15]=NULL - 7 SorterOpen 2 2 0 k(1,B) 0 cursor=2 - 8 Integer 0 12 0 0 r[12]=0; clear group by abort flag - 9 Null 0 13 0 0 r[13]=NULL; initialize group by comparison registers to NULL - 10 Gosub 19 49 0 0 ; go to clear accumulator subroutine - 11 OpenRead 4 7 0 k(5,B,B,B,B,B) 0 table=employees, root=7, iDb=0 - 12 Rewind 4 19 0 0 Rewind table employees - 13 Column 4 3 17 0 r[17]=employees.department - 14 Column 4 4 18 0 r[18]=employees.salary - 15 RealAffinity 18 0 0 0 - 16 MakeRecord 17 2 16 0 r[16]=mkrec(r[17..18]) - 17 SorterInsert 2 16 0 0 0 key=r[16] - 18 Next 4 13 0 0 - 19 OpenPseudo 3 16 2 0 2 columns in r[16] + 5 Once 52 0 0 0 goto 52 + 6 OpenEphemeral 1 0 0 0 cursor=1 is_table=false + 7 Null 0 13 0 0 r[13]=NULL + 8 SorterOpen 2 2 0 k(1,B) 0 cursor=2 + 9 Integer 0 10 0 0 r[10]=0; clear group by abort flag + 10 Null 0 11 0 0 r[11]=NULL; initialize group by comparison registers to NULL + 11 Gosub 17 49 0 0 ; go to clear accumulator subroutine + 12 OpenRead 4 7 0 k(5,B,B,B,B,B) 0 table=employees, root=7, iDb=0 + 13 Rewind 4 19 0 0 Rewind table employees + 14 ColumnRange 4 3 15 2 0 r[15..16]=employees.department..salary + 15 RealAffinity 16 0 0 0 + 16 MakeRecord 15 2 14 0 r[14]=mkrec(r[15..16]) + 17 SorterInsert 2 14 0 0 0 key=r[14] + 18 Next 4 14 0 1 + 19 OpenPseudo 3 14 2 0 2 columns in r[14] 20 SorterSort 2 35 0 0 - 21 SorterData 2 16 3 0 r[16]=data - 22 Column 3 0 20 0 r[20]=pseudo.column 0 - 23 Compare 13 20 1 k(1, Binary) 0 r[13..13]==r[20..20] + 21 SorterData 2 14 3 0 r[14]=data + 22 Column 3 0 18 0 r[18]=pseudo.column 0 + 23 Compare 11 18 1 k(1, Binary) 0 r[11..11]==r[18..18] 24 Jump 25 29 25 0 ; start new group if comparison is not equal - 25 Gosub 10 39 0 0 ; check if ended group had data, and output if so - 26 Move 20 13 1 0 r[13..13]=r[20..20] - 27 IfPos 12 52 0 0 r[12]>0 -> r[12]-=0, goto 52; check abort flag - 28 Gosub 19 49 0 0 ; goto clear accumulator subroutine - 29 Column 3 1 21 0 r[21]=pseudo.column 1 - 30 AggStep 0 21 15 avg 0 accum=r[15] step(r[21]) - 31 If 11 33 0 0 if r[11] goto 33; don't emit group columns if continuing existing group - 32 Column 3 0 14 0 r[14]=pseudo.column 0 - 33 Integer 1 11 0 0 r[11]=1; indicate data in accumulator + 25 Gosub 8 39 0 0 ; check if ended group had data, and output if so + 26 Move 18 11 1 0 r[11..11]=r[18..18] + 27 IfPos 10 52 0 0 r[10]>0 -> r[10]-=0, goto 52; check abort flag + 28 Gosub 17 49 0 0 ; goto clear accumulator subroutine + 29 Column 3 1 19 0 r[19]=pseudo.column 1 + 30 AggStep 0 19 13 avg 0 accum=r[13] step(r[19]) + 31 If 9 33 0 0 if r[9] goto 33; don't emit group columns if continuing existing group + 32 Column 3 0 12 0 r[12]=pseudo.column 0 + 33 Integer 1 9 0 0 r[9]=1; indicate data in accumulator 34 SorterNext 2 21 0 0 - 35 Gosub 10 39 0 0 ; emit row for final group + 35 Gosub 8 39 0 0 ; emit row for final group 36 Goto 0 52 0 0 ; group by finished - 37 Integer 1 12 0 0 r[12]=1 - 38 Return 10 0 0 0 - 39 IfPos 11 41 0 0 r[11]>0 -> r[11]-=0, goto 41; output group by row subroutine start - 40 Return 10 0 0 0 - 41 AggFinal 0 15 0 avg 0 accum=r[15] - 42 Copy 14 8 0 0 r[8]=r[14] - 43 Copy 15 9 0 0 r[9]=r[15] - 44 Copy 8 23 1 0 r[23]=r[8] - 45 Sequence 1 25 0 0 - 46 MakeRecord 23 3 22 0 r[22]=mkrec(r[23..25]); for ephemeral_subquery_t6 - 47 IdxInsert 1 22 0 8 key=r[22] - 48 Return 10 0 0 0 - 49 Null 0 14 15 0 r[14..15]=NULL; clear accumulator subroutine start - 50 Integer 0 11 0 0 r[11]=0 - 51 Return 19 0 0 0 - 52 Integer 1 27 0 0 r[27]=1; LIMIT counter - 53 IfNot 27 64 0 0 if !r[27] goto 64 - 54 Column 0 3 28 0 r[28]=employees.department - 55 IsNull 28 64 0 0 if (r[28]==NULL) goto 64 - 56 SeekGE 1 64 28 0 key=[28..28] - 57 IdxGT 1 64 28 0 key=[28..28] - 58 Column 1 0 6 0 r[6]=ephemeral(ephemeral_subquery_t6).department - 59 Column 1 1 7 0 r[7]=ephemeral(ephemeral_subquery_t6).avg_salary - 60 Column 1 1 26 0 r[26]=ephemeral(ephemeral_subquery_t6).avg_salary - 61 Copy 26 1 0 0 r[1]=r[26] - 62 DecrJumpZero 27 64 0 0 if (--r[27]==0) goto 64 - 63 Next 1 57 0 0 - 64 Return 5 0 1 0 - 65 Column 0 4 30 0 r[30]=employees.salary - 66 RealAffinity 30 0 0 0 - 67 Copy 1 31 0 0 r[31]=r[1] - 68 Le 30 31 74 Binary 0 if r[30]<=r[31] goto 74 - 69 Column 0 1 2 0 r[2]=employees.name - 70 Column 0 3 3 0 r[3]=employees.department - 71 Column 0 4 4 0 r[4]=employees.salary - 72 RealAffinity 4 0 0 0 - 73 ResultRow 2 3 0 0 output=r[2..4] - 74 Next 0 3 0 0 - 75 Halt 0 0 0 0 - 76 Transaction 0 1 11 0 iDb=0 tx_mode=Read - 77 Goto 0 1 0 0 + 37 Integer 1 10 0 0 r[10]=1 + 38 Return 8 0 0 0 + 39 IfPos 9 41 0 0 r[9]>0 -> r[9]-=0, goto 41; output group by row subroutine start + 40 Return 8 0 0 0 + 41 AggFinal 0 13 0 avg 0 accum=r[13] + 42 Copy 12 6 0 0 r[6]=r[12] + 43 Copy 13 7 0 0 r[7]=r[13] + 44 Copy 6 21 1 0 r[21]=r[6] + 45 Sequence 1 23 0 0 + 46 MakeRecord 21 3 20 0 r[20]=mkrec(r[21..23]); for ephemeral_subquery_t6 + 47 IdxInsert 1 20 0 8 key=r[20] + 48 Return 8 0 0 0 + 49 Null 0 12 13 0 r[12..13]=NULL; clear accumulator subroutine start + 50 Integer 0 9 0 0 r[9]=0 + 51 Return 17 0 0 0 + 52 Integer 1 25 0 0 r[25]=1; LIMIT counter + 53 IfNot 25 62 0 0 if !r[25] goto 62 + 54 Column 0 3 26 0 r[26]=employees.department + 55 IsNull 26 62 0 0 if (r[26]==NULL) goto 62 + 56 SeekGE 1 62 26 0 key=[26..26] + 57 IdxGT 1 62 26 0 key=[26..26] + 58 Column 1 1 24 0 r[24]=ephemeral(ephemeral_subquery_t6).avg_salary + 59 Copy 24 1 0 0 r[1]=r[24] + 60 DecrJumpZero 25 62 0 0 if (--r[25]==0) goto 62 + 61 Next 1 57 0 0 + 62 Return 5 0 1 0 + 63 Column 0 4 28 0 r[28]=employees.salary + 64 RealAffinity 28 0 0 0 + 65 Copy 1 29 0 0 r[29]=r[1] + 66 Le 28 29 71 Binary 0 if r[28]<=r[29] goto 71 + 67 Column 0 1 2 0 r[2]=employees.name + 68 ColumnRange 0 3 3 2 0 r[3..4]=employees.department..salary + 69 RealAffinity 4 0 0 0 + 70 ResultRow 2 3 0 0 output=r[2..4] + 71 Next 0 3 0 1 + 72 Halt 0 0 0 0 + 73 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 74 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__delete-returning-target-selfread.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__delete-returning-target-selfread.snap index 9e46d617..2059b8de 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__delete-returning-target-selfread.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__delete-returning-target-selfread.snap @@ -19,50 +19,48 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 44 0 0 Start at 44 + 0 Init 0 42 0 0 Start at 42 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true 2 OpenWrite 1 2 0 0 root=2; iDb=0 - 3 Rewind 1 37 0 0 Rewind table t + 3 Rewind 1 36 0 0 Rewind table t 4 Integer 3 2 0 0 r[2]=3 5 RowId 1 3 0 0 r[3]=t.rowid - 6 Ge 3 2 37 0 if r[3]>=r[2] goto 37 + 6 Ge 3 2 36 0 if r[3]>=r[2] goto 36 7 RowId 1 4 0 0 r[4]=t.rowid 8 RowId 1 5 0 0 r[5]=t.rowid - 9 Column 1 1 6 0 r[6]=t.grp - 10 Column 1 2 7 0 r[7]=t.val - 11 Delete 1 0 0 t 0 - 12 Copy 4 8 0 0 r[8]=r[4] - 13 Copy 6 9 0 0 r[9]=r[6] - 14 Copy 7 10 0 0 r[10]=r[7] - 15 BeginSubrtn 11 0 0 0 r[11]=NULL - 16 Null 0 1 0 0 r[1]=NULL - 17 Null 0 13 0 0 r[13]=NULL - 18 Integer 1 14 0 0 r[14]=1; LIMIT counter - 19 IfNot 14 24 0 0 if !r[14] goto 24 - 20 OpenRead 2 2 0 k(3,B,B,B) 0 table=t, root=2, iDb=0 - 21 Rewind 2 24 0 0 Rewind table t - 22 AggStep 0 15 13 count 0 accum=r[13] step(r[15]) - 23 Next 2 22 0 0 - 24 AggFinal 0 13 0 count 0 accum=r[13] - 25 Copy 13 12 0 0 r[12]=r[13] - 26 Copy 12 1 0 0 r[1]=r[12] - 27 Return 11 0 1 0 - 28 Copy 4 16 0 0 r[16]=r[4] - 29 Copy 6 17 0 0 r[17]=r[6] - 30 Copy 7 18 0 0 r[18]=r[7] - 31 Copy 16 19 0 0 r[19]=r[16] - 32 Copy 1 20 0 0 r[20]=r[1] - 33 MakeRecord 19 2 21 0 r[21]=mkrec(r[19..20]) - 34 NewRowid 0 22 0 0 r[22]=rowid - 35 Insert 0 21 22 4 intkey=r[22] data=r[21] - 36 Next 1 5 0 0 - 37 FkCheck 0 0 0 0 - 38 Rewind 0 43 0 0 Rewind ephemeral(t) - 39 Column 0 0 23 0 r[23]=ephemeral(t).id - 40 Column 0 1 24 0 r[24]=ephemeral(t).grp - 41 ResultRow 23 2 0 0 output=r[23..24] - 42 Next 0 39 0 0 - 43 Halt 0 0 0 0 - 44 Transaction 0 2 1 0 iDb=0 tx_mode=Write - 45 Integer 1 15 0 0 r[15]=1 - 46 Goto 0 1 0 0 + 9 ColumnRange 1 1 6 2 0 r[6..7]=t.grp..val + 10 Delete 1 0 0 t 0 + 11 Copy 4 8 0 0 r[8]=r[4] + 12 Copy 6 9 0 0 r[9]=r[6] + 13 Copy 7 10 0 0 r[10]=r[7] + 14 BeginSubrtn 11 0 0 0 r[11]=NULL + 15 Null 0 1 0 0 r[1]=NULL + 16 Null 0 13 0 0 r[13]=NULL + 17 Integer 1 14 0 0 r[14]=1; LIMIT counter + 18 IfNot 14 23 0 0 if !r[14] goto 23 + 19 OpenRead 2 2 0 k(3,B,B,B) 0 table=t, root=2, iDb=0 + 20 Rewind 2 23 0 0 Rewind table t + 21 AggStep 0 15 13 count 0 accum=r[13] step(r[15]) + 22 Next 2 21 0 1 + 23 AggFinal 0 13 0 count 0 accum=r[13] + 24 Copy 13 12 0 0 r[12]=r[13] + 25 Copy 12 1 0 0 r[1]=r[12] + 26 Return 11 0 1 0 + 27 Copy 4 16 0 0 r[16]=r[4] + 28 Copy 6 17 0 0 r[17]=r[6] + 29 Copy 7 18 0 0 r[18]=r[7] + 30 Copy 16 19 0 0 r[19]=r[16] + 31 Copy 1 20 0 0 r[20]=r[1] + 32 MakeRecord 19 2 21 0 r[21]=mkrec(r[19..20]) + 33 NewRowid 0 22 0 0 r[22]=rowid + 34 Insert 0 21 22 4 intkey=r[22] data=r[21] + 35 Next 1 5 0 0 + 36 FkCheck 0 0 0 0 + 37 Rewind 0 41 0 0 Rewind ephemeral(t) + 38 ColumnRange 0 0 23 2 0 r[23..24]=ephemeral(t).id..grp + 39 ResultRow 23 2 0 0 output=r[23..24] + 40 Next 0 38 0 0 + 41 Halt 0 0 0 0 + 42 Transaction 0 2 1 0 iDb=0 tx_mode=Write + 43 Integer 1 15 0 0 r[15]=1 + 44 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__derived-table-join.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__derived-table-join.snap index 4c8c2aa1..643cea60 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__derived-table-join.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__derived-table-join.snap @@ -23,68 +23,72 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN order_stats -| `--SCAN orders USING INDEX idx_orders_customer -`--SEARCH c USING INTEGER PRIMARY KEY (rowid=?) +|--SCAN customers AS c +`--SEARCH order_stats USING INDEX ephemeral_subquery_t3 (customer_id=? AND total_spent>?) + `--SCAN orders USING INDEX idx_orders_customer BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 55 0 0 Start at 55 - 1 InitCoroutine 1 42 2 0 - 2 Null 0 10 11 0 r[10..11]=NULL - 3 Integer 0 7 0 0 r[7]=0; clear group by abort flag - 4 Null 0 8 0 0 r[8]=NULL; initialize group by comparison registers to NULL - 5 Gosub 15 38 0 0 ; go to clear accumulator subroutine - 6 OpenRead 0 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 - 7 OpenRead 1 9 0 k(2,B) 0 index=idx_orders_customer, root=9, iDb=0 - 8 Rewind 1 25 0 0 Rewind index idx_orders_customer - 9 DeferredSeek 1 0 0 0 - 10 Column 1 0 13 0 r[13]=idx_orders_customer.customer_id - 11 Column 0 3 14 0 r[14]=orders.total_amount - 12 RealAffinity 14 0 0 0 - 13 Compare 8 13 1 k(1, Binary) 0 r[8..8]==r[13..13] - 14 Jump 15 19 15 0 ; start new group if comparison is not equal - 15 Gosub 5 29 0 0 ; check if ended group had data, and output if so - 16 Move 13 8 1 0 r[8..8]=r[13..13] - 17 IfPos 7 41 0 0 r[7]>0 -> r[7]-=0, goto 41; check abort flag - 18 Gosub 15 38 0 0 ; goto clear accumulator subroutine - 19 AggStep 0 16 10 count 0 accum=r[10] step(r[16]) - 20 AggStep 0 14 11 sum 0 accum=r[11] step(r[14]) - 21 If 6 23 0 0 if r[6] goto 23; don't emit group columns if continuing existing group - 22 Column 1 0 9 0 r[9]=idx_orders_customer.customer_id - 23 Integer 1 6 0 0 r[6]=1; indicate data in accumulator - 24 Next 1 9 0 0 - 25 Gosub 5 29 0 0 ; emit row for final group - 26 Goto 0 41 0 0 ; group by finished - 27 Integer 1 7 0 0 r[7]=1 - 28 Return 5 0 0 0 - 29 IfPos 6 31 0 0 r[6]>0 -> r[6]-=0, goto 31; output group by row subroutine start - 30 Return 5 0 0 0 - 31 AggFinal 0 10 0 count 0 accum=r[10] - 32 AggFinal 0 11 0 sum 0 accum=r[11] - 33 Copy 9 2 0 0 r[2]=r[9] - 34 Copy 10 3 0 0 r[3]=r[10] - 35 Copy 11 4 0 0 r[4]=r[11] - 36 Yield 1 0 0 0 - 37 Return 5 0 0 0 - 38 Null 0 9 11 0 r[9..11]=NULL; clear accumulator subroutine start - 39 Integer 0 6 0 0 r[6]=0 - 40 Return 15 0 0 0 - 41 EndCoroutine 1 0 0 0 - 42 OpenRead 2 6 0 k(4,B,B,B,B) 0 table=customers, root=6, iDb=0 - 43 InitCoroutine 1 0 2 0 - 44 Yield 1 54 0 0 - 45 Copy 4 21 0 0 r[21]=r[4] - 46 Le 21 22 53 Binary 0 if r[21]<=r[22] goto 53 - 47 Copy 2 23 0 0 r[23]=r[2] - 48 SeekRowid 2 23 53 0 if (r[23]!=cursor 2 for table customers.rowid) goto 53 - 49 Column 2 1 17 0 r[17]=customers.name - 50 Copy 3 18 0 0 r[18]=r[3] - 51 Copy 4 19 0 0 r[19]=r[4] - 52 ResultRow 17 3 0 0 output=r[17..19] - 53 Goto 0 44 0 0 - 54 Halt 0 0 0 0 - 55 Transaction 0 1 11 0 iDb=0 tx_mode=Read - 56 Integer 1 16 0 0 r[16]=1 - 57 Integer 500 22 0 0 r[22]=500 - 58 Goto 0 1 0 0 + 0 Init 0 60 0 0 Start at 60 + 1 Once 47 0 0 0 goto 47 + 2 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 3 Null 0 9 10 0 r[9..10]=NULL + 4 Integer 0 6 0 0 r[6]=0; clear group by abort flag + 5 Null 0 7 0 0 r[7]=NULL; initialize group by comparison registers to NULL + 6 Gosub 14 44 0 0 ; go to clear accumulator subroutine + 7 OpenRead 1 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 + 8 OpenRead 2 9 0 k(2,B) 0 index=idx_orders_customer, root=9, iDb=0 + 9 Rewind 2 26 0 0 Rewind index idx_orders_customer + 10 DeferredSeek 2 1 0 0 + 11 Column 2 0 12 0 r[12]=idx_orders_customer.customer_id + 12 Column 1 3 13 0 r[13]=orders.total_amount + 13 RealAffinity 13 0 0 0 + 14 Compare 7 12 1 k(1, Binary) 0 r[7..7]==r[12..12] + 15 Jump 16 20 16 0 ; start new group if comparison is not equal + 16 Gosub 4 30 0 0 ; check if ended group had data, and output if so + 17 Move 12 7 1 0 r[7..7]=r[12..12] + 18 IfPos 6 47 0 0 r[6]>0 -> r[6]-=0, goto 47; check abort flag + 19 Gosub 14 44 0 0 ; goto clear accumulator subroutine + 20 AggStep 0 15 9 count 0 accum=r[9] step(r[15]) + 21 AggStep 0 13 10 sum 0 accum=r[10] step(r[13]) + 22 If 5 24 0 0 if r[5] goto 24; don't emit group columns if continuing existing group + 23 Column 2 0 8 0 r[8]=idx_orders_customer.customer_id + 24 Integer 1 5 0 0 r[5]=1; indicate data in accumulator + 25 Next 2 10 0 1 + 26 Gosub 4 30 0 0 ; emit row for final group + 27 Goto 0 47 0 0 ; group by finished + 28 Integer 1 6 0 0 r[6]=1 + 29 Return 4 0 0 0 + 30 IfPos 5 32 0 0 r[5]>0 -> r[5]-=0, goto 32; output group by row subroutine start + 31 Return 4 0 0 0 + 32 AggFinal 0 9 0 count 0 accum=r[9] + 33 AggFinal 0 10 0 sum 0 accum=r[10] + 34 Copy 8 1 0 0 r[1]=r[8] + 35 Copy 9 2 0 0 r[2]=r[9] + 36 Copy 10 3 0 0 r[3]=r[10] + 37 Copy 1 17 0 0 r[17]=r[1] + 38 Copy 3 18 0 0 r[18]=r[3] + 39 Copy 2 19 0 0 r[19]=r[2] + 40 Sequence 0 20 0 0 + 41 MakeRecord 17 4 16 0 r[16]=mkrec(r[17..20]); for ephemeral_subquery_t3 + 42 IdxInsert 0 16 0 8 key=r[16] + 43 Return 4 0 0 0 + 44 Null 0 8 10 0 r[8..10]=NULL; clear accumulator subroutine start + 45 Integer 0 5 0 0 r[5]=0 + 46 Return 14 0 0 0 + 47 OpenRead 3 6 0 k(4,B,B,B,B) 0 table=customers, root=6, iDb=0 + 48 Rewind 3 59 0 0 Rewind table customers + 49 RowId 3 24 0 0 r[24]=customers.rowid + 50 Integer 500 25 0 0 r[25]=500 + 51 SeekGT 0 58 24 0 key=[24..25] + 52 IdxGT 0 58 24 0 key=[24..24] + 53 Column 3 1 21 0 r[21]=customers.name + 54 Column 0 2 22 0 r[22]=ephemeral(ephemeral_subquery_t3).total_orders + 55 Column 0 1 23 0 r[23]=ephemeral(ephemeral_subquery_t3).total_spent + 56 ResultRow 21 3 0 0 output=r[21..23] + 57 Next 0 52 0 0 + 58 Next 3 49 0 1 + 59 Halt 0 0 0 0 + 60 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 61 Integer 1 15 0 0 r[15]=1 + 62 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__derived-table-nested.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__derived-table-nested.snap index 917c7d64..3e8d1522 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__derived-table-nested.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__derived-table-nested.snap @@ -24,75 +24,86 @@ info: database: ':memory:' --- QUERY PLAN -`--SEARCH top_customers USING INDEX ephemeral_subquery_t4 (total_spent>?) - |--SCAN stats - | `--SCAN orders USING INDEX idx_orders_customer - `--SEARCH c USING INTEGER PRIMARY KEY (rowid=?) +|--SCAN top_customers +| |--SCAN customers AS c +| `--SEARCH stats USING INDEX ephemeral_subquery_t3 (customer_id=?) +| `--SCAN orders USING INDEX idx_orders_customer +`--USE SORTER FOR ORDER BY BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 63 0 0 Start at 63 - 1 OpenEphemeral 0 0 0 0 cursor=0 is_table=false - 2 InitCoroutine 3 40 3 0 - 3 Null 0 11 0 0 r[11]=NULL - 4 Integer 0 8 0 0 r[8]=0; clear group by abort flag - 5 Null 0 9 0 0 r[9]=NULL; initialize group by comparison registers to NULL - 6 Gosub 15 36 0 0 ; go to clear accumulator subroutine - 7 OpenRead 1 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 - 8 OpenRead 2 9 0 k(2,B) 0 index=idx_orders_customer, root=9, iDb=0 - 9 Rewind 2 25 0 0 Rewind index idx_orders_customer - 10 DeferredSeek 2 1 0 0 - 11 Column 2 0 13 0 r[13]=idx_orders_customer.customer_id - 12 Column 1 3 14 0 r[14]=orders.total_amount - 13 RealAffinity 14 0 0 0 - 14 Compare 9 13 1 k(1, Binary) 0 r[9..9]==r[13..13] - 15 Jump 16 20 16 0 ; start new group if comparison is not equal - 16 Gosub 6 29 0 0 ; check if ended group had data, and output if so - 17 Move 13 9 1 0 r[9..9]=r[13..13] - 18 IfPos 8 39 0 0 r[8]>0 -> r[8]-=0, goto 39; check abort flag - 19 Gosub 15 36 0 0 ; goto clear accumulator subroutine - 20 AggStep 0 14 11 sum 0 accum=r[11] step(r[14]) - 21 If 7 23 0 0 if r[7] goto 23; don't emit group columns if continuing existing group - 22 Column 2 0 10 0 r[10]=idx_orders_customer.customer_id - 23 Integer 1 7 0 0 r[7]=1; indicate data in accumulator - 24 Next 2 10 0 0 - 25 Gosub 6 29 0 0 ; emit row for final group - 26 Goto 0 39 0 0 ; group by finished - 27 Integer 1 8 0 0 r[8]=1 - 28 Return 6 0 0 0 - 29 IfPos 7 31 0 0 r[7]>0 -> r[7]-=0, goto 31; output group by row subroutine start - 30 Return 6 0 0 0 - 31 AggFinal 0 11 0 sum 0 accum=r[11] - 32 Copy 10 4 0 0 r[4]=r[10] - 33 Copy 11 5 0 0 r[5]=r[11] - 34 Yield 3 0 0 0 - 35 Return 6 0 0 0 - 36 Null 0 10 11 0 r[10..11]=NULL; clear accumulator subroutine start - 37 Integer 0 7 0 0 r[7]=0 - 38 Return 15 0 0 0 - 39 EndCoroutine 3 0 0 0 - 40 OpenRead 3 6 0 k(4,B,B,B,B) 0 table=customers, root=6, iDb=0 - 41 InitCoroutine 3 0 3 0 - 42 Yield 3 53 0 0 - 43 Copy 4 18 0 0 r[18]=r[4] - 44 SeekRowid 3 18 52 0 if (r[18]!=cursor 3 for table customers.rowid) goto 52 - 45 Column 3 1 16 0 r[16]=customers.name - 46 Copy 5 17 0 0 r[17]=r[5] - 47 Copy 17 20 0 0 r[20]=r[17] - 48 Copy 16 21 0 0 r[21]=r[16] - 49 Sequence 0 22 0 0 - 50 MakeRecord 20 3 19 0 r[19]=mkrec(r[20..22]); for ephemeral_subquery_t4 - 51 IdxInsert 0 19 0 8 key=r[19] - 52 Goto 0 42 0 0 - 53 Last 0 62 0 0 - 54 Integer 1000 25 0 0 r[25]=1000 - 55 IdxLE 0 62 25 0 key=[25..25] - 56 Column 0 1 1 0 r[1]=ephemeral(ephemeral_subquery_t4).name - 57 Column 0 0 2 0 r[2]=ephemeral(ephemeral_subquery_t4).total_spent - 58 Column 0 1 23 0 r[23]=ephemeral(ephemeral_subquery_t4).name - 59 Column 0 0 24 0 r[24]=ephemeral(ephemeral_subquery_t4).total_spent - 60 ResultRow 23 2 0 0 output=r[23..24] - 61 Prev 0 55 0 0 - 62 Halt 0 0 0 0 - 63 Transaction 0 1 11 0 iDb=0 tx_mode=Read - 64 Goto 0 1 0 0 + 0 Init 0 72 0 0 Start at 72 + 1 InitCoroutine 1 54 2 0 + 2 Once 43 0 0 0 goto 43 + 3 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 4 Null 0 9 0 0 r[9]=NULL + 5 Integer 0 6 0 0 r[6]=0; clear group by abort flag + 6 Null 0 7 0 0 r[7]=NULL; initialize group by comparison registers to NULL + 7 Gosub 13 40 0 0 ; go to clear accumulator subroutine + 8 OpenRead 1 4 0 k(4,B,B,B,B) 0 table=orders, root=4, iDb=0 + 9 OpenRead 2 9 0 k(2,B) 0 index=idx_orders_customer, root=9, iDb=0 + 10 Rewind 2 26 0 0 Rewind index idx_orders_customer + 11 DeferredSeek 2 1 0 0 + 12 Column 2 0 11 0 r[11]=idx_orders_customer.customer_id + 13 Column 1 3 12 0 r[12]=orders.total_amount + 14 RealAffinity 12 0 0 0 + 15 Compare 7 11 1 k(1, Binary) 0 r[7..7]==r[11..11] + 16 Jump 17 21 17 0 ; start new group if comparison is not equal + 17 Gosub 4 30 0 0 ; check if ended group had data, and output if so + 18 Move 11 7 1 0 r[7..7]=r[11..11] + 19 IfPos 6 43 0 0 r[6]>0 -> r[6]-=0, goto 43; check abort flag + 20 Gosub 13 40 0 0 ; goto clear accumulator subroutine + 21 AggStep 0 12 9 sum 0 accum=r[9] step(r[12]) + 22 If 5 24 0 0 if r[5] goto 24; don't emit group columns if continuing existing group + 23 Column 2 0 8 0 r[8]=idx_orders_customer.customer_id + 24 Integer 1 5 0 0 r[5]=1; indicate data in accumulator + 25 Next 2 11 0 1 + 26 Gosub 4 30 0 0 ; emit row for final group + 27 Goto 0 43 0 0 ; group by finished + 28 Integer 1 6 0 0 r[6]=1 + 29 Return 4 0 0 0 + 30 IfPos 5 32 0 0 r[5]>0 -> r[5]-=0, goto 32; output group by row subroutine start + 31 Return 4 0 0 0 + 32 AggFinal 0 9 0 sum 0 accum=r[9] + 33 Copy 8 2 0 0 r[2]=r[8] + 34 Copy 9 3 0 0 r[3]=r[9] + 35 Copy 2 15 1 0 r[15]=r[2] + 36 Sequence 0 17 0 0 + 37 MakeRecord 15 3 14 0 r[14]=mkrec(r[15..17]); for ephemeral_subquery_t3 + 38 IdxInsert 0 14 0 8 key=r[14] + 39 Return 4 0 0 0 + 40 Null 0 8 9 0 r[8..9]=NULL; clear accumulator subroutine start + 41 Integer 0 5 0 0 r[5]=0 + 42 Return 13 0 0 0 + 43 OpenRead 3 6 0 k(4,B,B,B,B) 0 table=customers, root=6, iDb=0 + 44 Rewind 3 53 0 0 Rewind table customers + 45 RowId 3 20 0 0 r[20]=customers.rowid + 46 SeekGE 0 52 20 0 key=[20..20] + 47 IdxGT 0 52 20 0 key=[20..20] + 48 Column 3 1 18 0 r[18]=customers.name + 49 Column 0 1 19 0 r[19]=ephemeral(ephemeral_subquery_t3).total_spent + 50 Yield 1 0 0 0 + 51 Next 0 47 0 0 + 52 Next 3 45 0 1 + 53 EndCoroutine 1 0 0 0 + 54 SorterOpen 4 1 0 k(1,-B) 0 cursor=4 + 55 InitCoroutine 1 0 2 0 + 56 Yield 1 64 0 0 + 57 Copy 19 25 0 0 r[25]=r[19] + 58 Le 25 26 63 Binary 0 if r[25]<=r[26] goto 63 + 59 Copy 19 27 0 0 r[27]=r[19] + 60 Copy 18 28 0 0 r[28]=r[18] + 61 MakeRecord 27 2 23 0 r[23]=mkrec(r[27..28]) + 62 SorterInsert 4 23 0 0 0 key=r[23] + 63 Goto 0 56 0 0 + 64 OpenPseudo 5 23 2 0 2 columns in r[23] + 65 SorterSort 4 71 0 0 + 66 SorterData 4 23 5 0 r[23]=data + 67 Column 5 1 21 0 r[21]=pseudo.column 1 + 68 Column 5 0 22 0 r[22]=pseudo.column 0 + 69 ResultRow 21 2 0 0 output=r[21..22] + 70 SorterNext 4 66 0 0 + 71 Halt 0 0 0 0 + 72 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 73 Integer 1000 26 0 0 r[26]=1000 + 74 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__exists-correlated.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__exists-correlated.snap index 0d27da51..547a6822 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__exists-correlated.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__exists-correlated.snap @@ -42,7 +42,7 @@ addr opcode p1 p2 p3 p4 p5 comment 15 Goto 0 18 0 0 ; semi-join: early out after first match 16 Next 2 7 0 0 17 Goto 0 18 0 0 ; semi-join: no match, skip outer row - 18 Next 0 5 0 0 + 18 Next 0 5 0 1 19 Halt 0 0 0 0 20 Transaction 0 1 11 0 iDb=0 tx_mode=Read 21 Integer 500 7 0 0 r[7]=500 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__exists-multiple-conditions.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__exists-multiple-conditions.snap index ddd133a0..620dc03a 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__exists-multiple-conditions.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__exists-multiple-conditions.snap @@ -56,7 +56,7 @@ addr opcode p1 p2 p3 p4 p5 comment 25 Column 0 3 4 0 r[4]=products.price 26 RealAffinity 4 0 0 0 27 ResultRow 2 3 0 0 output=r[2..4] - 28 Next 0 3 0 0 + 28 Next 0 3 0 1 29 Halt 0 0 0 0 30 Transaction 0 1 11 0 iDb=0 tx_mode=Read 31 String8 0 12 0 2024-01-01 0 r[12]='2024-01-01' diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-correlated-subquery-having.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-correlated-subquery-having.snap index 0aba015d..8bd9069d 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-correlated-subquery-having.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-correlated-subquery-having.snap @@ -19,68 +19,102 @@ info: --- QUERY PLAN |--SCAN grouped_values AS gv USING INDEX idx_grouped_values_grp -`--CORRELATED SCALAR SUBQUERY 1 - `--SEARCH gv2 USING INDEX idx_grouped_values_grp (grp=?) +`--SEARCH scalar_subquery_t2 USING INDEX ephemeral_subquery_t2 (correlation_key_0=?) LEFT-JOIN + `--SCAN grouped_values AS gv2 USING INDEX idx_grouped_values_grp BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 58 0 0 Start at 58 - 1 Null 0 9 0 0 r[9]=NULL - 2 Integer 0 6 0 0 r[6]=0; clear group by abort flag - 3 Null 0 7 0 0 r[7]=NULL; initialize group by comparison registers to NULL - 4 Gosub 13 54 0 0 ; go to clear accumulator subroutine - 5 OpenRead 0 2 0 k(3,B,B) 0 table=grouped_values, root=2, iDb=0 - 6 OpenRead 1 3 0 k(2,B) 0 index=idx_grouped_values_grp, root=3, iDb=0 - 7 Rewind 1 22 0 0 Rewind index idx_grouped_values_grp - 8 DeferredSeek 1 0 0 0 - 9 Column 1 0 11 0 r[11]=idx_grouped_values_grp.grp - 10 Column 0 1 12 0 r[12]=grouped_values.val - 11 Compare 7 11 1 k(1, Binary) 0 r[7..7]==r[11..11] - 12 Jump 13 17 13 0 ; start new group if comparison is not equal - 13 Gosub 4 26 0 0 ; check if ended group had data, and output if so - 14 Move 11 7 1 0 r[7..7]=r[11..11] - 15 IfPos 6 57 0 0 r[6]>0 -> r[6]-=0, goto 57; check abort flag - 16 Gosub 13 54 0 0 ; goto clear accumulator subroutine - 17 AggStep 0 12 9 sum 0 accum=r[9] step(r[12]) - 18 If 5 20 0 0 if r[5] goto 20; don't emit group columns if continuing existing group - 19 Column 1 0 8 0 r[8]=idx_grouped_values_grp.grp - 20 Integer 1 5 0 0 r[5]=1; indicate data in accumulator - 21 Next 1 8 0 0 - 22 Gosub 4 26 0 0 ; emit row for final group - 23 Goto 0 57 0 0 ; group by finished - 24 Integer 1 6 0 0 r[6]=1 - 25 Return 4 0 0 0 - 26 IfPos 5 28 0 0 r[5]>0 -> r[5]-=0, goto 28; output group by row subroutine start - 27 Return 4 0 0 0 - 28 AggFinal 0 9 0 sum 0 accum=r[9] - 29 BeginSubrtn 14 0 0 0 r[14]=NULL - 30 Null 0 1 0 0 r[1]=NULL - 31 Null 0 16 0 0 r[16]=NULL - 32 Integer 1 17 0 0 r[17]=1; LIMIT counter - 33 IfNot 17 44 0 0 if !r[17] goto 44 - 34 OpenRead 2 2 0 k(3,B,B) 0 table=grouped_values, root=2, iDb=0 - 35 OpenRead 3 3 0 k(2,B) 0 index=idx_grouped_values_grp, root=3, iDb=0 - 36 Copy 8 18 0 0 r[18]=r[8] - 37 IsNull 18 44 0 0 if (r[18]==NULL) goto 44 - 38 SeekGE 3 44 18 0 key=[18..18] - 39 IdxGT 3 44 18 0 key=[18..18] - 40 DeferredSeek 3 2 0 0 - 41 Column 2 1 19 0 r[19]=grouped_values.val - 42 AggStep 0 19 16 max(Binary) 0 accum=r[16] step(r[19]) - 43 Next 3 39 0 0 - 44 AggFinal 0 16 0 max 0 accum=r[16] - 45 Copy 16 15 0 0 r[15]=r[16] - 46 Copy 15 1 0 0 r[1]=r[15] - 47 Return 14 0 1 0 - 48 Copy 1 20 0 0 r[20]=r[1] - 49 IsNull 20 27 0 0 if (r[20]==NULL) goto 27 - 50 Copy 8 2 0 0 r[2]=r[8] - 51 Copy 9 3 0 0 r[3]=r[9] - 52 ResultRow 2 2 0 0 output=r[2..3] - 53 Return 4 0 0 0 - 54 Null 0 8 9 0 r[8..9]=NULL; clear accumulator subroutine start - 55 Integer 0 5 0 0 r[5]=0 - 56 Return 13 0 0 0 - 57 Halt 0 0 0 0 - 58 Transaction 0 1 2 0 iDb=0 tx_mode=Read - 59 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 92 0 0 Start at 92 + 1 Once 42 0 0 0 goto 42 + 2 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 3 Null 0 8 0 0 r[8]=NULL + 4 Integer 0 5 0 0 r[5]=0; clear group by abort flag + 5 Null 0 6 0 0 r[6]=NULL; initialize group by comparison registers to NULL + 6 Gosub 12 39 0 0 ; go to clear accumulator subroutine + 7 OpenRead 1 2 0 k(3,B,B) 0 table=grouped_values, root=2, iDb=0 + 8 OpenRead 2 3 0 k(2,B) 0 index=idx_grouped_values_grp, root=3, iDb=0 + 9 Rewind 2 24 0 0 Rewind index idx_grouped_values_grp + 10 DeferredSeek 2 1 0 0 + 11 Column 2 0 10 0 r[10]=idx_grouped_values_grp.grp + 12 Column 1 1 11 0 r[11]=grouped_values.val + 13 Compare 6 10 1 k(1, Binary) 0 r[6..6]==r[10..10] + 14 Jump 15 19 15 0 ; start new group if comparison is not equal + 15 Gosub 3 28 0 0 ; check if ended group had data, and output if so + 16 Move 10 6 1 0 r[6..6]=r[10..10] + 17 IfPos 5 42 0 0 r[5]>0 -> r[5]-=0, goto 42; check abort flag + 18 Gosub 12 39 0 0 ; goto clear accumulator subroutine + 19 AggStep 0 11 8 max(Binary) 0 accum=r[8] step(r[11]) + 20 If 4 22 0 0 if r[4] goto 22; don't emit group columns if continuing existing group + 21 Column 2 0 7 0 r[7]=idx_grouped_values_grp.grp + 22 Integer 1 4 0 0 r[4]=1; indicate data in accumulator + 23 Next 2 10 0 1 + 24 Gosub 3 28 0 0 ; emit row for final group + 25 Goto 0 42 0 0 ; group by finished + 26 Integer 1 5 0 0 r[5]=1 + 27 Return 3 0 0 0 + 28 IfPos 4 30 0 0 r[4]>0 -> r[4]-=0, goto 30; output group by row subroutine start + 29 Return 3 0 0 0 + 30 AggFinal 0 8 0 max 0 accum=r[8] + 31 Copy 8 1 0 0 r[1]=r[8] + 32 Copy 7 2 0 0 r[2]=r[7] + 33 Copy 2 14 0 0 r[14]=r[2] + 34 Copy 1 15 0 0 r[15]=r[1] + 35 Sequence 0 16 0 0 + 36 MakeRecord 14 3 13 0 r[13]=mkrec(r[14..16]); for ephemeral_subquery_t2 + 37 IdxInsert 0 13 0 8 key=r[13] + 38 Return 3 0 0 0 + 39 Null 0 7 8 0 r[7..8]=NULL; clear accumulator subroutine start + 40 Integer 0 4 0 0 r[4]=0 + 41 Return 12 0 0 0 + 42 Null 0 25 0 0 r[25]=NULL + 43 Integer 0 21 0 0 r[21]=0; clear group by abort flag + 44 Null 0 22 0 0 r[22]=NULL; initialize group by comparison registers to NULL + 45 Gosub 30 88 0 0 ; go to clear accumulator subroutine + 46 OpenRead 3 2 0 k(3,B,B) 0 table=grouped_values, root=2, iDb=0 + 47 OpenRead 4 3 0 k(2,B) 0 index=idx_grouped_values_grp, root=3, iDb=0 + 48 Rewind 4 75 0 0 Rewind index idx_grouped_values_grp + 49 DeferredSeek 4 3 0 0 + 50 Integer 0 31 0 0 r[31]=0 + 51 Column 4 0 32 0 r[32]=idx_grouped_values_grp.grp + 52 IsNull 32 71 0 0 if (r[32]==NULL) goto 71 + 53 SeekGE 0 71 32 0 key=[32..32] + 54 IdxGT 0 71 32 0 key=[32..32] + 55 Integer 1 31 0 0 r[31]=1 + 56 Column 4 0 27 0 r[27]=idx_grouped_values_grp.grp + 57 Column 0 1 28 0 r[28]=ephemeral(ephemeral_subquery_t2).MAX(gv2.val) + 58 Column 3 1 29 0 r[29]=grouped_values.val + 59 Compare 22 27 1 k(1, Binary) 0 r[22..22]==r[27..27] + 60 Jump 61 65 61 0 ; start new group if comparison is not equal + 61 Gosub 19 79 0 0 ; check if ended group had data, and output if so + 62 Move 27 22 1 0 r[22..22]=r[27..27] + 63 IfPos 21 91 0 0 r[21]>0 -> r[21]-=0, goto 91; check abort flag + 64 Gosub 30 88 0 0 ; goto clear accumulator subroutine + 65 AggStep 0 29 25 sum 0 accum=r[25] step(r[29]) + 66 If 20 69 0 0 if r[20] goto 69; don't emit group columns if continuing existing group + 67 Column 4 0 23 0 r[23]=idx_grouped_values_grp.grp + 68 Column 0 1 24 0 r[24]=ephemeral(ephemeral_subquery_t2).MAX(gv2.val) + 69 Integer 1 20 0 0 r[20]=1; indicate data in accumulator + 70 Next 0 54 0 0 + 71 IfPos 31 74 0 0 r[31]>0 -> r[31]-=0, goto 74 + 72 NullRow 0 0 0 0 Set cursor 0 to a (pseudo) NULL row + 73 Goto 0 55 0 0 + 74 Next 4 49 0 1 + 75 Gosub 19 79 0 0 ; emit row for final group + 76 Goto 0 91 0 0 ; group by finished + 77 Integer 1 21 0 0 r[21]=1 + 78 Return 19 0 0 0 + 79 IfPos 20 81 0 0 r[20]>0 -> r[20]-=0, goto 81; output group by row subroutine start + 80 Return 19 0 0 0 + 81 AggFinal 0 25 0 sum 0 accum=r[25] + 82 Copy 24 33 0 0 r[33]=r[24] + 83 IsNull 33 80 0 0 if (r[33]==NULL) goto 80 + 84 Copy 23 17 0 0 r[17]=r[23] + 85 Copy 25 18 0 0 r[18]=r[25] + 86 ResultRow 17 2 0 0 output=r[17..18] + 87 Return 19 0 0 0 + 88 Null 0 23 25 0 r[23..25]=NULL; clear accumulator subroutine start + 89 Integer 0 20 0 0 r[20]=0 + 90 Return 30 0 0 0 + 91 Halt 0 0 0 0 + 92 Transaction 0 1 2 0 iDb=0 tx_mode=Read + 93 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-correlated-subquery-order-by.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-correlated-subquery-order-by.snap index 88444a2d..6c8f8653 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-correlated-subquery-order-by.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-correlated-subquery-order-by.snap @@ -19,78 +19,111 @@ info: --- QUERY PLAN |--SCAN grouped_values AS gv USING INDEX idx_grouped_values_grp -|--CORRELATED SCALAR SUBQUERY 1 -| `--SEARCH gv2 USING INDEX idx_grouped_values_grp (grp=?) +|--SEARCH scalar_subquery_t2 USING INDEX ephemeral_subquery_t2 (correlation_key_0=?) LEFT-JOIN +| `--SCAN grouped_values AS gv2 USING INDEX idx_grouped_values_grp `--USE SORTER FOR ORDER BY BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 67 0 0 Start at 67 - 1 SorterOpen 0 2 0 k(2,-B,Binary) 0 cursor=0 - 2 Null 0 10 0 0 r[10]=NULL - 3 Integer 0 7 0 0 r[7]=0; clear group by abort flag - 4 Null 0 8 0 0 r[8]=NULL; initialize group by comparison registers to NULL - 5 Gosub 14 56 0 0 ; go to clear accumulator subroutine - 6 OpenRead 1 2 0 k(3,B,B) 0 table=grouped_values, root=2, iDb=0 - 7 OpenRead 2 3 0 k(2,B) 0 index=idx_grouped_values_grp, root=3, iDb=0 - 8 Rewind 2 23 0 0 Rewind index idx_grouped_values_grp - 9 DeferredSeek 2 1 0 0 - 10 Column 2 0 12 0 r[12]=idx_grouped_values_grp.grp - 11 Column 1 1 13 0 r[13]=grouped_values.val - 12 Compare 8 12 1 k(1, Binary) 0 r[8..8]==r[12..12] - 13 Jump 14 18 14 0 ; start new group if comparison is not equal - 14 Gosub 5 27 0 0 ; check if ended group had data, and output if so - 15 Move 12 8 1 0 r[8..8]=r[12..12] - 16 IfPos 7 59 0 0 r[7]>0 -> r[7]-=0, goto 59; check abort flag - 17 Gosub 14 56 0 0 ; goto clear accumulator subroutine - 18 AggStep 0 13 10 sum 0 accum=r[10] step(r[13]) - 19 If 6 21 0 0 if r[6] goto 21; don't emit group columns if continuing existing group - 20 Column 2 0 9 0 r[9]=idx_grouped_values_grp.grp - 21 Integer 1 6 0 0 r[6]=1; indicate data in accumulator - 22 Next 2 9 0 0 - 23 Gosub 5 27 0 0 ; emit row for final group - 24 Goto 0 59 0 0 ; group by finished - 25 Integer 1 7 0 0 r[7]=1 - 26 Return 5 0 0 0 - 27 IfPos 6 29 0 0 r[6]>0 -> r[6]-=0, goto 29; output group by row subroutine start - 28 Return 5 0 0 0 - 29 AggFinal 0 10 0 sum 0 accum=r[10] - 30 BeginSubrtn 15 0 0 0 r[15]=NULL - 31 Null 0 1 0 0 r[1]=NULL - 32 Null 0 17 0 0 r[17]=NULL - 33 Integer 1 18 0 0 r[18]=1; LIMIT counter - 34 IfNot 18 45 0 0 if !r[18] goto 45 - 35 OpenRead 3 2 0 k(3,B,B) 0 table=grouped_values, root=2, iDb=0 - 36 OpenRead 4 3 0 k(2,B) 0 index=idx_grouped_values_grp, root=3, iDb=0 - 37 Copy 9 19 0 0 r[19]=r[9] - 38 IsNull 19 45 0 0 if (r[19]==NULL) goto 45 - 39 SeekGE 4 45 19 0 key=[19..19] - 40 IdxGT 4 45 19 0 key=[19..19] - 41 DeferredSeek 4 3 0 0 - 42 Column 3 1 20 0 r[20]=grouped_values.val - 43 AggStep 0 20 17 max(Binary) 0 accum=r[17] step(r[20]) - 44 Next 4 40 0 0 - 45 AggFinal 0 17 0 max 0 accum=r[17] - 46 Copy 17 16 0 0 r[16]=r[17] - 47 Copy 16 1 0 0 r[1]=r[16] - 48 Return 15 0 1 0 - 49 Copy 1 21 0 0 r[21]=r[1] - 50 Sequence 0 22 0 0 - 51 Copy 9 23 0 0 r[23]=r[9] - 52 Copy 10 24 0 0 r[24]=r[10] - 53 MakeRecord 21 4 4 0 r[4]=mkrec(r[21..24]) - 54 SorterInsert 0 4 0 0 0 key=r[4] - 55 Return 5 0 0 0 - 56 Null 0 9 10 0 r[9..10]=NULL; clear accumulator subroutine start - 57 Integer 0 6 0 0 r[6]=0 - 58 Return 14 0 0 0 - 59 OpenPseudo 5 4 4 0 4 columns in r[4] - 60 SorterSort 0 66 0 0 - 61 SorterData 0 4 5 0 r[4]=data - 62 Column 5 2 2 0 r[2]=pseudo.column 2 - 63 Column 5 3 3 0 r[3]=pseudo.column 3 - 64 ResultRow 2 2 0 0 output=r[2..3] - 65 SorterNext 0 61 0 0 - 66 Halt 0 0 0 0 - 67 Transaction 0 1 2 0 iDb=0 tx_mode=Read - 68 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 100 0 0 Start at 100 + 1 Once 42 0 0 0 goto 42 + 2 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 3 Null 0 8 0 0 r[8]=NULL + 4 Integer 0 5 0 0 r[5]=0; clear group by abort flag + 5 Null 0 6 0 0 r[6]=NULL; initialize group by comparison registers to NULL + 6 Gosub 12 39 0 0 ; go to clear accumulator subroutine + 7 OpenRead 1 2 0 k(3,B,B) 0 table=grouped_values, root=2, iDb=0 + 8 OpenRead 2 3 0 k(2,B) 0 index=idx_grouped_values_grp, root=3, iDb=0 + 9 Rewind 2 24 0 0 Rewind index idx_grouped_values_grp + 10 DeferredSeek 2 1 0 0 + 11 Column 2 0 10 0 r[10]=idx_grouped_values_grp.grp + 12 Column 1 1 11 0 r[11]=grouped_values.val + 13 Compare 6 10 1 k(1, Binary) 0 r[6..6]==r[10..10] + 14 Jump 15 19 15 0 ; start new group if comparison is not equal + 15 Gosub 3 28 0 0 ; check if ended group had data, and output if so + 16 Move 10 6 1 0 r[6..6]=r[10..10] + 17 IfPos 5 42 0 0 r[5]>0 -> r[5]-=0, goto 42; check abort flag + 18 Gosub 12 39 0 0 ; goto clear accumulator subroutine + 19 AggStep 0 11 8 max(Binary) 0 accum=r[8] step(r[11]) + 20 If 4 22 0 0 if r[4] goto 22; don't emit group columns if continuing existing group + 21 Column 2 0 7 0 r[7]=idx_grouped_values_grp.grp + 22 Integer 1 4 0 0 r[4]=1; indicate data in accumulator + 23 Next 2 10 0 1 + 24 Gosub 3 28 0 0 ; emit row for final group + 25 Goto 0 42 0 0 ; group by finished + 26 Integer 1 5 0 0 r[5]=1 + 27 Return 3 0 0 0 + 28 IfPos 4 30 0 0 r[4]>0 -> r[4]-=0, goto 30; output group by row subroutine start + 29 Return 3 0 0 0 + 30 AggFinal 0 8 0 max 0 accum=r[8] + 31 Copy 8 1 0 0 r[1]=r[8] + 32 Copy 7 2 0 0 r[2]=r[7] + 33 Copy 2 14 0 0 r[14]=r[2] + 34 Copy 1 15 0 0 r[15]=r[1] + 35 Sequence 0 16 0 0 + 36 MakeRecord 14 3 13 0 r[13]=mkrec(r[14..16]); for ephemeral_subquery_t2 + 37 IdxInsert 0 13 0 8 key=r[13] + 38 Return 3 0 0 0 + 39 Null 0 7 8 0 r[7..8]=NULL; clear accumulator subroutine start + 40 Integer 0 4 0 0 r[4]=0 + 41 Return 12 0 0 0 + 42 SorterOpen 3 2 0 k(2,-B,Binary) 0 cursor=3 + 43 Null 0 26 0 0 r[26]=NULL + 44 Integer 0 22 0 0 r[22]=0; clear group by abort flag + 45 Null 0 23 0 0 r[23]=NULL; initialize group by comparison registers to NULL + 46 Gosub 31 90 0 0 ; go to clear accumulator subroutine + 47 OpenRead 4 2 0 k(3,B,B) 0 table=grouped_values, root=2, iDb=0 + 48 OpenRead 5 3 0 k(2,B) 0 index=idx_grouped_values_grp, root=3, iDb=0 + 49 Rewind 5 76 0 0 Rewind index idx_grouped_values_grp + 50 DeferredSeek 5 4 0 0 + 51 Integer 0 32 0 0 r[32]=0 + 52 Column 5 0 33 0 r[33]=idx_grouped_values_grp.grp + 53 IsNull 33 72 0 0 if (r[33]==NULL) goto 72 + 54 SeekGE 0 72 33 0 key=[33..33] + 55 IdxGT 0 72 33 0 key=[33..33] + 56 Integer 1 32 0 0 r[32]=1 + 57 Column 5 0 28 0 r[28]=idx_grouped_values_grp.grp + 58 Column 0 1 29 0 r[29]=ephemeral(ephemeral_subquery_t2).MAX(gv2.val) + 59 Column 4 1 30 0 r[30]=grouped_values.val + 60 Compare 23 28 1 k(1, Binary) 0 r[23..23]==r[28..28] + 61 Jump 62 66 62 0 ; start new group if comparison is not equal + 62 Gosub 20 80 0 0 ; check if ended group had data, and output if so + 63 Move 28 23 1 0 r[23..23]=r[28..28] + 64 IfPos 22 93 0 0 r[22]>0 -> r[22]-=0, goto 93; check abort flag + 65 Gosub 31 90 0 0 ; goto clear accumulator subroutine + 66 AggStep 0 30 26 sum 0 accum=r[26] step(r[30]) + 67 If 21 70 0 0 if r[21] goto 70; don't emit group columns if continuing existing group + 68 Column 5 0 24 0 r[24]=idx_grouped_values_grp.grp + 69 Column 0 1 25 0 r[25]=ephemeral(ephemeral_subquery_t2).MAX(gv2.val) + 70 Integer 1 21 0 0 r[21]=1; indicate data in accumulator + 71 Next 0 55 0 0 + 72 IfPos 32 75 0 0 r[32]>0 -> r[32]-=0, goto 75 + 73 NullRow 0 0 0 0 Set cursor 0 to a (pseudo) NULL row + 74 Goto 0 56 0 0 + 75 Next 5 50 0 1 + 76 Gosub 20 80 0 0 ; emit row for final group + 77 Goto 0 93 0 0 ; group by finished + 78 Integer 1 22 0 0 r[22]=1 + 79 Return 20 0 0 0 + 80 IfPos 21 82 0 0 r[21]>0 -> r[21]-=0, goto 82; output group by row subroutine start + 81 Return 20 0 0 0 + 82 AggFinal 0 26 0 sum 0 accum=r[26] + 83 Copy 25 34 0 0 r[34]=r[25] + 84 Sequence 3 35 0 0 + 85 Copy 24 36 0 0 r[36]=r[24] + 86 Copy 26 37 0 0 r[37]=r[26] + 87 MakeRecord 34 4 19 0 r[19]=mkrec(r[34..37]) + 88 SorterInsert 3 19 0 0 0 key=r[19] + 89 Return 20 0 0 0 + 90 Null 0 24 26 0 r[24..26]=NULL; clear accumulator subroutine start + 91 Integer 0 21 0 0 r[21]=0 + 92 Return 31 0 0 0 + 93 OpenPseudo 6 19 4 0 4 columns in r[19] + 94 SorterSort 3 99 0 0 + 95 SorterData 3 19 6 0 r[19]=data + 96 ColumnRange 6 2 17 2 0 r[17..18]=pseudo.column 2..column 3 + 97 ResultRow 17 2 0 0 output=r[17..18] + 98 SorterNext 3 95 0 0 + 99 Halt 0 0 0 0 + 100 Transaction 0 1 2 0 iDb=0 tx_mode=Read + 101 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-subquery-builds-once.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-subquery-builds-once.snap new file mode 100644 index 00000000..ae860a2e --- /dev/null +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__grouped-subquery-builds-once.snap @@ -0,0 +1,125 @@ +--- +source: subqueries.sqltest +expression: |- + SELECT top.t, + (SELECT count(*) + FROM repeat_middle middle_rows + WHERE middle_rows.tk = top.t + AND middle_rows.sk IN ( + SELECT inner_rows.k + FROM repeat_inner inner_rows + GROUP BY inner_rows.k + HAVING avg(inner_rows.x) > 0 + )) + FROM repeat_top top; +info: + statement_type: SELECT + tables: + - repeat_inner + - repeat_middle + - repeat_top + setup_blocks: + - repeated_grouped_subquery + database: ':memory:' +--- +QUERY PLAN +|--SCAN repeat_top AS top +`--CORRELATED SCALAR SUBQUERY 1 + |--LIST SUBQUERY 2 + | |--SCAN repeat_inner AS inner_rows + | `--USE SORTER FOR GROUP BY + `--SCAN repeat_middle AS middle_rows + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 88 0 0 Start at 88 + 1 OpenRead 1 2 0 k(2,B) 0 table=repeat_top, root=2, iDb=0 + 2 Rewind 1 87 0 0 Rewind table repeat_top + 3 BeginSubrtn 4 0 0 0 r[4]=NULL + 4 Null 0 1 0 0 r[1]=NULL + 5 Once 51 0 0 0 goto 51 + 6 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 7 Null 0 11 0 0 r[11]=NULL + 8 SorterOpen 2 2 0 k(1,B) 0 cursor=2 + 9 Integer 0 8 0 0 r[8]=0; clear group by abort flag + 10 Null 0 9 0 0 r[9]=NULL; initialize group by comparison registers to NULL + 11 Gosub 15 48 0 0 ; go to clear accumulator subroutine + 12 OpenRead 4 4 0 k(3,B,B) 0 table=repeat_inner, root=4, iDb=0 + 13 Rewind 4 18 0 0 Rewind table repeat_inner + 14 ColumnRange 4 0 13 2 0 r[13..14]=repeat_inner.k..x + 15 MakeRecord 13 2 12 0 r[12]=mkrec(r[13..14]) + 16 SorterInsert 2 12 0 0 0 key=r[12] + 17 Next 4 14 0 1 + 18 OpenPseudo 3 12 2 0 2 columns in r[12] + 19 SorterSort 2 34 0 0 + 20 SorterData 2 12 3 0 r[12]=data + 21 Column 3 0 16 0 r[16]=pseudo.column 0 + 22 Compare 9 16 1 k(1, Binary) 0 r[9..9]==r[16..16] + 23 Jump 24 28 24 0 ; start new group if comparison is not equal + 24 Gosub 6 38 0 0 ; check if ended group had data, and output if so + 25 Move 16 9 1 0 r[9..9]=r[16..16] + 26 IfPos 8 51 0 0 r[8]>0 -> r[8]-=0, goto 51; check abort flag + 27 Gosub 15 48 0 0 ; goto clear accumulator subroutine + 28 Column 3 1 17 0 r[17]=pseudo.column 1 + 29 AggStep 0 17 11 avg 0 accum=r[11] step(r[17]) + 30 If 7 32 0 0 if r[7] goto 32; don't emit group columns if continuing existing group + 31 Column 3 0 10 0 r[10]=pseudo.column 0 + 32 Integer 1 7 0 0 r[7]=1; indicate data in accumulator + 33 SorterNext 2 20 0 0 + 34 Gosub 6 38 0 0 ; emit row for final group + 35 Goto 0 51 0 0 ; group by finished + 36 Integer 1 8 0 0 r[8]=1 + 37 Return 6 0 0 0 + 38 IfPos 7 40 0 0 r[7]>0 -> r[7]-=0, goto 40; output group by row subroutine start + 39 Return 6 0 0 0 + 40 AggFinal 0 11 0 avg 0 accum=r[11] + 41 Copy 11 19 0 0 r[19]=r[11] + 42 Integer 0 20 0 0 r[20]=0 + 43 Le 19 20 39 0 if r[19]<=r[20] goto 39 + 44 Copy 10 5 0 0 r[5]=r[10] + 45 MakeRecord 5 1 21 0 r[21]=mkrec(r[5..5]); for ephemeral_index_where_sub_t4 + 46 IdxInsert 0 21 0 8 key=r[21] + 47 Return 6 0 0 0 + 48 Null 0 10 11 0 r[10..11]=NULL; clear accumulator subroutine start + 49 Integer 0 7 0 0 r[7]=0 + 50 Return 15 0 0 0 + 51 Null 0 23 0 0 r[23]=NULL + 52 Integer 1 24 0 0 r[24]=1; LIMIT counter + 53 IfNot 24 79 0 0 if !r[24] goto 79 + 54 OpenRead 5 3 0 k(3,B,B) 0 table=repeat_middle, root=3, iDb=0 + 55 Rewind 5 79 0 0 Rewind table repeat_middle + 56 Column 5 1 26 0 r[26]=repeat_middle.tk + 57 Column 1 0 27 0 r[27]=repeat_top.t + 58 Ne 26 27 78 Binary 0 if r[26]!=r[27] goto 78 + 59 Integer 0 28 0 0 r[28]=0 + 60 Column 5 0 29 0 r[29]=repeat_middle.sk + 61 IsNull 29 65 0 0 if (r[29]==NULL) goto 65 + 62 Affinity 29 1 0 0 r[29..30] = C + 63 NotFound 0 65 29 0 if not found goto 65 + 64 Goto 0 71 0 0 + 65 Rewind 0 73 0 0 Rewind ephemeral(ephemeral_index_where_sub_t4) + 66 Column 0 0 30 0 r[30]=ephemeral(ephemeral_index_where_sub_t4).k + 67 Ne 29 30 69 Binary 0 if r[29]!=r[30] goto 69 + 68 Goto 0 75 0 0 + 69 Next 0 66 0 0 + 70 Goto 0 73 0 0 + 71 Integer 1 28 0 0 r[28]=1 + 72 Goto 0 76 0 0 + 73 Integer 0 28 0 0 r[28]=0 + 74 Goto 0 76 0 0 + 75 Null 0 28 0 0 r[28]=NULL + 76 IfNot 28 78 1 0 if !r[28] goto 78 + 77 AggStep 0 31 23 count 0 accum=r[23] step(r[31]) + 78 Next 5 56 0 1 + 79 AggFinal 0 23 0 count 0 accum=r[23] + 80 Copy 23 22 0 0 r[22]=r[23] + 81 Copy 22 1 0 0 r[1]=r[22] + 82 Return 4 0 1 0 + 83 Column 1 0 2 0 r[2]=repeat_top.t + 84 Copy 1 3 0 0 r[3]=r[1] + 85 ResultRow 2 2 0 0 output=r[2..3] + 86 Next 1 3 0 1 + 87 Halt 0 0 0 0 + 88 Transaction 0 1 3 0 iDb=0 tx_mode=Read + 89 Integer 1 31 0 0 r[31]=1 + 90 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__in-subquery-aggregation.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__in-subquery-aggregation.snap index 9d727bda..f1d8d74b 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__in-subquery-aggregation.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__in-subquery-aggregation.snap @@ -49,7 +49,7 @@ addr opcode p1 p2 p3 p4 p5 comment 21 If 3 23 0 0 if r[3] goto 23; don't emit group columns if continuing existing group 22 Column 2 0 6 0 r[6]=idx_orders_customer.customer_id 23 Integer 1 3 0 0 r[3]=1; indicate data in accumulator - 24 Next 2 10 0 0 + 24 Next 2 10 0 1 25 Gosub 2 29 0 0 ; emit row for final group 26 Goto 0 42 0 0 ; group by finished 27 Integer 1 4 0 0 r[4]=1 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__not-exists-never-ordered.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__not-exists-never-ordered.snap index bc4d8a17..c86ce757 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__not-exists-never-ordered.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__not-exists-never-ordered.snap @@ -19,4 +19,4 @@ info: --- QUERY PLAN |--SCAN products AS p -`--SEARCH oi USING INDEX idx_order_items_product (product_id=?) +`--SEARCH oi USING COVERING INDEX idx_order_items_product (product_id=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__not-exists-no-orders.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__not-exists-no-orders.snap index faec5d5b..72bf11ca 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__not-exists-no-orders.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__not-exists-no-orders.snap @@ -19,4 +19,4 @@ info: --- QUERY PLAN |--SCAN customers AS c -`--SEARCH o USING INDEX idx_orders_customer (customer_id=?) +`--SEARCH o USING COVERING INDEX idx_orders_customer (customer_id=?) diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__scalar-subquery-cross-table.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__scalar-subquery-cross-table.snap index 7237fdf4..79edba00 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__scalar-subquery-cross-table.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__scalar-subquery-cross-table.snap @@ -16,34 +16,68 @@ info: --- QUERY PLAN |--SCAN customers AS c -`--CORRELATED SCALAR SUBQUERY 1 - `--SEARCH orders USING INDEX idx_orders_customer (customer_id=?) +`--SEARCH scalar_subquery_t2 USING INDEX ephemeral_subquery_t2 (correlation_key_0=?) LEFT-JOIN + `--SCAN orders USING COVERING INDEX idx_orders_customer BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 23 0 0 Start at 23 - 1 OpenRead 0 6 0 k(4,B,B,B,B) 0 table=customers, root=6, iDb=0 - 2 Rewind 0 22 0 0 Rewind table customers - 3 BeginSubrtn 4 0 0 0 r[4]=NULL - 4 Null 0 1 0 0 r[1]=NULL - 5 Null 0 6 0 0 r[6]=NULL - 6 Integer 1 7 0 0 r[7]=1; LIMIT counter - 7 IfNot 7 14 0 0 if !r[7] goto 14 - 8 OpenRead 1 9 0 k(2,B) 0 index=idx_orders_customer, root=9, iDb=0 - 9 RowId 0 8 0 0 r[8]=customers.rowid - 10 SeekGE 1 14 8 0 key=[8..8] - 11 IdxGT 1 14 8 0 key=[8..8] - 12 AggStep 0 9 6 count 0 accum=r[6] step(r[9]) - 13 Next 1 11 0 0 - 14 AggFinal 0 6 0 count 0 accum=r[6] - 15 Copy 6 5 0 0 r[5]=r[6] - 16 Copy 5 1 0 0 r[1]=r[5] - 17 Return 4 0 1 0 - 18 Column 0 1 2 0 r[2]=customers.name - 19 Copy 1 3 0 0 r[3]=r[1] - 20 ResultRow 2 2 0 0 output=r[2..3] - 21 Next 0 3 0 0 - 22 Halt 0 0 0 0 - 23 Transaction 0 1 11 0 iDb=0 tx_mode=Read - 24 Integer 1 9 0 0 r[9]=1 - 25 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 57 0 0 Start at 57 + 1 Once 39 0 0 0 goto 39 + 2 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 3 Null 0 8 0 0 r[8]=NULL + 4 Integer 0 5 0 0 r[5]=0; clear group by abort flag + 5 Null 0 6 0 0 r[6]=NULL; initialize group by comparison registers to NULL + 6 Gosub 11 36 0 0 ; go to clear accumulator subroutine + 7 OpenRead 1 9 0 k(2,B) 0 index=idx_orders_customer, root=9, iDb=0 + 8 Rewind 1 21 0 0 Rewind index idx_orders_customer + 9 Column 1 0 10 0 r[10]=idx_orders_customer.customer_id + 10 Compare 6 10 1 k(1, Binary) 0 r[6..6]==r[10..10] + 11 Jump 12 16 12 0 ; start new group if comparison is not equal + 12 Gosub 3 25 0 0 ; check if ended group had data, and output if so + 13 Move 10 6 1 0 r[6..6]=r[10..10] + 14 IfPos 5 39 0 0 r[5]>0 -> r[5]-=0, goto 39; check abort flag + 15 Gosub 11 36 0 0 ; goto clear accumulator subroutine + 16 AggStep 0 12 8 count 0 accum=r[8] step(r[12]) + 17 If 4 19 0 0 if r[4] goto 19; don't emit group columns if continuing existing group + 18 Column 1 0 7 0 r[7]=idx_orders_customer.customer_id + 19 Integer 1 4 0 0 r[4]=1; indicate data in accumulator + 20 Next 1 9 0 1 + 21 Gosub 3 25 0 0 ; emit row for final group + 22 Goto 0 39 0 0 ; group by finished + 23 Integer 1 5 0 0 r[5]=1 + 24 Return 3 0 0 0 + 25 IfPos 4 27 0 0 r[4]>0 -> r[4]-=0, goto 27; output group by row subroutine start + 26 Return 3 0 0 0 + 27 AggFinal 0 8 0 count 0 accum=r[8] + 28 Copy 8 1 0 0 r[1]=r[8] + 29 Copy 7 2 0 0 r[2]=r[7] + 30 Copy 2 14 0 0 r[14]=r[2] + 31 Copy 1 15 0 0 r[15]=r[1] + 32 Sequence 0 16 0 0 + 33 MakeRecord 14 3 13 0 r[13]=mkrec(r[14..16]); for ephemeral_subquery_t2 + 34 IdxInsert 0 13 0 8 key=r[13] + 35 Return 3 0 0 0 + 36 Null 0 7 8 0 r[7..8]=NULL; clear accumulator subroutine start + 37 Integer 0 4 0 0 r[4]=0 + 38 Return 11 0 0 0 + 39 OpenRead 2 6 0 k(4,B,B,B,B) 0 table=customers, root=6, iDb=0 + 40 Rewind 2 56 0 0 Rewind table customers + 41 Integer 0 19 0 0 r[19]=0 + 42 RowId 2 20 0 0 r[20]=customers.rowid + 43 SeekGE 0 52 20 0 key=[20..20] + 44 IdxGT 0 52 20 0 key=[20..20] + 45 Integer 1 19 0 0 r[19]=1 + 46 Column 2 1 17 0 r[17]=customers.name + 47 Column 0 1 18 0 r[18]=ephemeral(ephemeral_subquery_t2).count(*) + 48 NotNull 18 50 0 0 r[18]!=NULL -> goto 50 + 49 Integer 0 18 0 0 r[18]=0 + 50 ResultRow 17 2 0 0 output=r[17..18] + 51 Next 0 44 0 0 + 52 IfPos 19 55 0 0 r[19]>0 -> r[19]-=0, goto 55 + 53 NullRow 0 0 0 0 Set cursor 0 to a (pseudo) NULL row + 54 Goto 0 45 0 0 + 55 Next 2 41 0 1 + 56 Halt 0 0 0 0 + 57 Transaction 0 1 11 0 iDb=0 tx_mode=Read + 58 Integer 1 12 0 0 r[12]=1 + 59 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__scalar-subquery-max-price.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__scalar-subquery-max-price.snap index 2470c1aa..0a976f73 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__scalar-subquery-max-price.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__scalar-subquery-max-price.snap @@ -34,7 +34,7 @@ addr opcode p1 p2 p3 p4 p5 comment 9 Column 0 3 6 0 r[6]=products.price 10 RealAffinity 6 0 0 0 11 AggStep 0 6 4 max(Binary) 0 accum=r[4] step(r[6]) - 12 Next 0 9 0 0 + 12 Next 0 9 0 1 13 AggFinal 0 4 0 max 0 accum=r[4] 14 Copy 4 3 0 0 r[3]=r[4] 15 Copy 3 1 0 0 r[1]=r[3] @@ -49,7 +49,7 @@ addr opcode p1 p2 p3 p4 p5 comment 24 RealAffinity 8 0 0 0 25 Copy 1 9 0 0 r[9]=r[1] 26 ResultRow 7 3 0 0 output=r[7..9] - 27 Next 1 19 0 0 + 27 Next 1 19 0 1 28 Halt 0 0 0 0 29 Transaction 0 1 11 0 iDb=0 tx_mode=Read 30 Integer 100 12 0 0 r[12]=100 diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__subquery-within-cte.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__subquery-within-cte.snap index bcde1ff6..869132d8 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__subquery-within-cte.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__subquery-within-cte.snap @@ -42,7 +42,7 @@ addr opcode p1 p2 p3 p4 p5 comment 8 RowId 1 2 0 0 r[2]=categories.rowid 9 MakeRecord 2 1 4 0 r[4]=mkrec(r[2..2]); for ephemeral_index_where_sub_t2 10 IdxInsert 0 4 0 8 key=r[4] - 11 Next 1 6 0 0 + 11 Next 1 6 0 1 12 OpenRead 2 2 0 k(5,B,B,B,B,B) 0 table=products, root=2, iDb=0 13 OpenRead 3 8 0 k(2,B) 0 index=idx_products_category, root=8, iDb=0 14 NullRow 0 0 0 0 Set cursor 0 to a (pseudo) NULL row diff --git a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__update-returning-target-selfread.snap b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__update-returning-target-selfread.snap index e9ed573c..efa1912f 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__update-returning-target-selfread.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/subqueries/snapshots/subqueries__update-returning-target-selfread.snap @@ -20,7 +20,7 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 50 0 0 Start at 50 + 0 Init 0 49 0 0 Start at 49 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true 2 OpenWrite 1 2 0 0 root=2; iDb=0 3 Rewind 1 43 0 0 Rewind table t @@ -49,7 +49,7 @@ addr opcode p1 p2 p3 p4 p5 comment 26 Column 2 2 19 0 r[19]=t.val 27 Lt 19 20 29 Binary 0 if r[19]r[36] goto 18 - 10 Column 2 8 27 0 r[27]=lineitem.L_RETURNFLAG - 11 Column 2 9 28 0 r[28]=lineitem.L_LINESTATUS - 12 Column 2 4 29 0 r[29]=lineitem.L_QUANTITY - 13 Column 2 5 30 0 r[30]=lineitem.L_EXTENDEDPRICE - 14 Column 2 6 31 0 r[31]=lineitem.L_DISCOUNT - 15 Column 2 7 32 0 r[32]=lineitem.L_TAX - 16 MakeRecord 27 6 26 0 r[26]=mkrec(r[27..32]) - 17 SorterInsert 0 26 0 0 0 key=r[26] - 18 Next 2 8 0 0 - 19 OpenPseudo 1 26 6 0 6 columns in r[26] - 20 SorterSort 0 58 0 0 - 21 SorterData 0 26 1 0 r[26]=data - 22 Column 1 0 37 0 r[37]=pseudo.column 0 - 23 Column 1 1 38 0 r[38]=pseudo.column 1 - 24 Compare 14 37 2 k(2, Binary, Binary) 0 r[14..15]==r[37..38] - 25 Jump 26 30 26 0 ; start new group if comparison is not equal - 26 Gosub 11 62 0 0 ; check if ended group had data, and output if so - 27 Move 37 14 2 0 r[14..15]=r[37..38] - 28 IfPos 13 87 0 0 r[13]>0 -> r[13]-=0, goto 87; check abort flag - 29 Gosub 33 84 0 0 ; goto clear accumulator subroutine - 30 Column 1 2 39 0 r[39]=pseudo.column 2 - 31 Column 1 3 40 0 r[40]=pseudo.column 3 - 32 Column 1 4 41 0 r[41]=pseudo.column 4 - 33 Column 1 5 42 0 r[42]=pseudo.column 5 - 34 AggStep 0 39 18 sum 0 accum=r[18] step(r[39]) - 35 AggStep 0 40 19 sum 0 accum=r[19] step(r[40]) - 36 Copy 40 44 0 0 r[44]=r[40] - 37 Copy 41 47 0 0 r[47]=r[41] - 38 Subtract 46 47 45 0 r[45]=r[46]-r[47] - 39 Multiply 44 45 43 0 r[43]=r[44]*r[45] - 40 AggStep 0 43 20 sum 0 accum=r[20] step(r[43]) - 41 Copy 40 51 0 0 r[51]=r[40] - 42 Copy 41 54 0 0 r[54]=r[41] - 43 Subtract 53 54 52 0 r[52]=r[53]-r[54] - 44 Multiply 51 52 49 0 r[49]=r[51]*r[52] - 45 Copy 42 56 0 0 r[56]=r[42] - 46 Add 55 56 50 0 r[50]=r[55]+r[56] - 47 Multiply 49 50 48 0 r[48]=r[49]*r[50] - 48 AggStep 0 48 21 sum 0 accum=r[21] step(r[48]) - 49 AggStep 0 39 22 avg 0 accum=r[22] step(r[39]) - 50 AggStep 0 40 23 avg 0 accum=r[23] step(r[40]) - 51 AggStep 0 41 24 avg 0 accum=r[24] step(r[41]) - 52 AggStep 0 57 25 count 0 accum=r[25] step(r[57]) - 53 If 12 56 0 0 if r[12] goto 56; don't emit group columns if continuing existing group - 54 Column 1 0 16 0 r[16]=pseudo.column 0 - 55 Column 1 1 17 0 r[17]=pseudo.column 1 - 56 Integer 1 12 0 0 r[12]=1; indicate data in accumulator - 57 SorterNext 0 21 0 0 - 58 Gosub 11 62 0 0 ; emit row for final group - 59 Goto 0 87 0 0 ; group by finished - 60 Integer 1 13 0 0 r[13]=1 - 61 Return 11 0 0 0 - 62 IfPos 12 64 0 0 r[12]>0 -> r[12]-=0, goto 64; output group by row subroutine start - 63 Return 11 0 0 0 - 64 AggFinal 0 18 0 sum 0 accum=r[18] - 65 AggFinal 0 19 0 sum 0 accum=r[19] - 66 AggFinal 0 20 0 sum 0 accum=r[20] - 67 AggFinal 0 21 0 sum 0 accum=r[21] - 68 AggFinal 0 22 0 avg 0 accum=r[22] - 69 AggFinal 0 23 0 avg 0 accum=r[23] - 70 AggFinal 0 24 0 avg 0 accum=r[24] - 71 AggFinal 0 25 0 count 0 accum=r[25] - 72 Copy 16 1 0 0 r[1]=r[16] - 73 Copy 17 2 0 0 r[2]=r[17] - 74 Copy 18 3 0 0 r[3]=r[18] - 75 Copy 19 4 0 0 r[4]=r[19] - 76 Copy 20 5 0 0 r[5]=r[20] - 77 Copy 21 6 0 0 r[6]=r[21] - 78 Copy 22 7 0 0 r[7]=r[22] - 79 Copy 23 8 0 0 r[8]=r[23] - 80 Copy 24 9 0 0 r[9]=r[24] - 81 Copy 25 10 0 0 r[10]=r[25] - 82 ResultRow 1 10 0 0 output=r[1..10] - 83 Return 11 0 0 0 - 84 Null 0 16 25 0 r[16..25]=NULL; clear accumulator subroutine start - 85 Integer 0 12 0 0 r[12]=0 - 86 Return 33 0 0 0 - 87 Halt 0 0 0 0 - 88 Transaction 0 1 8 0 iDb=0 tx_mode=Read - 89 String8 0 36 0 1998-12-01 0 r[36]='1998-12-01' - 90 Integer 1 46 0 0 r[46]=1 - 91 Integer 1 53 0 0 r[53]=1 - 92 Integer 1 55 0 0 r[55]=1 - 93 Integer 1 57 0 0 r[57]=1 - 94 Goto 0 1 0 0 + 9 Gt 35 36 14 Binary 0 if r[35]>r[36] goto 14 + 10 ColumnRange 2 8 27 2 0 r[27..28]=lineitem.L_RETURNFLAG..L_LINESTATUS + 11 ColumnRange 2 4 29 4 0 r[29..32]=lineitem.L_QUANTITY..L_TAX + 12 MakeRecord 27 6 26 0 r[26]=mkrec(r[27..32]) + 13 SorterInsert 0 26 0 0 0 key=r[26] + 14 Next 2 8 0 1 + 15 OpenPseudo 1 26 6 0 6 columns in r[26] + 16 SorterSort 0 49 0 0 + 17 SorterData 0 26 1 0 r[26]=data + 18 ColumnRange 1 0 37 2 0 r[37..38]=pseudo.column 0..column 1 + 19 Compare 14 37 2 k(2, Binary, Binary) 0 r[14..15]==r[37..38] + 20 Jump 21 25 21 0 ; start new group if comparison is not equal + 21 Gosub 11 53 0 0 ; check if ended group had data, and output if so + 22 Move 37 14 2 0 r[14..15]=r[37..38] + 23 IfPos 13 78 0 0 r[13]>0 -> r[13]-=0, goto 78; check abort flag + 24 Gosub 33 75 0 0 ; goto clear accumulator subroutine + 25 ColumnRange 1 2 39 4 0 r[39..42]=pseudo.column 2..column 5 + 26 AggStep 0 39 18 sum 0 accum=r[18] step(r[39]) + 27 AggStep 0 40 19 sum 0 accum=r[19] step(r[40]) + 28 Copy 40 44 0 0 r[44]=r[40] + 29 Copy 41 47 0 0 r[47]=r[41] + 30 Subtract 46 47 45 0 r[45]=r[46]-r[47] + 31 Multiply 44 45 43 0 r[43]=r[44]*r[45] + 32 AggStep 0 43 20 sum 0 accum=r[20] step(r[43]) + 33 Copy 40 51 0 0 r[51]=r[40] + 34 Copy 41 54 0 0 r[54]=r[41] + 35 Subtract 53 54 52 0 r[52]=r[53]-r[54] + 36 Multiply 51 52 49 0 r[49]=r[51]*r[52] + 37 Copy 42 56 0 0 r[56]=r[42] + 38 Add 55 56 50 0 r[50]=r[55]+r[56] + 39 Multiply 49 50 48 0 r[48]=r[49]*r[50] + 40 AggStep 0 48 21 sum 0 accum=r[21] step(r[48]) + 41 AggStep 0 39 22 avg 0 accum=r[22] step(r[39]) + 42 AggStep 0 40 23 avg 0 accum=r[23] step(r[40]) + 43 AggStep 0 41 24 avg 0 accum=r[24] step(r[41]) + 44 AggStep 0 57 25 count 0 accum=r[25] step(r[57]) + 45 If 12 47 0 0 if r[12] goto 47; don't emit group columns if continuing existing group + 46 ColumnRange 1 0 16 2 0 r[16..17]=pseudo.column 0..column 1 + 47 Integer 1 12 0 0 r[12]=1; indicate data in accumulator + 48 SorterNext 0 17 0 0 + 49 Gosub 11 53 0 0 ; emit row for final group + 50 Goto 0 78 0 0 ; group by finished + 51 Integer 1 13 0 0 r[13]=1 + 52 Return 11 0 0 0 + 53 IfPos 12 55 0 0 r[12]>0 -> r[12]-=0, goto 55; output group by row subroutine start + 54 Return 11 0 0 0 + 55 AggFinal 0 18 0 sum 0 accum=r[18] + 56 AggFinal 0 19 0 sum 0 accum=r[19] + 57 AggFinal 0 20 0 sum 0 accum=r[20] + 58 AggFinal 0 21 0 sum 0 accum=r[21] + 59 AggFinal 0 22 0 avg 0 accum=r[22] + 60 AggFinal 0 23 0 avg 0 accum=r[23] + 61 AggFinal 0 24 0 avg 0 accum=r[24] + 62 AggFinal 0 25 0 count 0 accum=r[25] + 63 Copy 16 1 0 0 r[1]=r[16] + 64 Copy 17 2 0 0 r[2]=r[17] + 65 Copy 18 3 0 0 r[3]=r[18] + 66 Copy 19 4 0 0 r[4]=r[19] + 67 Copy 20 5 0 0 r[5]=r[20] + 68 Copy 21 6 0 0 r[6]=r[21] + 69 Copy 22 7 0 0 r[7]=r[22] + 70 Copy 23 8 0 0 r[8]=r[23] + 71 Copy 24 9 0 0 r[9]=r[24] + 72 Copy 25 10 0 0 r[10]=r[25] + 73 ResultRow 1 10 0 0 output=r[1..10] + 74 Return 11 0 0 0 + 75 Null 0 16 25 0 r[16..25]=NULL; clear accumulator subroutine start + 76 Integer 0 12 0 0 r[12]=0 + 77 Return 33 0 0 0 + 78 Halt 0 0 0 0 + 79 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 80 String8 0 36 0 1998-12-01 0 r[36]='1998-12-01' + 81 Integer 1 46 0 0 r[46]=1 + 82 Integer 1 53 0 0 r[53]=1 + 83 Integer 1 55 0 0 r[55]=1 + 84 Integer 1 57 0 0 r[57]=1 + 85 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q11-important-stock.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q11-important-stock.snap index f54a9545..606cc772 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q11-important-stock.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q11-important-stock.snap @@ -37,104 +37,269 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN partsupp USING INDEX sqlite_autoindex_partsupp_1 -|--SEARCH supplier USING INTEGER PRIMARY KEY (rowid=?) -|--SEARCH nation USING INTEGER PRIMARY KEY (rowid=?) +|--MATERIALIZE hash build input for supplier +| |--HASH JOIN supplier +| `--SCAN nation +|--HASH JOIN partsupp +|--SCAN nation +|--USE SORTER FOR GROUP BY |--SCALAR SUBQUERY 1 -| |--SCAN partsupp -| |--SEARCH supplier USING INTEGER PRIMARY KEY (rowid=?) -| `--SEARCH nation USING INTEGER PRIMARY KEY (rowid=?) +| |--MATERIALIZE hash build input for supplier +| | |--HASH JOIN supplier +| | `--SCAN nation +| |--HASH JOIN partsupp +| `--SCAN nation `--USE SORTER FOR ORDER BY BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 87 0 0 Start at 87 - 1 SorterOpen 0 1 0 k(1,-B) 0 cursor=0 - 2 Null 0 10 0 0 r[10]=NULL - 3 Integer 0 7 0 0 r[7]=0; clear group by abort flag - 4 Null 0 8 0 0 r[8]=NULL; initialize group by comparison registers to NULL - 5 Gosub 14 76 0 0 ; go to clear accumulator subroutine - 6 OpenRead 1 6 0 k(6,B,B,B,B,B) 0 table=partsupp, root=6, iDb=0 - 7 OpenRead 2 7 0 k(3,B,B) 0 index=sqlite_autoindex_partsupp_1, root=7, iDb=0 - 8 OpenRead 3 5 0 k(7,B,B,B,B,B,B,B) 0 table=supplier, root=5, iDb=0 - 9 OpenRead 4 2 0 k(4,B,B,B,B) 0 table=nation, root=2, iDb=0 - 10 Rewind 2 33 0 0 Rewind index sqlite_autoindex_partsupp_1 - 11 DeferredSeek 2 1 0 0 - 12 Column 2 1 15 0 r[15]=sqlite_autoindex_partsupp_1.ps_suppkey - 13 SeekRowid 3 15 32 0 if (r[15]!=cursor 3 for table supplier.rowid) goto 32 - 14 Column 3 3 16 0 r[16]=supplier.S_NATIONKEY - 15 SeekRowid 4 16 32 0 if (r[16]!=cursor 4 for table nation.rowid) goto 32 - 16 Column 4 1 18 0 r[18]=nation.N_NAME - 17 Ne 18 19 32 Binary 0 if r[18]!=r[19] goto 32 - 18 Column 2 0 12 0 r[12]=sqlite_autoindex_partsupp_1.ps_partkey - 19 Column 1 3 20 0 r[20]=partsupp.PS_SUPPLYCOST - 20 Column 1 2 21 0 r[21]=partsupp.PS_AVAILQTY - 21 Multiply 20 21 13 0 r[13]=r[20]*r[21] - 22 Compare 8 12 1 k(1, Binary) 0 r[8..8]==r[12..12] - 23 Jump 24 28 24 0 ; start new group if comparison is not equal - 24 Gosub 5 37 0 0 ; check if ended group had data, and output if so - 25 Move 12 8 1 0 r[8..8]=r[12..12] - 26 IfPos 7 79 0 0 r[7]>0 -> r[7]-=0, goto 79; check abort flag - 27 Gosub 14 76 0 0 ; goto clear accumulator subroutine - 28 AggStep 0 13 10 sum 0 accum=r[10] step(r[13]) - 29 If 6 31 0 0 if r[6] goto 31; don't emit group columns if continuing existing group - 30 Column 2 0 9 0 r[9]=sqlite_autoindex_partsupp_1.ps_partkey - 31 Integer 1 6 0 0 r[6]=1; indicate data in accumulator - 32 Next 2 11 0 0 - 33 Gosub 5 37 0 0 ; emit row for final group - 34 Goto 0 79 0 0 ; group by finished - 35 Integer 1 7 0 0 r[7]=1 - 36 Return 5 0 0 0 - 37 IfPos 6 39 0 0 r[6]>0 -> r[6]-=0, goto 39; output group by row subroutine start - 38 Return 5 0 0 0 - 39 AggFinal 0 10 0 sum 0 accum=r[10] - 40 Once 68 0 0 0 goto 68 - 41 BeginSubrtn 22 0 0 0 r[22]=NULL - 42 Null 0 1 0 0 r[1]=NULL - 43 Null 0 24 0 0 r[24]=NULL - 44 Integer 1 25 0 0 r[25]=1; LIMIT counter - 45 IfNot 25 62 0 0 if !r[25] goto 62 - 46 OpenRead 5 6 0 k(6,B,B,B,B,B) 0 table=partsupp, root=6, iDb=0 - 47 OpenRead 6 5 0 k(7,B,B,B,B,B,B,B) 0 table=supplier, root=5, iDb=0 - 48 OpenRead 7 2 0 k(4,B,B,B,B) 0 table=nation, root=2, iDb=0 - 49 Rewind 5 62 0 0 Rewind table partsupp - 50 Column 5 1 26 0 r[26]=partsupp.PS_SUPPKEY - 51 SeekRowid 6 26 61 0 if (r[26]!=cursor 6 for table supplier.rowid) goto 61 - 52 Column 6 3 27 0 r[27]=supplier.S_NATIONKEY - 53 SeekRowid 7 27 61 0 if (r[27]!=cursor 7 for table nation.rowid) goto 61 - 54 Column 7 1 29 0 r[29]=nation.N_NAME - 55 String8 0 30 0 ARGENTINA 0 r[30]='ARGENTINA' - 56 Ne 29 30 61 Binary 0 if r[29]!=r[30] goto 61 - 57 Column 5 3 32 0 r[32]=partsupp.PS_SUPPLYCOST - 58 Column 5 2 33 0 r[33]=partsupp.PS_AVAILQTY - 59 Multiply 32 33 31 0 r[31]=r[32]*r[33] - 60 AggStep 0 31 24 sum 0 accum=r[24] step(r[31]) - 61 Next 5 50 0 0 - 62 AggFinal 0 24 0 sum 0 accum=r[24] - 63 Copy 24 34 0 0 r[34]=r[24] - 64 Real 0 35 0 0.0001 0 r[35]=0.0001 - 65 Multiply 34 35 23 0 r[23]=r[34]*r[35] - 66 Copy 23 1 0 0 r[1]=r[23] - 67 Return 22 0 1 0 - 68 Copy 10 37 0 0 r[37]=r[10] - 69 Copy 1 38 0 0 r[38]=r[1] - 70 Le 37 38 38 0 if r[37]<=r[38] goto 38 - 71 Copy 10 39 0 0 r[39]=r[10] - 72 Copy 9 40 0 0 r[40]=r[9] - 73 MakeRecord 39 2 4 0 r[4]=mkrec(r[39..40]) - 74 SorterInsert 0 4 0 0 0 key=r[4] - 75 Return 5 0 0 0 - 76 Null 0 9 10 0 r[9..10]=NULL; clear accumulator subroutine start - 77 Integer 0 6 0 0 r[6]=0 - 78 Return 14 0 0 0 - 79 OpenPseudo 8 4 2 0 2 columns in r[4] - 80 SorterSort 0 86 0 0 - 81 SorterData 0 4 8 0 r[4]=data - 82 Column 8 1 2 0 r[2]=pseudo.column 1 - 83 Column 8 0 3 0 r[3]=pseudo.column 0 - 84 ResultRow 2 2 0 0 output=r[2..3] - 85 SorterNext 0 81 0 0 - 86 Halt 0 0 0 0 - 87 Transaction 0 1 8 0 iDb=0 tx_mode=Read - 88 String8 0 19 0 ARGENTINA 0 r[19]='ARGENTINA' - 89 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 247 0 0 Start at 247 + 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true + 2 OpenRead 1 5 0 k(7,B,B,B,B,B,B,B) 0 table=supplier, root=5, iDb=0 + 3 OpenRead 2 2 0 k(4,B,B,B,B) 0 table=nation, root=2, iDb=0 + 4 Once 17 0 0 0 goto 17 + 5 OpenRead 3 2 0 k(4,B,B,B,B) 0 table=nation, root=2, iDb=0 + 6 Rewind 3 16 0 0 Rewind table nation + 7 Column 3 1 11 0 r[11]=nation.N_NAME + 8 Ne 11 12 15 Binary 0 if r[11]!=r[12] goto 15 + 9 RowId 3 9 0 0 r[9]=nation.rowid + 10 Affinity 9 1 0 0 r[9..10] = C + 11 RowId 3 13 0 0 r[13]=nation.rowid + 12 Column 3 1 14 0 r[14]=nation.N_NAME + 13 HashBuild 3 9 1 r=[3] budget=67108864 payload=r[13]..r[14] 0 + 14 FilterAdd 3 9 1 0 bloom_filter_add(r[9..10]) + 15 Next 3 7 0 0 + 16 HashBuildFinalize 3 0 0 0 + 17 Rewind 1 41 0 0 Rewind table supplier + 18 Column 1 3 15 0 r[15]=supplier.S_NATIONKEY + 19 Affinity 15 1 0 0 r[15..16] = C + 20 Filter 3 40 15 1 if !bloom_filter(r[15..16]) goto 40 + 21 RowId 1 18 0 0 r[18]=supplier.rowid + 22 Integer 0 19 0 0 r[19]=0 + 23 HashProbe 3 15 1 r[20]=40 payload=r[16]..r[17] 0 + 24 Column 1 3 22 0 r[22]=supplier.S_NATIONKEY + 25 Copy 16 23 0 0 r[23]=r[16] + 26 Ne 22 23 37 Binary 0 if r[22]!=r[23] goto 37 + 27 RowId 1 2 0 0 r[2]=supplier.rowid + 28 RowId 1 3 0 0 r[3]=supplier.rowid + 29 Column 1 3 4 0 r[4]=supplier.S_NATIONKEY + 30 RowId 1 5 0 0 r[5]=supplier.rowid + 31 Copy 16 6 0 0 r[6]=r[16] + 32 Copy 17 7 0 0 r[7]=r[17] + 33 Copy 20 8 0 0 r[8]=r[20] + 34 MakeRecord 2 7 24 0 r[24]=mkrec(r[2..8]); for hash_build_input_t8 + 35 NewRowid 0 25 0 0 r[25]=rowid + 36 Insert 0 24 25 hash_build_input_t8 4 intkey=r[25] data=r[24] + 37 IfPos 19 48 0 0 r[19]>0 -> r[19]-=0, goto 48 + 38 HashNext 3 20 40 payload=r[16]..r[17] 0 + 39 Goto 0 24 0 0 + 40 Next 1 18 0 0 + 41 HashGraceInit 3 0 53 0 hash_table_id=3 + 42 Integer 1 19 0 0 r[19]=1 + 43 HashGraceLoadPart 3 0 52 0 hash_table_id=3 + 44 HashGraceNextProbe 3 15 50 0 hash_table_id=3 keys=r[15]..r[15] probe_rowid_dest=r[18] + 45 SeekRowid 1 18 44 0 if (r[18]!=cursor 1 for table supplier.rowid) goto 44 + 46 HashProbe 3 15 1 r[20]=44 payload=r[16]..r[17] 0 + 47 Goto 0 24 0 0 + 48 HashNext 3 20 44 payload=r[16]..r[17] 0 + 49 Goto 0 24 0 0 + 50 HashGraceAdvPart 3 0 52 0 hash_table_id=3 + 51 Goto 0 43 0 0 + 52 Integer 0 19 0 0 r[19]=0 + 53 SorterOpen 4 1 0 k(1,-B) 0 cursor=4 + 54 Null 0 34 0 0 r[34]=NULL + 55 SorterOpen 5 3 0 k(1,-B) 0 cursor=5 + 56 Integer 0 31 0 0 r[31]=0; clear group by abort flag + 57 Null 0 32 0 0 r[32]=NULL; initialize group by comparison registers to NULL + 58 Gosub 39 236 0 0 ; go to clear accumulator subroutine + 59 OpenRead 7 6 0 k(6,B,B,B,B,B) 0 table=partsupp, root=6, iDb=0 + 60 OpenRead 2 2 0 k(4,B,B,B,B) 0 table=nation, root=2, iDb=0 + 61 Once 70 0 0 0 goto 70 + 62 Rewind 0 69 0 0 Rewind ephemeral(hash_build_input_t8) + 63 Column 0 0 40 0 r[40]=ephemeral(hash_build_input_t8).key_0 + 64 Affinity 40 1 0 0 r[40..41] = C + 65 ColumnRange 0 1 41 6 0 r[41..46]=ephemeral(hash_build_input_t8).payload_0..payload_5 + 66 HashBuild 0 40 1 r=[2] budget=67108864 payload=r[41]..r[46] 0 + 67 FilterAdd 0 40 1 0 bloom_filter_add(r[40..41]) + 68 Next 0 63 0 0 + 69 HashBuildFinalize 2 0 0 0 + 70 Rewind 7 86 0 0 Rewind table partsupp + 71 Column 7 1 47 0 r[47]=partsupp.PS_SUPPKEY + 72 Affinity 47 1 0 0 r[47..48] = C + 73 Filter 0 85 47 1 if !bloom_filter(r[47..48]) goto 85 + 74 RowId 7 54 0 0 r[54]=partsupp.rowid + 75 Integer 0 55 0 0 r[55]=0 + 76 HashProbe 2 47 1 r[56]=85 payload=r[48]..r[53] 0 + 77 Column 7 0 36 0 r[36]=partsupp.PS_PARTKEY + 78 Column 7 3 37 0 r[37]=partsupp.PS_SUPPLYCOST + 79 Column 7 2 38 0 r[38]=partsupp.PS_AVAILQTY + 80 MakeRecord 36 3 35 0 r[35]=mkrec(r[36..38]) + 81 SorterInsert 5 35 0 0 0 key=r[35] + 82 IfPos 55 93 0 0 r[55]>0 -> r[55]-=0, goto 93 + 83 HashNext 2 56 85 payload=r[48]..r[53] 0 + 84 Goto 0 77 0 0 + 85 Next 7 71 0 0 + 86 HashGraceInit 2 0 98 0 hash_table_id=2 + 87 Integer 1 55 0 0 r[55]=1 + 88 HashGraceLoadPart 2 0 97 0 hash_table_id=2 + 89 HashGraceNextProbe 2 47 95 0 hash_table_id=2 keys=r[47]..r[47] probe_rowid_dest=r[54] + 90 SeekRowid 7 54 89 0 if (r[54]!=cursor 7 for table partsupp.rowid) goto 89 + 91 HashProbe 2 47 1 r[56]=89 payload=r[48]..r[53] 0 + 92 Goto 0 77 0 0 + 93 HashNext 2 56 89 payload=r[48]..r[53] 0 + 94 Goto 0 77 0 0 + 95 HashGraceAdvPart 2 0 97 0 hash_table_id=2 + 96 Goto 0 88 0 0 + 97 Integer 0 55 0 0 r[55]=0 + 98 HashClose 2 0 0 0 + 99 OpenPseudo 6 35 3 0 3 columns in r[35] + 100 SorterSort 5 118 0 0 + 101 SorterData 5 35 6 0 r[35]=data + 102 Column 6 0 57 0 r[57]=pseudo.column 0 + 103 Compare 32 57 1 k(1, Binary) 0 r[32..32]==r[57..57] + 104 Jump 105 109 105 0 ; start new group if comparison is not equal + 105 Gosub 29 122 0 0 ; check if ended group had data, and output if so + 106 Move 57 32 1 0 r[32..32]=r[57..57] + 107 IfPos 31 239 0 0 r[31]>0 -> r[31]-=0, goto 239; check abort flag + 108 Gosub 39 236 0 0 ; goto clear accumulator subroutine + 109 ColumnRange 6 1 58 2 0 r[58..59]=pseudo.column 1..column 2 + 110 Copy 58 61 0 0 r[61]=r[58] + 111 Copy 59 62 0 0 r[62]=r[59] + 112 Multiply 61 62 60 0 r[60]=r[61]*r[62] + 113 AggStep 0 60 34 sum 0 accum=r[34] step(r[60]) + 114 If 30 116 0 0 if r[30] goto 116; don't emit group columns if continuing existing group + 115 Column 6 0 33 0 r[33]=pseudo.column 0 + 116 Integer 1 30 0 0 r[30]=1; indicate data in accumulator + 117 SorterNext 5 101 0 0 + 118 Gosub 29 122 0 0 ; emit row for final group + 119 Goto 0 239 0 0 ; group by finished + 120 Integer 1 31 0 0 r[31]=1 + 121 Return 29 0 0 0 + 122 IfPos 30 124 0 0 r[30]>0 -> r[30]-=0, goto 124; output group by row subroutine start + 123 Return 29 0 0 0 + 124 AggFinal 0 34 0 sum 0 accum=r[34] + 125 Once 228 0 0 0 goto 228 + 126 BeginSubrtn 63 0 0 0 r[63]=NULL + 127 Null 0 1 0 0 r[1]=NULL + 128 OpenEphemeral 9 1 0 0 cursor=9 is_table=true + 129 OpenRead 10 5 0 k(7,B,B,B,B,B,B,B) 0 table=supplier, root=5, iDb=0 + 130 OpenRead 11 2 0 k(4,B,B,B,B) 0 table=nation, root=2, iDb=0 + 131 Once 145 0 0 0 goto 145 + 132 OpenRead 12 2 0 k(4,B,B,B,B) 0 table=nation, root=2, iDb=0 + 133 Rewind 12 144 0 0 Rewind table nation + 134 Column 12 1 73 0 r[73]=nation.N_NAME + 135 String8 0 74 0 ARGENTINA 0 r[74]='ARGENTINA' + 136 Ne 73 74 143 Binary 0 if r[73]!=r[74] goto 143 + 137 RowId 12 71 0 0 r[71]=nation.rowid + 138 Affinity 71 1 0 0 r[71..72] = C + 139 RowId 12 75 0 0 r[75]=nation.rowid + 140 Column 12 1 76 0 r[76]=nation.N_NAME + 141 HashBuild 12 71 1 r=[7] budget=67108864 payload=r[75]..r[76] 0 + 142 FilterAdd 12 71 1 0 bloom_filter_add(r[71..72]) + 143 Next 12 134 0 0 + 144 HashBuildFinalize 7 0 0 0 + 145 Rewind 10 169 0 0 Rewind table supplier + 146 Column 10 3 77 0 r[77]=supplier.S_NATIONKEY + 147 Affinity 77 1 0 0 r[77..78] = C + 148 Filter 12 168 77 1 if !bloom_filter(r[77..78]) goto 168 + 149 RowId 10 80 0 0 r[80]=supplier.rowid + 150 Integer 0 81 0 0 r[81]=0 + 151 HashProbe 7 77 1 r[82]=168 payload=r[78]..r[79] 0 + 152 Column 10 3 84 0 r[84]=supplier.S_NATIONKEY + 153 Copy 78 85 0 0 r[85]=r[78] + 154 Ne 84 85 165 Binary 0 if r[84]!=r[85] goto 165 + 155 RowId 10 64 0 0 r[64]=supplier.rowid + 156 RowId 10 65 0 0 r[65]=supplier.rowid + 157 Column 10 3 66 0 r[66]=supplier.S_NATIONKEY + 158 RowId 10 67 0 0 r[67]=supplier.rowid + 159 Copy 78 68 0 0 r[68]=r[78] + 160 Copy 79 69 0 0 r[69]=r[79] + 161 Copy 82 70 0 0 r[70]=r[82] + 162 MakeRecord 64 7 86 0 r[86]=mkrec(r[64..70]); for hash_build_input_t9 + 163 NewRowid 9 87 0 0 r[87]=rowid + 164 Insert 9 86 87 hash_build_input_t9 4 intkey=r[87] data=r[86] + 165 IfPos 81 176 0 0 r[81]>0 -> r[81]-=0, goto 176 + 166 HashNext 7 82 168 payload=r[78]..r[79] 0 + 167 Goto 0 152 0 0 + 168 Next 10 146 0 0 + 169 HashGraceInit 7 0 181 0 hash_table_id=7 + 170 Integer 1 81 0 0 r[81]=1 + 171 HashGraceLoadPart 7 0 180 0 hash_table_id=7 + 172 HashGraceNextProbe 7 77 178 0 hash_table_id=7 keys=r[77]..r[77] probe_rowid_dest=r[80] + 173 SeekRowid 10 80 172 0 if (r[80]!=cursor 10 for table supplier.rowid) goto 172 + 174 HashProbe 7 77 1 r[82]=172 payload=r[78]..r[79] 0 + 175 Goto 0 152 0 0 + 176 HashNext 7 82 172 payload=r[78]..r[79] 0 + 177 Goto 0 152 0 0 + 178 HashGraceAdvPart 7 0 180 0 hash_table_id=7 + 179 Goto 0 171 0 0 + 180 Integer 0 81 0 0 r[81]=0 + 181 Null 0 89 0 0 r[89]=NULL + 182 Integer 1 90 0 0 r[90]=1; LIMIT counter + 183 IfNot 90 222 0 0 if !r[90] goto 222 + 184 OpenRead 13 6 0 k(6,B,B,B,B,B) 0 table=partsupp, root=6, iDb=0 + 185 OpenRead 11 2 0 k(4,B,B,B,B) 0 table=nation, root=2, iDb=0 + 186 Once 195 0 0 0 goto 195 + 187 Rewind 9 194 0 0 Rewind ephemeral(hash_build_input_t9) + 188 Column 9 0 91 0 r[91]=ephemeral(hash_build_input_t9).key_0 + 189 Affinity 91 1 0 0 r[91..92] = C + 190 ColumnRange 9 1 92 6 0 r[92..97]=ephemeral(hash_build_input_t9).payload_0..payload_5 + 191 HashBuild 9 91 1 r=[6] budget=67108864 payload=r[92]..r[97] 0 + 192 FilterAdd 9 91 1 0 bloom_filter_add(r[91..92]) + 193 Next 9 188 0 0 + 194 HashBuildFinalize 6 0 0 0 + 195 Rewind 13 210 0 0 Rewind table partsupp + 196 Column 13 1 98 0 r[98]=partsupp.PS_SUPPKEY + 197 Affinity 98 1 0 0 r[98..99] = C + 198 Filter 9 209 98 1 if !bloom_filter(r[98..99]) goto 209 + 199 RowId 13 105 0 0 r[105]=partsupp.rowid + 200 Integer 0 106 0 0 r[106]=0 + 201 HashProbe 6 98 1 r[107]=209 payload=r[99]..r[104] 0 + 202 Column 13 3 109 0 r[109]=partsupp.PS_SUPPLYCOST + 203 Column 13 2 110 0 r[110]=partsupp.PS_AVAILQTY + 204 Multiply 109 110 108 0 r[108]=r[109]*r[110] + 205 AggStep 0 108 89 sum 0 accum=r[89] step(r[108]) + 206 IfPos 106 217 0 0 r[106]>0 -> r[106]-=0, goto 217 + 207 HashNext 6 107 209 payload=r[99]..r[104] 0 + 208 Goto 0 202 0 0 + 209 Next 13 196 0 0 + 210 HashGraceInit 6 0 222 0 hash_table_id=6 + 211 Integer 1 106 0 0 r[106]=1 + 212 HashGraceLoadPart 6 0 221 0 hash_table_id=6 + 213 HashGraceNextProbe 6 98 219 0 hash_table_id=6 keys=r[98]..r[98] probe_rowid_dest=r[105] + 214 SeekRowid 13 105 213 0 if (r[105]!=cursor 13 for table partsupp.rowid) goto 213 + 215 HashProbe 6 98 1 r[107]=213 payload=r[99]..r[104] 0 + 216 Goto 0 202 0 0 + 217 HashNext 6 107 213 payload=r[99]..r[104] 0 + 218 Goto 0 202 0 0 + 219 HashGraceAdvPart 6 0 221 0 hash_table_id=6 + 220 Goto 0 212 0 0 + 221 Integer 0 106 0 0 r[106]=0 + 222 AggFinal 0 89 0 sum 0 accum=r[89] + 223 Copy 89 111 0 0 r[111]=r[89] + 224 Real 0 112 0 0.0001 0 r[112]=0.0001 + 225 Multiply 111 112 88 0 r[88]=r[111]*r[112] + 226 Copy 88 1 0 0 r[1]=r[88] + 227 Return 63 0 1 0 + 228 Copy 34 114 0 0 r[114]=r[34] + 229 Copy 1 115 0 0 r[115]=r[1] + 230 Le 114 115 123 0 if r[114]<=r[115] goto 123 + 231 Copy 34 116 0 0 r[116]=r[34] + 232 Copy 33 117 0 0 r[117]=r[33] + 233 MakeRecord 116 2 28 0 r[28]=mkrec(r[116..117]) + 234 SorterInsert 4 28 0 0 0 key=r[28] + 235 Return 29 0 0 0 + 236 Null 0 33 34 0 r[33..34]=NULL; clear accumulator subroutine start + 237 Integer 0 30 0 0 r[30]=0 + 238 Return 39 0 0 0 + 239 OpenPseudo 15 28 2 0 2 columns in r[28] + 240 SorterSort 4 246 0 0 + 241 SorterData 4 28 15 0 r[28]=data + 242 Column 15 1 26 0 r[26]=pseudo.column 1 + 243 Column 15 0 27 0 r[27]=pseudo.column 0 + 244 ResultRow 26 2 0 0 output=r[26..27] + 245 SorterNext 4 241 0 0 + 246 Halt 0 0 0 0 + 247 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 248 String8 0 12 0 ARGENTINA 0 r[12]='ARGENTINA' + 249 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q12-shipping-modes.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q12-shipping-modes.snap index d92bcd6c..99a21c31 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q12-shipping-modes.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q12-shipping-modes.snap @@ -44,102 +44,100 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 93 0 0 Start at 93 + 0 Init 0 91 0 0 Start at 91 1 Null 0 9 10 0 r[9..10]=NULL 2 SorterOpen 0 2 0 k(1,B) 0 cursor=0 3 Integer 0 6 0 0 r[6]=0; clear group by abort flag 4 Null 0 7 0 0 r[7]=NULL; initialize group by comparison registers to NULL - 5 Gosub 14 89 0 0 ; go to clear accumulator subroutine + 5 Gosub 14 87 0 0 ; go to clear accumulator subroutine 6 OpenRead 2 9 0 k(9,B,B,B,B,B,B,B,B,B) 0 table=orders, root=9, iDb=0 7 OpenRead 3 10 0 k(17,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B) 0 table=lineitem, root=10, iDb=0 - 8 Rewind 3 29 0 0 Rewind table lineitem + 8 Rewind 3 27 0 0 Rewind table lineitem 9 Column 3 14 15 0 r[15]=lineitem.L_SHIPMODE 10 Eq 15 16 12 Binary 0 if r[15]==r[16] goto 12 - 11 Ne 15 17 28 Binary 0 if r[15]!=r[17] goto 28 - 12 Column 3 11 19 0 r[19]=lineitem.L_COMMITDATE - 13 Column 3 12 20 0 r[20]=lineitem.L_RECEIPTDATE - 14 Ge 19 20 28 Binary 0 if r[19]>=r[20] goto 28 - 15 Column 3 10 22 0 r[22]=lineitem.L_SHIPDATE - 16 Column 3 11 23 0 r[23]=lineitem.L_COMMITDATE - 17 Ge 22 23 28 Binary 0 if r[22]>=r[23] goto 28 - 18 Column 3 12 25 0 r[25]=lineitem.L_RECEIPTDATE - 19 Lt 25 26 28 Binary 0 if r[25]=r[29] goto 28 - 22 Column 3 0 30 0 r[30]=lineitem.L_ORDERKEY - 23 SeekRowid 2 30 28 0 if (r[30]!=cursor 2 for table orders.rowid) goto 28 - 24 Column 3 14 12 0 r[12]=lineitem.L_SHIPMODE - 25 Column 2 5 13 0 r[13]=orders.O_ORDERPRIORITY - 26 MakeRecord 12 2 11 0 r[11]=mkrec(r[12..13]) - 27 SorterInsert 0 11 0 0 0 key=r[11] - 28 Next 3 9 0 0 - 29 OpenPseudo 1 11 2 0 2 columns in r[11] - 30 SorterSort 0 76 0 0 - 31 SorterData 0 11 1 0 r[11]=data - 32 Column 1 0 31 0 r[31]=pseudo.column 0 - 33 Compare 7 31 1 k(1, Binary) 0 r[7..7]==r[31..31] - 34 Jump 35 39 35 0 ; start new group if comparison is not equal - 35 Gosub 4 80 0 0 ; check if ended group had data, and output if so - 36 Move 31 7 1 0 r[7..7]=r[31..31] - 37 IfPos 6 92 0 0 r[6]>0 -> r[6]-=0, goto 92; check abort flag - 38 Gosub 14 89 0 0 ; goto clear accumulator subroutine - 39 Column 1 1 32 0 r[32]=pseudo.column 1 - 40 Copy 32 37 0 0 r[37]=r[32] - 41 String8 0 38 0 1-URGENT 0 r[38]='1-URGENT' - 42 Integer 1 35 0 0 r[35]=1 - 43 Eq 37 38 45 0 if r[37]==r[38] goto 45 - 44 ZeroOrNull 37 35 38 0 ((r[37]=NULL)|(r[38]=NULL)) ? r[35]=NULL : r[35]=0 - 45 Copy 32 39 0 0 r[39]=r[32] - 46 String8 0 40 0 2-HIGH 0 r[40]='2-HIGH' - 47 Integer 1 36 0 0 r[36]=1 - 48 Eq 39 40 50 0 if r[39]==r[40] goto 50 - 49 ZeroOrNull 39 36 40 0 ((r[39]=NULL)|(r[40]=NULL)) ? r[36]=NULL : r[36]=0 - 50 Or 36 35 34 0 r[34]=(r[35] || r[36]) - 51 IfNot 34 54 1 0 if !r[34] goto 54 - 52 Integer 1 33 0 0 r[33]=1 - 53 Goto 0 55 0 0 - 54 Integer 0 33 0 0 r[33]=0 - 55 AggStep 0 33 9 sum 0 accum=r[9] step(r[33]) - 56 Copy 32 45 0 0 r[45]=r[32] - 57 String8 0 46 0 1-URGENT 0 r[46]='1-URGENT' - 58 Integer 1 43 0 0 r[43]=1 - 59 Ne 45 46 61 0 if r[45]!=r[46] goto 61 - 60 ZeroOrNull 45 43 46 0 ((r[45]=NULL)|(r[46]=NULL)) ? r[43]=NULL : r[43]=0 - 61 Copy 32 47 0 0 r[47]=r[32] - 62 String8 0 48 0 2-HIGH 0 r[48]='2-HIGH' - 63 Integer 1 44 0 0 r[44]=1 - 64 Ne 47 48 66 0 if r[47]!=r[48] goto 66 - 65 ZeroOrNull 47 44 48 0 ((r[47]=NULL)|(r[48]=NULL)) ? r[44]=NULL : r[44]=0 - 66 And 44 43 42 0 r[42]=(r[43] && r[44]) - 67 IfNot 42 70 1 0 if !r[42] goto 70 - 68 Integer 1 41 0 0 r[41]=1 - 69 Goto 0 71 0 0 - 70 Integer 0 41 0 0 r[41]=0 - 71 AggStep 0 41 10 sum 0 accum=r[10] step(r[41]) - 72 If 5 74 0 0 if r[5] goto 74; don't emit group columns if continuing existing group - 73 Column 1 0 8 0 r[8]=pseudo.column 0 - 74 Integer 1 5 0 0 r[5]=1; indicate data in accumulator - 75 SorterNext 0 31 0 0 - 76 Gosub 4 80 0 0 ; emit row for final group - 77 Goto 0 92 0 0 ; group by finished - 78 Integer 1 6 0 0 r[6]=1 - 79 Return 4 0 0 0 - 80 IfPos 5 82 0 0 r[5]>0 -> r[5]-=0, goto 82; output group by row subroutine start - 81 Return 4 0 0 0 - 82 AggFinal 0 9 0 sum 0 accum=r[9] - 83 AggFinal 0 10 0 sum 0 accum=r[10] - 84 Copy 8 1 0 0 r[1]=r[8] - 85 Copy 9 2 0 0 r[2]=r[9] - 86 Copy 10 3 0 0 r[3]=r[10] - 87 ResultRow 1 3 0 0 output=r[1..3] - 88 Return 4 0 0 0 - 89 Null 0 8 10 0 r[8..10]=NULL; clear accumulator subroutine start - 90 Integer 0 5 0 0 r[5]=0 - 91 Return 14 0 0 0 - 92 Halt 0 0 0 0 - 93 Transaction 0 1 8 0 iDb=0 tx_mode=Read - 94 String8 0 16 0 FOB 0 r[16]='FOB' - 95 String8 0 17 0 SHIP 0 r[17]='SHIP' - 96 String8 0 26 0 1994-01-01 0 r[26]='1994-01-01' - 97 String8 0 29 0 1995-01-01 0 r[29]='1995-01-01' - 98 Goto 0 1 0 0 + 11 Ne 15 17 26 Binary 0 if r[15]!=r[17] goto 26 + 12 ColumnRange 3 11 19 2 0 r[19..20]=lineitem.L_COMMITDATE..L_RECEIPTDATE + 13 Ge 19 20 26 Binary 0 if r[19]>=r[20] goto 26 + 14 ColumnRange 3 10 22 2 0 r[22..23]=lineitem.L_SHIPDATE..L_COMMITDATE + 15 Ge 22 23 26 Binary 0 if r[22]>=r[23] goto 26 + 16 Column 3 12 25 0 r[25]=lineitem.L_RECEIPTDATE + 17 Lt 25 26 26 Binary 0 if r[25]=r[29] goto 26 + 20 Column 3 0 30 0 r[30]=lineitem.L_ORDERKEY + 21 SeekRowid 2 30 26 0 if (r[30]!=cursor 2 for table orders.rowid) goto 26 + 22 Column 3 14 12 0 r[12]=lineitem.L_SHIPMODE + 23 Column 2 5 13 0 r[13]=orders.O_ORDERPRIORITY + 24 MakeRecord 12 2 11 0 r[11]=mkrec(r[12..13]) + 25 SorterInsert 0 11 0 0 0 key=r[11] + 26 Next 3 9 0 1 + 27 OpenPseudo 1 11 2 0 2 columns in r[11] + 28 SorterSort 0 74 0 0 + 29 SorterData 0 11 1 0 r[11]=data + 30 Column 1 0 31 0 r[31]=pseudo.column 0 + 31 Compare 7 31 1 k(1, Binary) 0 r[7..7]==r[31..31] + 32 Jump 33 37 33 0 ; start new group if comparison is not equal + 33 Gosub 4 78 0 0 ; check if ended group had data, and output if so + 34 Move 31 7 1 0 r[7..7]=r[31..31] + 35 IfPos 6 90 0 0 r[6]>0 -> r[6]-=0, goto 90; check abort flag + 36 Gosub 14 87 0 0 ; goto clear accumulator subroutine + 37 Column 1 1 32 0 r[32]=pseudo.column 1 + 38 Copy 32 37 0 0 r[37]=r[32] + 39 String8 0 38 0 1-URGENT 0 r[38]='1-URGENT' + 40 Integer 1 35 0 0 r[35]=1 + 41 Eq 37 38 43 0 if r[37]==r[38] goto 43 + 42 ZeroOrNull 37 35 38 0 ((r[37]=NULL)|(r[38]=NULL)) ? r[35]=NULL : r[35]=0 + 43 Copy 32 39 0 0 r[39]=r[32] + 44 String8 0 40 0 2-HIGH 0 r[40]='2-HIGH' + 45 Integer 1 36 0 0 r[36]=1 + 46 Eq 39 40 48 0 if r[39]==r[40] goto 48 + 47 ZeroOrNull 39 36 40 0 ((r[39]=NULL)|(r[40]=NULL)) ? r[36]=NULL : r[36]=0 + 48 Or 36 35 34 0 r[34]=(r[35] || r[36]) + 49 IfNot 34 52 1 0 if !r[34] goto 52 + 50 Integer 1 33 0 0 r[33]=1 + 51 Goto 0 53 0 0 + 52 Integer 0 33 0 0 r[33]=0 + 53 AggStep 0 33 9 sum 0 accum=r[9] step(r[33]) + 54 Copy 32 45 0 0 r[45]=r[32] + 55 String8 0 46 0 1-URGENT 0 r[46]='1-URGENT' + 56 Integer 1 43 0 0 r[43]=1 + 57 Ne 45 46 59 0 if r[45]!=r[46] goto 59 + 58 ZeroOrNull 45 43 46 0 ((r[45]=NULL)|(r[46]=NULL)) ? r[43]=NULL : r[43]=0 + 59 Copy 32 47 0 0 r[47]=r[32] + 60 String8 0 48 0 2-HIGH 0 r[48]='2-HIGH' + 61 Integer 1 44 0 0 r[44]=1 + 62 Ne 47 48 64 0 if r[47]!=r[48] goto 64 + 63 ZeroOrNull 47 44 48 0 ((r[47]=NULL)|(r[48]=NULL)) ? r[44]=NULL : r[44]=0 + 64 And 44 43 42 0 r[42]=(r[43] && r[44]) + 65 IfNot 42 68 1 0 if !r[42] goto 68 + 66 Integer 1 41 0 0 r[41]=1 + 67 Goto 0 69 0 0 + 68 Integer 0 41 0 0 r[41]=0 + 69 AggStep 0 41 10 sum 0 accum=r[10] step(r[41]) + 70 If 5 72 0 0 if r[5] goto 72; don't emit group columns if continuing existing group + 71 Column 1 0 8 0 r[8]=pseudo.column 0 + 72 Integer 1 5 0 0 r[5]=1; indicate data in accumulator + 73 SorterNext 0 29 0 0 + 74 Gosub 4 78 0 0 ; emit row for final group + 75 Goto 0 90 0 0 ; group by finished + 76 Integer 1 6 0 0 r[6]=1 + 77 Return 4 0 0 0 + 78 IfPos 5 80 0 0 r[5]>0 -> r[5]-=0, goto 80; output group by row subroutine start + 79 Return 4 0 0 0 + 80 AggFinal 0 9 0 sum 0 accum=r[9] + 81 AggFinal 0 10 0 sum 0 accum=r[10] + 82 Copy 8 1 0 0 r[1]=r[8] + 83 Copy 9 2 0 0 r[2]=r[9] + 84 Copy 10 3 0 0 r[3]=r[10] + 85 ResultRow 1 3 0 0 output=r[1..3] + 86 Return 4 0 0 0 + 87 Null 0 8 10 0 r[8..10]=NULL; clear accumulator subroutine start + 88 Integer 0 5 0 0 r[5]=0 + 89 Return 14 0 0 0 + 90 Halt 0 0 0 0 + 91 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 92 String8 0 16 0 FOB 0 r[16]='FOB' + 93 String8 0 17 0 SHIP 0 r[17]='SHIP' + 94 String8 0 26 0 1994-01-01 0 r[26]='1994-01-01' + 95 String8 0 29 0 1995-01-01 0 r[29]='1995-01-01' + 96 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q14-promotion-effect.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q14-promotion-effect.snap index 5da8e9bb..d1ace96b 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q14-promotion-effect.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q14-promotion-effect.snap @@ -57,7 +57,7 @@ addr opcode p1 p2 p3 p4 p5 commen 26 Subtract 22 23 21 0 r[21]=r[22]-r[23] 27 Multiply 20 21 19 0 r[19]=r[20]*r[21] 28 AggStep 0 19 3 sum 0 accum=r[3] step(r[19]) - 29 Next 0 5 0 0 + 29 Next 0 5 0 1 30 AggFinal 0 2 0 sum 0 accum=r[2] 31 AggFinal 0 3 0 sum 0 accum=r[3] 32 Copy 2 27 0 0 r[27]=r[2] diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q16-parts-supplier.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q16-parts-supplier.snap index 13291ef9..32484d24 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q16-parts-supplier.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q16-parts-supplier.snap @@ -44,7 +44,7 @@ QUERY PLAN |--LIST SUBQUERY 1 | `--SCAN supplier |--SCAN part -|--SEARCH partsupp USING INDEX sqlite_autoindex_partsupp_1 (ps_partkey=?) +|--SEARCH partsupp USING COVERING INDEX sqlite_autoindex_partsupp_1 (ps_partkey=?) |--USE SORTER FOR GROUP BY |--USE HASH TABLE FOR count(DISTINCT) `--USE SORTER FOR ORDER BY diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q17-small-quantity-order.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q17-small-quantity-order.snap index 8e49287d..2d1e08cf 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q17-small-quantity-order.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q17-small-quantity-order.snap @@ -27,56 +27,124 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN lineitem -|--SEARCH part USING INTEGER PRIMARY KEY (rowid=?) -`--CORRELATED SCALAR SUBQUERY 1 - `--SCAN lineitem +|--HASH JOIN lineitem +|--SEARCH scalar_subquery_t3 USING INDEX ephemeral_subquery_t3 (correlation_key_0=?) +| |--SCAN lineitem +| `--USE SORTER FOR GROUP BY +`--SCAN part BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 40 0 0 Start at 40 - 1 Null 0 3 0 0 r[3]=NULL - 2 OpenRead 0 10 0 k(17,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B) 0 table=lineitem, root=10, iDb=0 - 3 OpenRead 1 4 0 k(9,B,B,B,B,B,B,B,B,B) 0 table=part, root=4, iDb=0 - 4 Rewind 0 35 0 0 Rewind table lineitem - 5 Column 0 1 4 0 r[4]=lineitem.L_PARTKEY - 6 SeekRowid 1 4 34 0 if (r[4]!=cursor 1 for table part.rowid) goto 34 - 7 Column 1 3 6 0 r[6]=part.P_BRAND - 8 Ne 6 7 34 Binary 0 if r[6]!=r[7] goto 34 - 9 Column 1 6 9 0 r[9]=part.P_CONTAINER - 10 Ne 9 10 34 Binary 0 if r[9]!=r[10] goto 34 - 11 BeginSubrtn 11 0 0 0 r[11]=NULL - 12 Null 0 1 0 0 r[1]=NULL - 13 Null 0 13 0 0 r[13]=NULL - 14 Integer 1 14 0 0 r[14]=1; LIMIT counter - 15 IfNot 14 24 0 0 if !r[14] goto 24 - 16 OpenRead 2 10 0 k(17,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B) 0 table=lineitem, root=10, iDb=0 - 17 Rewind 2 24 0 0 Rewind table lineitem - 18 Column 2 1 16 0 r[16]=lineitem.L_PARTKEY - 19 RowId 1 17 0 0 r[17]=part.rowid - 20 Ne 16 17 23 Binary 0 if r[16]!=r[17] goto 23 - 21 Column 2 4 18 0 r[18]=lineitem.L_QUANTITY - 22 AggStep 0 18 13 avg 0 accum=r[13] step(r[18]) - 23 Next 2 18 0 0 - 24 AggFinal 0 13 0 avg 0 accum=r[13] - 25 Copy 13 20 0 0 r[20]=r[13] - 26 Multiply 19 20 12 0 r[12]=r[19]*r[20] - 27 Copy 12 1 0 0 r[1]=r[12] - 28 Return 11 0 1 0 - 29 Column 0 4 22 0 r[22]=lineitem.L_QUANTITY - 30 Copy 1 23 0 0 r[23]=r[1] - 31 Ge 22 23 34 Binary 0 if r[22]>=r[23] goto 34 - 32 Column 0 5 24 0 r[24]=lineitem.L_EXTENDEDPRICE - 33 AggStep 0 24 3 sum 0 accum=r[3] step(r[24]) - 34 Next 0 5 0 0 - 35 AggFinal 0 3 0 sum 0 accum=r[3] - 36 Copy 3 25 0 0 r[25]=r[3] - 37 Divide 25 26 2 0 r[2]=r[25]/r[26] - 38 ResultRow 2 1 0 0 output=r[2] - 39 Halt 0 0 0 0 - 40 Transaction 0 1 8 0 iDb=0 tx_mode=Read - 41 String8 0 7 0 Brand#52 0 r[7]='Brand#52' - 42 String8 0 10 0 LG CAN 0 r[10]='LG CAN' - 43 Real 0 19 0 0.2 0 r[19]=0.2 - 44 Real 0 26 0 7.0 0 r[26]=7 - 45 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 108 0 0 Start at 108 + 1 Once 51 0 0 0 goto 51 + 2 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 3 Null 0 8 0 0 r[8]=NULL + 4 SorterOpen 1 2 0 k(1,B) 0 cursor=1 + 5 Integer 0 5 0 0 r[5]=0; clear group by abort flag + 6 Null 0 6 0 0 r[6]=NULL; initialize group by comparison registers to NULL + 7 Gosub 12 48 0 0 ; go to clear accumulator subroutine + 8 OpenRead 3 10 0 k(17,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B) 0 table=lineitem, root=10, iDb=0 + 9 Rewind 3 15 0 0 Rewind table lineitem + 10 Column 3 1 10 0 r[10]=lineitem.L_PARTKEY + 11 Column 3 4 11 0 r[11]=lineitem.L_QUANTITY + 12 MakeRecord 10 2 9 0 r[9]=mkrec(r[10..11]) + 13 SorterInsert 1 9 0 0 0 key=r[9] + 14 Next 3 10 0 1 + 15 OpenPseudo 2 9 2 0 2 columns in r[9] + 16 SorterSort 1 31 0 0 + 17 SorterData 1 9 2 0 r[9]=data + 18 Column 2 0 13 0 r[13]=pseudo.column 0 + 19 Compare 6 13 1 k(1, Binary) 0 r[6..6]==r[13..13] + 20 Jump 21 25 21 0 ; start new group if comparison is not equal + 21 Gosub 3 35 0 0 ; check if ended group had data, and output if so + 22 Move 13 6 1 0 r[6..6]=r[13..13] + 23 IfPos 5 51 0 0 r[5]>0 -> r[5]-=0, goto 51; check abort flag + 24 Gosub 12 48 0 0 ; goto clear accumulator subroutine + 25 Column 2 1 14 0 r[14]=pseudo.column 1 + 26 AggStep 0 14 8 avg 0 accum=r[8] step(r[14]) + 27 If 4 29 0 0 if r[4] goto 29; don't emit group columns if continuing existing group + 28 Column 2 0 7 0 r[7]=pseudo.column 0 + 29 Integer 1 4 0 0 r[4]=1; indicate data in accumulator + 30 SorterNext 1 17 0 0 + 31 Gosub 3 35 0 0 ; emit row for final group + 32 Goto 0 51 0 0 ; group by finished + 33 Integer 1 5 0 0 r[5]=1 + 34 Return 3 0 0 0 + 35 IfPos 4 37 0 0 r[4]>0 -> r[4]-=0, goto 37; output group by row subroutine start + 36 Return 3 0 0 0 + 37 AggFinal 0 8 0 avg 0 accum=r[8] + 38 Real 0 15 0 0.2 0 r[15]=0.2 + 39 Copy 8 16 0 0 r[16]=r[8] + 40 Multiply 15 16 1 0 r[1]=r[15]*r[16] + 41 Copy 7 2 0 0 r[2]=r[7] + 42 Copy 2 18 0 0 r[18]=r[2] + 43 Copy 1 19 0 0 r[19]=r[1] + 44 Sequence 0 20 0 0 + 45 MakeRecord 18 3 17 0 r[17]=mkrec(r[18..20]); for ephemeral_subquery_t3 + 46 IdxInsert 0 17 0 8 key=r[17] + 47 Return 3 0 0 0 + 48 Null 0 7 8 0 r[7..8]=NULL; clear accumulator subroutine start + 49 Integer 0 4 0 0 r[4]=0 + 50 Return 12 0 0 0 + 51 Null 0 22 0 0 r[22]=NULL + 52 OpenRead 4 10 0 k(17,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B) 0 table=lineitem, root=10, iDb=0 + 53 OpenRead 5 4 0 k(9,B,B,B,B,B,B,B,B,B) 0 table=part, root=4, iDb=0 + 54 Once 70 0 0 0 goto 70 + 55 OpenRead 6 4 0 k(9,B,B,B,B,B,B,B,B,B) 0 table=part, root=4, iDb=0 + 56 Rewind 6 69 0 0 Rewind table part + 57 Column 6 3 25 0 r[25]=part.P_BRAND + 58 Ne 25 26 68 Binary 0 if r[25]!=r[26] goto 68 + 59 Column 6 6 28 0 r[28]=part.P_CONTAINER + 60 Ne 28 29 68 Binary 0 if r[28]!=r[29] goto 68 + 61 RowId 6 23 0 0 r[23]=part.rowid + 62 Affinity 23 1 0 0 r[23..24] = C + 63 RowId 6 30 0 0 r[30]=part.rowid + 64 Column 6 3 31 0 r[31]=part.P_BRAND + 65 Column 6 6 32 0 r[32]=part.P_CONTAINER + 66 HashBuild 6 23 1 r=[2] budget=67108864 payload=r[30]..r[32] 0 + 67 FilterAdd 6 23 1 0 bloom_filter_add(r[23..24]) + 68 Next 6 57 0 0 + 69 HashBuildFinalize 2 0 0 0 + 70 Rewind 4 90 0 0 Rewind table lineitem + 71 Column 4 1 33 0 r[33]=lineitem.L_PARTKEY + 72 Affinity 33 1 0 0 r[33..34] = C + 73 Filter 6 89 33 1 if !bloom_filter(r[33..34]) goto 89 + 74 RowId 4 37 0 0 r[37]=lineitem.rowid + 75 Integer 0 38 0 0 r[38]=0 + 76 HashProbe 2 33 1 r[39]=89 payload=r[34]..r[36] 0 + 77 Copy 34 40 0 0 r[40]=r[34] + 78 SeekGE 0 86 40 0 key=[40..40] + 79 IdxGT 0 86 40 0 key=[40..40] + 80 Column 4 4 42 0 r[42]=lineitem.L_QUANTITY + 81 Column 0 1 43 0 r[43]=ephemeral(ephemeral_subquery_t3).0.2 * avg(l_quantity) + 82 Ge 42 43 85 Binary 0 if r[42]>=r[43] goto 85 + 83 Column 4 5 44 0 r[44]=lineitem.L_EXTENDEDPRICE + 84 AggStep 0 44 22 sum 0 accum=r[22] step(r[44]) + 85 Next 0 79 0 0 + 86 IfPos 38 97 0 0 r[38]>0 -> r[38]-=0, goto 97 + 87 HashNext 2 39 89 payload=r[34]..r[36] 0 + 88 Goto 0 77 0 0 + 89 Next 4 71 0 0 + 90 HashGraceInit 2 0 102 0 hash_table_id=2 + 91 Integer 1 38 0 0 r[38]=1 + 92 HashGraceLoadPart 2 0 101 0 hash_table_id=2 + 93 HashGraceNextProbe 2 33 99 0 hash_table_id=2 keys=r[33]..r[33] probe_rowid_dest=r[37] + 94 SeekRowid 4 37 93 0 if (r[37]!=cursor 4 for table lineitem.rowid) goto 93 + 95 HashProbe 2 33 1 r[39]=93 payload=r[34]..r[36] 0 + 96 Goto 0 77 0 0 + 97 HashNext 2 39 93 payload=r[34]..r[36] 0 + 98 Goto 0 77 0 0 + 99 HashGraceAdvPart 2 0 101 0 hash_table_id=2 + 100 Goto 0 92 0 0 + 101 Integer 0 38 0 0 r[38]=0 + 102 HashClose 2 0 0 0 + 103 AggFinal 0 22 0 sum 0 accum=r[22] + 104 Copy 22 45 0 0 r[45]=r[22] + 105 Divide 45 46 21 0 r[21]=r[45]/r[46] + 106 ResultRow 21 1 0 0 output=r[21] + 107 Halt 0 0 0 0 + 108 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 109 String8 0 26 0 Brand#52 0 r[26]='Brand#52' + 110 String8 0 29 0 LG CAN 0 r[29]='LG CAN' + 111 Real 0 46 0 7.0 0 r[46]=7 + 112 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q19-discounted-revenue.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q19-discounted-revenue.snap index d5ded03e..c8bf91da 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q19-discounted-revenue.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q19-discounted-revenue.snap @@ -50,18 +50,18 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 63 0 0 Start at 63 + 0 Init 0 67 0 0 Start at 67 1 Null 0 2 0 0 r[2]=NULL 2 OpenRead 0 10 0 k(17,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B) 0 table=lineitem, root=10, iDb=0 3 OpenRead 1 4 0 k(9,B,B,B,B,B,B,B,B,B) 0 table=part, root=4, iDb=0 - 4 Rewind 0 59 0 0 Rewind table lineitem + 4 Rewind 0 63 0 0 Rewind table lineitem 5 Column 0 14 3 0 r[3]=lineitem.L_SHIPMODE 6 Eq 3 4 8 Binary 0 if r[3]==r[4] goto 8 - 7 Ne 3 5 58 Binary 0 if r[3]!=r[5] goto 58 + 7 Ne 3 5 62 Binary 0 if r[3]!=r[5] goto 62 8 Column 0 13 7 0 r[7]=lineitem.L_SHIPINSTRUCT - 9 Ne 7 8 58 Binary 0 if r[7]!=r[8] goto 58 + 9 Ne 7 8 62 Binary 0 if r[7]!=r[8] goto 62 10 Column 0 1 9 0 r[9]=lineitem.L_PARTKEY - 11 SeekRowid 1 9 58 0 if (r[9]!=cursor 1 for table part.rowid) goto 58 + 11 SeekRowid 1 9 62 0 if (r[9]!=cursor 1 for table part.rowid) goto 62 12 Column 1 3 11 0 r[11]=part.P_BRAND 13 Ne 11 12 25 Binary 0 if r[11]!=r[12] goto 25 14 Column 1 6 13 0 r[13]=part.P_CONTAINER @@ -89,63 +89,70 @@ addr opcode p1 p2 p3 p4 p5 commen 36 Column 1 5 45 0 r[45]=part.P_SIZE 37 Le 45 46 51 Binary 0 if r[45]<=r[46] goto 51 38 Column 1 3 48 0 r[48]=part.P_BRAND - 39 Ne 48 49 58 Binary 0 if r[48]!=r[49] goto 58 + 39 Ne 48 49 62 Binary 0 if r[48]!=r[49] goto 62 40 Column 1 6 50 0 r[50]=part.P_CONTAINER 41 Eq 50 51 45 Binary 0 if r[50]==r[51] goto 45 42 Eq 50 52 45 Binary 0 if r[50]==r[52] goto 45 43 Eq 50 53 45 Binary 0 if r[50]==r[53] goto 45 - 44 Ne 50 54 58 Binary 0 if r[50]!=r[54] goto 58 + 44 Ne 50 54 62 Binary 0 if r[50]!=r[54] goto 62 45 Column 0 4 56 0 r[56]=lineitem.L_QUANTITY - 46 Lt 56 57 58 Binary 0 if r[56]r[60] goto 58 + 48 Gt 59 60 62 Binary 0 if r[59]>r[60] goto 62 49 Column 1 5 64 0 r[64]=part.P_SIZE - 50 Gt 64 65 58 Binary 0 if r[64]>r[65] goto 58 - 51 Column 1 5 67 0 r[67]=part.P_SIZE - 52 Lt 67 68 58 Binary 0 if r[67]r[65] goto 62 + 51 Column 1 3 66 0 r[66]=part.P_BRAND + 52 Eq 66 67 55 Binary 0 if r[66]==r[67] goto 55 + 53 Eq 66 68 55 Binary 0 if r[66]==r[68] goto 55 + 54 Ne 66 69 62 Binary 0 if r[66]!=r[69] goto 62 + 55 Column 1 5 71 0 r[71]=part.P_SIZE + 56 Lt 71 72 62 Binary 0 if r[71] ( + select + 0.5 * sum(l_quantity) + from + lineitem + where + l_partkey = ps_partkey + and l_suppkey = ps_suppkey + and l_shipdate >= json_extract('"1994-01-01"', '$') + and l_shipdate < '1995-01-01' + ); +info: + statement_type: SELECT + tables: + - lineitem + - partsupp + setup_blocks: + - schema + database: ':memory:' +--- +QUERY PLAN +|--SCAN partsupp +`--SEARCH lineitem USING COVERING INDEX ephemeral_lineitem_t3 (L_PARTKEY=? AND L_SUPPKEY=? AND L_SHIPDATE>=? AND L_SHIPDATE=r[19] goto 20 + 14 ColumnRange 1 1 20 2 0 r[20..21]=lineitem.L_PARTKEY..L_SUPPKEY + 15 Column 1 10 22 0 r[22]=lineitem.L_SHIPDATE + 16 Column 1 4 23 0 r[23]=lineitem.L_QUANTITY + 17 RowId 1 24 0 0 r[24]=lineitem.rowid + 18 MakeRecord 20 5 25 0 r[25]=mkrec(r[20..24]); for ephemeral_lineitem_t3 + 19 IdxInsert 2 25 20 0 key=r[25] + 20 Next 1 12 0 0 + 21 ColumnRange 0 0 26 2 0 r[26..27]=partsupp.PS_PARTKEY..PS_SUPPKEY + 22 String8 0 29 0 "1994-01-01" 0 r[29]='"1994-01-01"' + 23 String8 0 30 0 $ 0 r[30]='$' + 24 Function 0 29 28 json_extract 0 r[28]=func(r[29..30]) + 25 IsNull 28 52 0 0 if (r[28]==NULL) goto 52 + 26 Affinity 26 3 0 0 r[26..29] = C, C, C + 27 SeekGE 2 52 26 0 key=[26..28] + 28 String8 0 28 0 1995-01-01 0 r[28]='1995-01-01' + 29 Affinity 26 3 0 0 r[26..29] = C, C, C + 30 IdxGE 2 52 26 0 key=[26..28] + 31 DeferredSeek 2 1 0 0 + 32 Integer 1 16 0 0 r[16]=1 + 33 RowId 0 11 0 0 r[11]=partsupp.rowid + 34 ColumnRange 0 1 12 2 0 r[12..13]=partsupp.PS_SUPPKEY..PS_AVAILQTY + 35 Column 2 3 14 0 r[14]=ephemeral(ephemeral_lineitem_t3).L_QUANTITY + 36 Compare 5 11 1 k(1, Binary) 0 r[5..5]==r[11..11] + 37 Jump 38 42 38 0 ; start new group if comparison is not equal + 38 Gosub 2 61 0 0 ; check if ended group had data, and output if so + 39 Move 11 5 1 0 r[5..5]=r[11..11] + 40 IfPos 4 75 0 0 r[4]>0 -> r[4]-=0, goto 75; check abort flag + 41 Gosub 15 72 0 0 ; goto clear accumulator subroutine + 42 IdxRowId 2 32 0 0 r[32]=cursor 2 for index ephemeral(ephemeral_lineitem_t3).rowid + 43 Integer 1 31 0 0 r[31]=1 + 44 NotNull 32 46 0 0 r[32]!=NULL -> goto 46 + 45 Integer 0 31 0 0 r[31]=0 + 46 IfNot 31 48 1 0 if !r[31] goto 48 + 47 AggStep 0 14 9 sum 0 accum=r[9] step(r[14]) + 48 If 3 50 0 0 if r[3] goto 50; don't emit group columns if continuing existing group + 49 ColumnRange 0 1 6 2 0 r[6..7]=partsupp.PS_SUPPKEY..PS_AVAILQTY + 50 Integer 1 3 0 0 r[3]=1; indicate data in accumulator + 51 Next 2 30 0 0 + 52 IfPos 16 56 0 0 r[16]>0 -> r[16]-=0, goto 56 + 53 NullRow 1 0 0 0 Set cursor 1 to a (pseudo) NULL row + 54 NullRow 2 0 0 0 Set cursor 2 to a (pseudo) NULL row + 55 Goto 0 32 0 0 + 56 Next 0 8 0 1 + 57 Gosub 2 61 0 0 ; emit row for final group + 58 Goto 0 75 0 0 ; group by finished + 59 Integer 1 4 0 0 r[4]=1 + 60 Return 2 0 0 0 + 61 IfPos 3 63 0 0 r[3]>0 -> r[3]-=0, goto 63; output group by row subroutine start + 62 Return 2 0 0 0 + 63 AggFinal 0 9 0 sum 0 accum=r[9] + 64 Copy 7 34 0 0 r[34]=r[7] + 65 Real 0 36 0 0.5 0 r[36]=0.5 + 66 Copy 9 37 0 0 r[37]=r[9] + 67 Multiply 36 37 35 0 r[35]=r[36]*r[37] + 68 Le 34 35 62 Binary 0 if r[34]<=r[35] goto 62 + 69 Copy 6 1 0 0 r[1]=r[6] + 70 ResultRow 1 1 0 0 output=r[1] + 71 Return 2 0 0 0 + 72 Null 0 6 9 0 r[6..9]=NULL; clear accumulator subroutine start + 73 Integer 0 3 0 0 r[3]=0 + 74 Return 15 0 0 0 + 75 Halt 0 0 0 0 + 76 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 77 String8 0 19 0 1995-01-01 0 r[19]='1995-01-01' + 78 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q20-potential-part-promotion.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q20-potential-part-promotion.snap index 14843f48..2c82840f 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q20-potential-part-promotion.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q20-potential-part-promotion.snap @@ -51,11 +51,214 @@ info: --- QUERY PLAN |--LIST SUBQUERY 1 -| |--LIST SUBQUERY 2 +| |--MATERIALIZE hash build input for partsupp +| | |--LIST SUBQUERY 2 +| | | `--SCAN part +| | `--SEARCH partsupp USING INDEX sqlite_autoindex_partsupp_1 (ps_partkey=?) +| |--LIST SUBQUERY 3 | | `--SCAN part -| |--SEARCH partsupp USING INDEX sqlite_autoindex_partsupp_1 (ps_partkey=?) -| `--CORRELATED SCALAR SUBQUERY 3 -| `--SCAN lineitem +| |--HASH JOIN lineitem +| `--USE SORTER FOR GROUP BY |--SEARCH supplier USING INTEGER PRIMARY KEY (rowid=?) |--SEARCH nation USING INTEGER PRIMARY KEY (rowid=?) `--USE SORTER FOR ORDER BY + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 190 0 0 Start at 190 + 1 Once 167 0 0 0 goto 167 + 2 OpenEphemeral 1 0 0 0 cursor=1 is_table=false + 3 OpenEphemeral 2 1 0 0 cursor=2 is_table=true + 4 Once 15 0 0 0 goto 15 + 5 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 6 OpenRead 3 4 0 k(9,B,B,B,B,B,B,B,B,B) 0 table=part, root=4, iDb=0 + 7 Rewind 3 15 0 0 Rewind table part + 8 Column 3 1 5 0 r[5]=part.P_NAME + 9 Function 1 4 3 like(2) 0 r[3]=func(r[4..5]) + 10 IfNot 3 14 1 0 if !r[3] goto 14 + 11 RowId 3 2 0 0 r[2]=part.rowid + 12 MakeRecord 2 1 6 0 r[6]=mkrec(r[2..2]); for ephemeral_index_where_sub_t5 + 13 IdxInsert 0 6 0 8 key=r[6] + 14 Next 3 8 0 1 + 15 OpenRead 4 6 0 k(6,B,B,B,B,B) 0 table=partsupp, root=6, iDb=0 + 16 OpenRead 5 7 0 k(3,B,B) 0 index=sqlite_autoindex_partsupp_1, root=7, iDb=0 + 17 NullRow 0 0 0 0 Set cursor 0 to a (pseudo) NULL row + 18 Rewind 0 51 0 0 Rewind ephemeral(ephemeral_index_where_sub_t5) + 19 Column 0 0 13 0 r[13]=ephemeral(ephemeral_index_where_sub_t5).P_PARTKEY + 20 IsNull 13 50 0 0 if (r[13]==NULL) goto 50 + 21 SeekGE 5 50 13 0 key=[13..13] + 22 IdxGT 5 50 13 0 key=[13..13] + 23 DeferredSeek 5 4 0 0 + 24 Integer 0 14 0 0 r[14]=0 + 25 Column 5 0 15 0 r[15]=sqlite_autoindex_partsupp_1.ps_partkey + 26 IsNull 15 30 0 0 if (r[15]==NULL) goto 30 + 27 Affinity 15 1 0 0 r[15..16] = C + 28 NotFound 0 30 15 0 if not found goto 30 + 29 Goto 0 36 0 0 + 30 Rewind 0 38 0 0 Rewind ephemeral(ephemeral_index_where_sub_t5) + 31 Column 0 0 16 0 r[16]=ephemeral(ephemeral_index_where_sub_t5).P_PARTKEY + 32 Ne 15 16 34 Binary 0 if r[15]!=r[16] goto 34 + 33 Goto 0 40 0 0 + 34 Next 0 31 0 0 + 35 Goto 0 38 0 0 + 36 Integer 1 14 0 0 r[14]=1 + 37 Goto 0 41 0 0 + 38 Integer 0 14 0 0 r[14]=0 + 39 Goto 0 41 0 0 + 40 Null 0 14 0 0 r[14]=NULL + 41 IfNot 14 49 1 0 if !r[14] goto 49 + 42 ColumnRange 5 0 7 2 0 r[7..8]=sqlite_autoindex_partsupp_1.ps_partkey..ps_suppkey + 43 ColumnRange 5 0 9 2 0 r[9..10]=sqlite_autoindex_partsupp_1.ps_partkey..ps_suppkey + 44 Column 4 2 11 0 r[11]=partsupp.PS_AVAILQTY + 45 RowId 4 12 0 0 r[12]=partsupp.rowid + 46 MakeRecord 7 6 17 0 r[17]=mkrec(r[7..12]); for hash_build_input_t9 + 47 NewRowid 2 18 0 0 r[18]=rowid + 48 Insert 2 17 18 hash_build_input_t9 4 intkey=r[18] data=r[17] + 49 Next 5 22 0 0 + 50 Next 0 19 0 0 + 51 Once 62 0 0 0 goto 62 + 52 OpenEphemeral 0 0 0 0 cursor=0 is_table=false + 53 OpenRead 3 4 0 k(9,B,B,B,B,B,B,B,B,B) 0 table=part, root=4, iDb=0 + 54 Rewind 3 62 0 0 Rewind table part + 55 Column 3 1 22 0 r[22]=part.P_NAME + 56 Function 1 21 20 like(2) 0 r[20]=func(r[21..22]) + 57 IfNot 20 61 1 0 if !r[20] goto 61 + 58 RowId 3 19 0 0 r[19]=part.rowid + 59 MakeRecord 19 1 23 0 r[23]=mkrec(r[19..19]); for ephemeral_index_where_sub_t5 + 60 IdxInsert 0 23 0 8 key=r[23] + 61 Next 3 55 0 1 + 62 Null 0 32 0 0 r[32]=NULL + 63 SorterOpen 6 5 0 k(1,B) 0 cursor=6 + 64 Integer 0 27 0 0 r[27]=0; clear group by abort flag + 65 Null 0 28 0 0 r[28]=NULL; initialize group by comparison registers to NULL + 66 Gosub 39 164 0 0 ; go to clear accumulator subroutine + 67 OpenRead 8 10 0 k(17,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B) 0 table=lineitem, root=10, iDb=0 + 68 Integer 0 40 0 0 r[40]=0 + 69 Once 78 0 0 0 goto 78 + 70 Rewind 2 77 0 0 Rewind ephemeral(hash_build_input_t9) + 71 ColumnRange 2 0 41 2 0 r[41..42]=ephemeral(hash_build_input_t9).key_0..key_1 + 72 Affinity 41 2 0 0 r[41..43] = C, C + 73 ColumnRange 2 2 43 4 0 r[43..46]=ephemeral(hash_build_input_t9).payload_0..payload_3 + 74 HashBuild 2 41 2 r=[4] budget=67108864 payload=r[43]..r[46] 0 + 75 FilterAdd 2 41 2 0 bloom_filter_add(r[41..43]) + 76 Next 2 71 0 0 + 77 HashBuildFinalize 4 0 0 0 + 78 HashResetMatched 4 0 0 0 + 79 Rewind 8 105 0 0 Rewind table lineitem + 80 ColumnRange 8 1 47 2 0 r[47..48]=lineitem.L_PARTKEY..L_SUPPKEY + 81 Affinity 47 2 0 0 r[47..49] = C, C + 82 Filter 2 104 47 2 if !bloom_filter(r[47..49]) goto 104 + 83 RowId 8 53 0 0 r[53]=lineitem.rowid + 84 Integer 0 54 0 0 r[54]=0 + 85 HashProbe 4 47 2 r[55]=104 payload=r[49]..r[52] 0 + 86 Column 8 10 57 0 r[57]=lineitem.L_SHIPDATE + 87 Lt 57 58 101 Binary 0 if r[57]=r[61] goto 101 + 90 HashMarkMatched 4 0 0 0 + 91 Gosub 62 93 0 0 + 92 Goto 0 101 0 0 + 93 Copy 52 34 0 0 r[34]=r[52] + 94 Copy 50 35 0 0 r[35]=r[50] + 95 Copy 51 36 0 0 r[36]=r[51] + 96 Column 8 4 37 0 r[37]=lineitem.L_QUANTITY + 97 RowId 8 38 0 0 r[38]=lineitem.rowid + 98 MakeRecord 34 5 33 0 r[33]=mkrec(r[34..38]) + 99 SorterInsert 6 33 0 0 0 key=r[33] + 100 Return 62 0 0 0 + 101 IfPos 54 117 0 0 r[54]>0 -> r[54]-=0, goto 117 + 102 HashNext 4 55 104 payload=r[49]..r[52] 0 + 103 Goto 0 86 0 0 + 104 Next 8 80 0 0 + 105 NullRow 8 0 0 0 Set cursor 8 to a (pseudo) NULL row + 106 HashScanUnmatched 4 55 110 0 hash_table_id=4 payload=r[49]..r[52] + 107 Gosub 62 93 0 0 + 108 HashNextUnmatched 4 55 110 0 hash_table_id=4 payload=r[49]..r[52] + 109 Goto 0 107 0 0 + 110 HashGraceInit 4 0 127 0 hash_table_id=4 + 111 Integer 1 54 0 0 r[54]=1 + 112 HashGraceLoadPart 4 0 126 0 hash_table_id=4 + 113 HashGraceNextProbe 4 47 119 0 hash_table_id=4 keys=r[47]..r[48] probe_rowid_dest=r[53] + 114 SeekRowid 8 53 113 0 if (r[53]!=cursor 8 for table lineitem.rowid) goto 113 + 115 HashProbe 4 47 2 r[55]=113 payload=r[49]..r[52] 0 + 116 Goto 0 86 0 0 + 117 HashNext 4 55 113 payload=r[49]..r[52] 0 + 118 Goto 0 86 0 0 + 119 NullRow 8 0 0 0 Set cursor 8 to a (pseudo) NULL row + 120 HashScanUnmatched 4 55 124 0 hash_table_id=4 payload=r[49]..r[52] + 121 Gosub 62 93 0 0 + 122 HashNextUnmatched 4 55 124 0 hash_table_id=4 payload=r[49]..r[52] + 123 Goto 0 121 0 0 + 124 HashGraceAdvPart 4 0 126 0 hash_table_id=4 + 125 Goto 0 112 0 0 + 126 Integer 0 54 0 0 r[54]=0 + 127 OpenPseudo 7 33 5 0 5 columns in r[33] + 128 SorterSort 6 148 0 0 + 129 SorterData 6 33 7 0 r[33]=data + 130 Column 7 0 63 0 r[63]=pseudo.column 0 + 131 Compare 28 63 1 k(1, Binary) 0 r[28..28]==r[63..63] + 132 Jump 133 137 133 0 ; start new group if comparison is not equal + 133 Gosub 25 152 0 0 ; check if ended group had data, and output if so + 134 Move 63 28 1 0 r[28..28]=r[63..63] + 135 IfPos 27 167 0 0 r[27]>0 -> r[27]-=0, goto 167; check abort flag + 136 Gosub 39 164 0 0 ; goto clear accumulator subroutine + 137 ColumnRange 7 3 64 2 0 r[64..65]=pseudo.column 3..column 4 + 138 Copy 65 67 0 0 r[67]=r[65] + 139 Integer 1 66 0 0 r[66]=1 + 140 NotNull 67 142 0 0 r[67]!=NULL -> goto 142 + 141 Integer 0 66 0 0 r[66]=0 + 142 IfNot 66 144 1 0 if !r[66] goto 144 + 143 AggStep 0 64 32 sum 0 accum=r[32] step(r[64]) + 144 If 26 146 0 0 if r[26] goto 146; don't emit group columns if continuing existing group + 145 ColumnRange 7 1 29 2 0 r[29..30]=pseudo.column 1..column 2 + 146 Integer 1 26 0 0 r[26]=1; indicate data in accumulator + 147 SorterNext 6 129 0 0 + 148 Gosub 25 152 0 0 ; emit row for final group + 149 Goto 0 167 0 0 ; group by finished + 150 Integer 1 27 0 0 r[27]=1 + 151 Return 25 0 0 0 + 152 IfPos 26 154 0 0 r[26]>0 -> r[26]-=0, goto 154; output group by row subroutine start + 153 Return 25 0 0 0 + 154 AggFinal 0 32 0 sum 0 accum=r[32] + 155 Copy 30 69 0 0 r[69]=r[30] + 156 Real 0 71 0 0.5 0 r[71]=0.5 + 157 Copy 32 72 0 0 r[72]=r[32] + 158 Multiply 71 72 70 0 r[70]=r[71]*r[72] + 159 Le 69 70 153 Binary 0 if r[69]<=r[70] goto 153 + 160 Copy 29 24 0 0 r[24]=r[29] + 161 MakeRecord 24 1 73 0 r[73]=mkrec(r[24..24]); for ephemeral_index_where_sub_t3 + 162 IdxInsert 1 73 0 8 key=r[73] + 163 Return 25 0 0 0 + 164 Null 0 29 32 0 r[29..32]=NULL; clear accumulator subroutine start + 165 Integer 0 26 0 0 r[26]=0 + 166 Return 39 0 0 0 + 167 SorterOpen 10 1 0 k(1,B) 0 cursor=10 + 168 OpenRead 11 5 0 k(7,B,B,B,B,B,B,B) 0 table=supplier, root=5, iDb=0 + 169 OpenRead 12 2 0 k(4,B,B,B,B) 0 table=nation, root=2, iDb=0 + 170 NullRow 1 0 0 0 Set cursor 1 to a (pseudo) NULL row + 171 Rewind 1 183 0 0 Rewind ephemeral(ephemeral_index_where_sub_t3) + 172 Column 1 0 77 0 r[77]=ephemeral(ephemeral_index_where_sub_t3).PS_SUPPKEY + 173 IsNull 77 182 0 0 if (r[77]==NULL) goto 182 + 174 SeekRowid 11 77 182 0 if (r[77]!=cursor 11 for table supplier.rowid) goto 182 + 175 Column 11 3 78 0 r[78]=supplier.S_NATIONKEY + 176 SeekRowid 12 78 182 0 if (r[78]!=cursor 12 for table nation.rowid) goto 182 + 177 Column 12 1 80 0 r[80]=nation.N_NAME + 178 Ne 80 81 182 Binary 0 if r[80]!=r[81] goto 182 + 179 ColumnRange 11 1 82 2 0 r[82..83]=supplier.S_NAME..S_ADDRESS + 180 MakeRecord 82 2 76 0 r[76]=mkrec(r[82..83]) + 181 SorterInsert 10 76 0 0 0 key=r[76] + 182 Next 1 172 0 0 + 183 OpenPseudo 13 76 2 0 2 columns in r[76] + 184 SorterSort 10 189 0 0 + 185 SorterData 10 76 13 0 r[76]=data + 186 ColumnRange 13 0 74 2 0 r[74..75]=pseudo.column 0..column 1 + 187 ResultRow 74 2 0 0 output=r[74..75] + 188 SorterNext 10 185 0 0 + 189 Halt 0 0 0 0 + 190 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 191 String8 0 4 0 frosted% 0 r[4]='frosted%' + 192 String8 0 21 0 frosted% 0 r[21]='frosted%' + 193 String8 0 58 0 1994-01-01 0 r[58]='1994-01-01' + 194 String8 0 61 0 1995-01-01 0 r[61]='1995-01-01' + 195 String8 0 81 0 IRAN 0 r[81]='IRAN' + 196 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q21-suppliers-kept-waiting.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q21-suppliers-kept-waiting.snap index 32e32f5e..f3626f3e 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q21-suppliers-kept-waiting.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q21-suppliers-kept-waiting.snap @@ -51,11 +51,13 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN orders -|--SEARCH l1 USING INDEX sqlite_autoindex_lineitem_1 (l_orderkey=?) -|--SEARCH supplier USING INTEGER PRIMARY KEY (rowid=?) -|--SEARCH nation USING INTEGER PRIMARY KEY (rowid=?) -|--USE SORTER FOR GROUP BY +|--MATERIALIZE hash build input for supplier +| |--HASH JOIN supplier +| `--SCAN nation +|--HASH JOIN lineitem AS l1 +|--SEARCH orders USING INTEGER PRIMARY KEY (rowid=?) |--SEARCH l2 USING INDEX sqlite_autoindex_lineitem_1 (l_orderkey=?) |--SEARCH l3 USING INDEX sqlite_autoindex_lineitem_1 (l_orderkey=?) +|--SCAN nation +|--USE SORTER FOR GROUP BY `--USE SORTER FOR ORDER BY diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q22-global-sales-opportunity.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q22-global-sales-opportunity.snap index e8e8384f..72233d86 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q22-global-sales-opportunity.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q22-global-sales-opportunity.snap @@ -51,6 +51,6 @@ QUERY PLAN |--SCAN custsale | |--SCALAR SUBQUERY 1 | | `--SCAN customer -| |--SCAN customer -| `--SEARCH orders USING INDEX ephemeral_orders_t5 (O_CUSTKEY=?) +| |--HASH JOIN orders +| `--SCAN customer `--USE SORTER FOR GROUP BY diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q4-order-priority.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q4-order-priority.snap index 293562ac..88171080 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q4-order-priority.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q4-order-priority.snap @@ -38,66 +38,65 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 58 0 0 Start at 58 + 0 Init 0 57 0 0 Start at 57 1 Null 0 9 0 0 r[9]=NULL 2 SorterOpen 0 1 0 k(1,B) 0 cursor=0 3 Integer 0 6 0 0 r[6]=0; clear group by abort flag 4 Null 0 7 0 0 r[7]=NULL; initialize group by comparison registers to NULL - 5 Gosub 12 54 0 0 ; go to clear accumulator subroutine + 5 Gosub 12 53 0 0 ; go to clear accumulator subroutine 6 OpenRead 2 9 0 k(9,B,B,B,B,B,B,B,B,B) 0 table=orders, root=9, iDb=0 7 OpenRead 3 10 0 k(17,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B) 0 table=lineitem, root=10, iDb=0 8 OpenRead 4 11 0 k(3,B,B) 0 index=sqlite_autoindex_lineitem_1, root=11, iDb=0 - 9 Rewind 2 28 0 0 Rewind table orders + 9 Rewind 2 27 0 0 Rewind table orders 10 Column 2 4 14 0 r[14]=orders.O_ORDERDATE - 11 Lt 14 15 27 Binary 0 if r[14]=r[18] goto 27 + 13 Ge 17 18 26 Binary 0 if r[17]>=r[18] goto 26 14 RowId 2 19 0 0 r[19]=orders.rowid - 15 SeekGE 4 26 19 0 key=[19..19] - 16 IdxGT 4 26 19 0 key=[19..19] + 15 SeekGE 4 25 19 0 key=[19..19] + 16 IdxGT 4 25 19 0 key=[19..19] 17 DeferredSeek 4 3 0 0 - 18 Column 3 11 21 0 r[21]=lineitem.L_COMMITDATE - 19 Column 3 12 22 0 r[22]=lineitem.L_RECEIPTDATE - 20 Ge 21 22 25 Binary 0 if r[21]>=r[22] goto 25 - 21 Column 2 5 11 0 r[11]=orders.O_ORDERPRIORITY - 22 MakeRecord 11 1 10 0 r[10]=mkrec(r[11..11]) - 23 SorterInsert 0 10 0 0 0 key=r[10] - 24 Goto 0 27 0 0 ; semi-join: early out after first match - 25 Next 4 16 0 0 - 26 Goto 0 27 0 0 ; semi-join: no match, skip outer row - 27 Next 2 10 0 0 - 28 OpenPseudo 1 10 1 0 1 columns in r[10] - 29 SorterSort 0 43 0 0 - 30 SorterData 0 10 1 0 r[10]=data - 31 Column 1 0 23 0 r[23]=pseudo.column 0 - 32 Compare 7 23 1 k(1, Binary) 0 r[7..7]==r[23..23] - 33 Jump 34 38 34 0 ; start new group if comparison is not equal - 34 Gosub 4 47 0 0 ; check if ended group had data, and output if so - 35 Move 23 7 1 0 r[7..7]=r[23..23] - 36 IfPos 6 57 0 0 r[6]>0 -> r[6]-=0, goto 57; check abort flag - 37 Gosub 12 54 0 0 ; goto clear accumulator subroutine - 38 AggStep 0 24 9 count 0 accum=r[9] step(r[24]) - 39 If 5 41 0 0 if r[5] goto 41; don't emit group columns if continuing existing group - 40 Column 1 0 8 0 r[8]=pseudo.column 0 - 41 Integer 1 5 0 0 r[5]=1; indicate data in accumulator - 42 SorterNext 0 30 0 0 - 43 Gosub 4 47 0 0 ; emit row for final group - 44 Goto 0 57 0 0 ; group by finished - 45 Integer 1 6 0 0 r[6]=1 - 46 Return 4 0 0 0 - 47 IfPos 5 49 0 0 r[5]>0 -> r[5]-=0, goto 49; output group by row subroutine start - 48 Return 4 0 0 0 - 49 AggFinal 0 9 0 count 0 accum=r[9] - 50 Copy 8 2 0 0 r[2]=r[8] - 51 Copy 9 3 0 0 r[3]=r[9] - 52 ResultRow 2 2 0 0 output=r[2..3] - 53 Return 4 0 0 0 - 54 Null 0 8 9 0 r[8..9]=NULL; clear accumulator subroutine start - 55 Integer 0 5 0 0 r[5]=0 - 56 Return 12 0 0 0 - 57 Halt 0 0 0 0 - 58 Transaction 0 1 8 0 iDb=0 tx_mode=Read - 59 String8 0 15 0 1997-06-01 0 r[15]='1997-06-01' - 60 String8 0 18 0 1997-09-01 0 r[18]='1997-09-01' - 61 Integer 1 24 0 0 r[24]=1 - 62 Goto 0 1 0 0 + 18 ColumnRange 3 11 21 2 0 r[21..22]=lineitem.L_COMMITDATE..L_RECEIPTDATE + 19 Ge 21 22 24 Binary 0 if r[21]>=r[22] goto 24 + 20 Column 2 5 11 0 r[11]=orders.O_ORDERPRIORITY + 21 MakeRecord 11 1 10 0 r[10]=mkrec(r[11..11]) + 22 SorterInsert 0 10 0 0 0 key=r[10] + 23 Goto 0 26 0 0 ; semi-join: early out after first match + 24 Next 4 16 0 0 + 25 Goto 0 26 0 0 ; semi-join: no match, skip outer row + 26 Next 2 10 0 1 + 27 OpenPseudo 1 10 1 0 1 columns in r[10] + 28 SorterSort 0 42 0 0 + 29 SorterData 0 10 1 0 r[10]=data + 30 Column 1 0 23 0 r[23]=pseudo.column 0 + 31 Compare 7 23 1 k(1, Binary) 0 r[7..7]==r[23..23] + 32 Jump 33 37 33 0 ; start new group if comparison is not equal + 33 Gosub 4 46 0 0 ; check if ended group had data, and output if so + 34 Move 23 7 1 0 r[7..7]=r[23..23] + 35 IfPos 6 56 0 0 r[6]>0 -> r[6]-=0, goto 56; check abort flag + 36 Gosub 12 53 0 0 ; goto clear accumulator subroutine + 37 AggStep 0 24 9 count 0 accum=r[9] step(r[24]) + 38 If 5 40 0 0 if r[5] goto 40; don't emit group columns if continuing existing group + 39 Column 1 0 8 0 r[8]=pseudo.column 0 + 40 Integer 1 5 0 0 r[5]=1; indicate data in accumulator + 41 SorterNext 0 29 0 0 + 42 Gosub 4 46 0 0 ; emit row for final group + 43 Goto 0 56 0 0 ; group by finished + 44 Integer 1 6 0 0 r[6]=1 + 45 Return 4 0 0 0 + 46 IfPos 5 48 0 0 r[5]>0 -> r[5]-=0, goto 48; output group by row subroutine start + 47 Return 4 0 0 0 + 48 AggFinal 0 9 0 count 0 accum=r[9] + 49 Copy 8 2 0 0 r[2]=r[8] + 50 Copy 9 3 0 0 r[3]=r[9] + 51 ResultRow 2 2 0 0 output=r[2..3] + 52 Return 4 0 0 0 + 53 Null 0 8 9 0 r[8..9]=NULL; clear accumulator subroutine start + 54 Integer 0 5 0 0 r[5]=0 + 55 Return 12 0 0 0 + 56 Halt 0 0 0 0 + 57 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 58 String8 0 15 0 1997-06-01 0 r[15]='1997-06-01' + 59 String8 0 18 0 1997-09-01 0 r[18]='1997-09-01' + 60 Integer 1 24 0 0 r[24]=1 + 61 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q5-local-supplier-volume.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q5-local-supplier-volume.snap index f433be2e..c441030e 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q5-local-supplier-volume.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q5-local-supplier-volume.snap @@ -35,10 +35,10 @@ info: --- QUERY PLAN |--SCAN orders -|--SEARCH lineitem USING INDEX sqlite_autoindex_lineitem_1 (l_orderkey=?) -|--SEARCH supplier USING INTEGER PRIMARY KEY (rowid=?) |--SEARCH customer USING INTEGER PRIMARY KEY (rowid=?) |--SEARCH nation USING INTEGER PRIMARY KEY (rowid=?) |--SEARCH region USING INTEGER PRIMARY KEY (rowid=?) +|--SEARCH lineitem USING INDEX sqlite_autoindex_lineitem_1 (l_orderkey=?) +|--SEARCH supplier USING INTEGER PRIMARY KEY (rowid=?) |--USE SORTER FOR GROUP BY `--USE SORTER FOR ORDER BY diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q6-forecasting-revenue.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q6-forecasting-revenue.snap index a4ac4c1b..6e5d4a9a 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q6-forecasting-revenue.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q6-forecasting-revenue.snap @@ -22,38 +22,37 @@ QUERY PLAN `--SCAN lineitem BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 23 0 0 Start at 23 - 1 Null 0 2 0 0 r[2]=NULL - 2 OpenRead 0 10 0 k(17,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B) 0 table=lineitem, root=10, iDb=0 - 3 Rewind 0 19 0 0 Rewind table lineitem - 4 Column 0 10 4 0 r[4]=lineitem.L_SHIPDATE - 5 Lt 4 5 18 Binary 0 if r[4]=r[8] goto 18 - 8 Column 0 6 10 0 r[10]=lineitem.L_DISCOUNT - 9 Lt 10 11 18 Binary 0 if r[10]r[16] goto 18 - 12 Column 0 4 20 0 r[20]=lineitem.L_QUANTITY - 13 Ge 20 21 18 Binary 0 if r[20]>=r[21] goto 18 - 14 Column 0 5 23 0 r[23]=lineitem.L_EXTENDEDPRICE - 15 Column 0 6 24 0 r[24]=lineitem.L_DISCOUNT - 16 Multiply 23 24 22 0 r[22]=r[23]*r[24] - 17 AggStep 0 22 2 sum 0 accum=r[2] step(r[22]) - 18 Next 0 4 0 0 - 19 AggFinal 0 2 0 sum 0 accum=r[2] - 20 Copy 2 1 0 0 r[1]=r[2] - 21 ResultRow 1 1 0 0 output=r[1] - 22 Halt 0 0 0 0 - 23 Transaction 0 1 8 0 iDb=0 tx_mode=Read - 24 String8 0 5 0 1994-01-01 0 r[5]='1994-01-01' - 25 String8 0 8 0 1995-01-01 0 r[8]='1995-01-01' - 26 Real 0 12 0 0.08 0 r[12]=0.08 - 27 Real 0 13 0 0.01 0 r[13]=0.01 - 28 Subtract 12 13 11 0 r[11]=r[12]-r[13] - 29 Real 0 17 0 0.08 0 r[17]=0.08 - 30 Real 0 18 0 0.01 0 r[18]=0.01 - 31 Add 17 18 16 0 r[16]=r[17]+r[18] - 32 Integer 24 21 0 0 r[21]=24 - 33 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 22 0 0 Start at 22 + 1 Null 0 2 0 0 r[2]=NULL + 2 OpenRead 0 10 0 k(17,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B,B) 0 table=lineitem, root=10, iDb=0 + 3 Rewind 0 18 0 0 Rewind table lineitem + 4 Column 0 10 4 0 r[4]=lineitem.L_SHIPDATE + 5 Lt 4 5 17 Binary 0 if r[4]=r[8] goto 17 + 8 Column 0 6 10 0 r[10]=lineitem.L_DISCOUNT + 9 Lt 10 11 17 Binary 0 if r[10]r[16] goto 17 + 12 Column 0 4 20 0 r[20]=lineitem.L_QUANTITY + 13 Ge 20 21 17 Binary 0 if r[20]>=r[21] goto 17 + 14 ColumnRange 0 5 23 2 0 r[23..24]=lineitem.L_EXTENDEDPRICE..L_DISCOUNT + 15 Multiply 23 24 22 0 r[22]=r[23]*r[24] + 16 AggStep 0 22 2 sum 0 accum=r[2] step(r[22]) + 17 Next 0 4 0 1 + 18 AggFinal 0 2 0 sum 0 accum=r[2] + 19 Copy 2 1 0 0 r[1]=r[2] + 20 ResultRow 1 1 0 0 output=r[1] + 21 Halt 0 0 0 0 + 22 Transaction 0 1 8 0 iDb=0 tx_mode=Read + 23 String8 0 5 0 1994-01-01 0 r[5]='1994-01-01' + 24 String8 0 8 0 1995-01-01 0 r[8]='1995-01-01' + 25 Real 0 12 0 0.08 0 r[12]=0.08 + 26 Real 0 13 0 0.01 0 r[13]=0.01 + 27 Subtract 12 13 11 0 r[11]=r[12]-r[13] + 28 Real 0 17 0 0.08 0 r[17]=0.08 + 29 Real 0 18 0 0.01 0 r[18]=0.01 + 30 Add 17 18 16 0 r[16]=r[17]+r[18] + 31 Integer 24 21 0 0 r[21]=24 + 32 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q7-volume-shipping.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q7-volume-shipping.snap index 4945b2d4..81cfd64a 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q7-volume-shipping.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q7-volume-shipping.snap @@ -51,10 +51,12 @@ info: --- QUERY PLAN |--SCAN shipping -| |--SCAN lineitem -| |--SEARCH orders USING INTEGER PRIMARY KEY (rowid=?) -| |--SEARCH customer USING INTEGER PRIMARY KEY (rowid=?) -| |--SEARCH n2 USING INTEGER PRIMARY KEY (rowid=?) +| |--MATERIALIZE hash build input for customer +| | |--HASH JOIN customer +| | `--SCAN nation AS n2 +| |--HASH JOIN orders +| |--SEARCH lineitem USING INDEX sqlite_autoindex_lineitem_1 (l_orderkey=?) | |--SEARCH supplier USING INTEGER PRIMARY KEY (rowid=?) -| `--SEARCH n1 USING INTEGER PRIMARY KEY (rowid=?) +| |--SEARCH n1 USING INTEGER PRIMARY KEY (rowid=?) +| `--SCAN nation AS n2 `--USE SORTER FOR GROUP BY diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q9-product-type-profit.snap b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q9-product-type-profit.snap index 53956ad6..6a5866a3 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q9-product-type-profit.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/snapshots/tpch__q9-product-type-profit.snap @@ -43,10 +43,11 @@ info: --- QUERY PLAN |--SCAN profit -| |--SCAN lineitem -| |--SEARCH part USING INTEGER PRIMARY KEY (rowid=?) -| |--SEARCH supplier USING INTEGER PRIMARY KEY (rowid=?) -| |--SEARCH nation USING INTEGER PRIMARY KEY (rowid=?) +| |--MATERIALIZE hash build input for partsupp +| | |--SCAN part +| | `--SEARCH partsupp USING INDEX sqlite_autoindex_partsupp_1 (ps_partkey=?) +| |--HASH JOIN lineitem | |--SEARCH orders USING INTEGER PRIMARY KEY (rowid=?) -| `--SEARCH partsupp USING INDEX sqlite_autoindex_partsupp_1 (ps_partkey=? AND ps_suppkey=?) +| |--SEARCH supplier USING INTEGER PRIMARY KEY (rowid=?) +| `--SEARCH nation USING INTEGER PRIMARY KEY (rowid=?) `--USE SORTER FOR GROUP BY diff --git a/conformance/sqlite-sqltests/snapshot_tests/tpch/tpch.sqltest b/conformance/sqlite-sqltests/snapshot_tests/tpch/tpch.sqltest index 3d5d64b7..e96af714 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/tpch/tpch.sqltest +++ b/conformance/sqlite-sqltests/snapshot_tests/tpch/tpch.sqltest @@ -727,7 +727,7 @@ snapshot q19-discounted-revenue { # Q20: Potential Part Promotion @setup schema -snapshot-eqp q20-potential-part-promotion { +snapshot q20-potential-part-promotion { select s_name, s_address @@ -767,6 +767,27 @@ snapshot-eqp q20-potential-part-promotion { s_name; } +# A possibly failing bound must not run while the automatic index is built. +@setup schema +snapshot q20-function-bound-stays-outside-autoindex-build { + select + ps_suppkey + from + partsupp + where + ps_availqty > ( + select + 0.5 * sum(l_quantity) + from + lineitem + where + l_partkey = ps_partkey + and l_suppkey = ps_suppkey + and l_shipdate >= json_extract('"1994-01-01"', '$') + and l_shipdate < '1995-01-01' + ); +} + # Q21: Suppliers Who Kept Orders Waiting @setup schema snapshot-eqp q21-suppliers-kept-waiting { diff --git a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__aggregate-windows.snap b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__aggregate-windows.snap index a8ad1ee3..9bab3690 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__aggregate-windows.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__aggregate-windows.snap @@ -27,8 +27,8 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 175 0 0 Start at 175 - 1 InitCoroutine 1 102 2 0 + 0 Init 0 159 0 0 Start at 159 + 1 InitCoroutine 1 100 2 0 2 InitCoroutine 2 14 3 0 3 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=employees, root=2, iDb=0 4 OpenRead 1 4 0 k(2,B) 0 index=idx_employees_department, root=4, iDb=0 @@ -39,7 +39,7 @@ addr opcode p1 p2 p3 p4 p5 comment 9 Column 0 1 5 0 r[5]=employees.name 10 Column 0 3 6 0 r[6]=employees.salary 11 Yield 2 0 0 0 - 12 Next 1 6 0 0 + 12 Next 1 6 0 1 13 EndCoroutine 2 0 0 0 14 OpenEphemeral 2 1 0 0 cursor=2 is_table=true 15 OpenDup 3 2 0 0 new_cursor=3, original_cursor=2 @@ -47,10 +47,10 @@ addr opcode p1 p2 p3 p4 p5 comment 17 Null 0 27 0 0 r[27]=NULL 18 Null 0 28 0 0 r[28]=NULL 19 InitCoroutine 2 0 3 0 - 20 Yield 2 62 0 0 + 20 Yield 2 61 0 0 21 Compare 3 28 1 k(1, Binary) 0 r[3..3]==r[28..28]; compare partition keys to detect new partition 22 Jump 23 26 23 0 - 23 Gosub 29 63 0 0 ; detected new partition + 23 Gosub 29 62 0 0 ; detected new partition 24 Null 0 27 0 0 r[27]=NULL 25 Copy 3 28 0 0 r[28]=r[3] 26 NotNull 27 34 0 0 r[27]!=NULL -> goto 34 @@ -60,11 +60,11 @@ addr opcode p1 p2 p3 p4 p5 comment 30 Insert 3 31 27 buffer_table_window_1 2 intkey=r[27] data=r[31] 31 Rewind 2 33 0 0 Rewind ephemeral(buffer_table_window_1) 32 Rewind 4 33 0 0 Rewind ephemeral(buffer_table_window_1) - 33 Goto 0 61 0 0 + 33 Goto 0 60 0 0 34 MakeRecord 3 4 32 0 r[32]=mkrec(r[3..6]) 35 NewRowid 3 27 0 0 r[27]=rowid 36 Insert 3 32 27 buffer_table_window_1 2 intkey=r[27] data=r[32] - 37 Goto 0 61 0 0 + 37 Goto 0 60 0 0 ; peer of previous row: buffer only, handle at group end 38 AggStep 0 33 15 count 0 accum=r[15] step(r[33]) 39 Column 4 3 34 0 r[34]=ephemeral(buffer_table_window_1).salary 40 AggStep 0 34 16 min(Binary) 0 accum=r[16] step(r[34]) @@ -79,132 +79,116 @@ addr opcode p1 p2 p3 p4 p5 comment 49 AggValue 0 16 20 min 0 accum=r[16] dest=r[20] 50 AggValue 0 17 21 max 0 accum=r[17] dest=r[21] 51 AggValue 0 18 22 avg 0 accum=r[18] dest=r[22] - 52 Column 2 1 23 0 r[23]=ephemeral(buffer_table_window_1).emp_id - 53 Column 2 2 24 0 r[24]=ephemeral(buffer_table_window_1).name - 54 Column 2 0 25 0 r[25]=ephemeral(buffer_table_window_1).department - 55 Column 2 3 26 0 r[26]=ephemeral(buffer_table_window_1).salary - 56 Gosub 30 91 0 0 - 57 Delete 2 0 0 buffer_table_window_1 0 - 58 Next 2 60 0 0 - 59 Goto 0 61 0 0 - 60 Goto 0 52 0 0 - 61 Goto 0 20 0 0 - 62 Null 0 29 0 0 r[29]=NULL; return remaining buffered rows - 63 Rewind 3 88 0 0 Rewind ephemeral(buffer_table_window_1) - 64 AggStep 0 37 15 count 0 accum=r[15] step(r[37]) - 65 Column 4 3 38 0 r[38]=ephemeral(buffer_table_window_1).salary - 66 AggStep 0 38 16 min(Binary) 0 accum=r[16] step(r[38]) - 67 Column 4 3 39 0 r[39]=ephemeral(buffer_table_window_1).salary - 68 AggStep 0 39 17 max(Binary) 0 accum=r[17] step(r[39]) - 69 Column 4 3 40 0 r[40]=ephemeral(buffer_table_window_1).salary - 70 AggStep 0 40 18 avg 0 accum=r[18] step(r[40]) - 71 Next 4 73 0 0 - 72 Goto 0 74 0 0 - 73 Goto 0 64 0 0 - 74 AggValue 0 15 19 count 0 accum=r[15] dest=r[19] - 75 AggValue 0 16 20 min 0 accum=r[16] dest=r[20] - 76 AggValue 0 17 21 max 0 accum=r[17] dest=r[21] - 77 AggValue 0 18 22 avg 0 accum=r[18] dest=r[22] - 78 Column 2 1 23 0 r[23]=ephemeral(buffer_table_window_1).emp_id - 79 Column 2 2 24 0 r[24]=ephemeral(buffer_table_window_1).name - 80 Column 2 0 25 0 r[25]=ephemeral(buffer_table_window_1).department - 81 Column 2 3 26 0 r[26]=ephemeral(buffer_table_window_1).salary - 82 Gosub 30 91 0 0 - 83 Delete 2 0 0 buffer_table_window_1 0 - 84 Next 2 86 0 0 - 85 Goto 0 88 0 0 - 86 Goto 0 78 0 0 - 87 Goto 0 74 0 0 - 88 ResetSorter 2 0 0 0 cursor=2 - 89 Return 29 0 1 0 - 90 Goto 0 101 0 0 - 91 Copy 23 7 0 0 r[7]=r[23] - 92 Copy 24 8 0 0 r[8]=r[24] - 93 Copy 25 9 0 0 r[9]=r[25] - 94 Copy 26 10 0 0 r[10]=r[26] - 95 Copy 19 11 0 0 r[11]=r[19] - 96 Copy 20 12 0 0 r[12]=r[20] - 97 Copy 21 13 0 0 r[13]=r[21] - 98 Copy 22 14 0 0 r[14]=r[22] - 99 Yield 1 0 0 0 - 100 Return 30 0 0 0 - 101 EndCoroutine 1 0 0 0 - 102 OpenEphemeral 5 1 0 0 cursor=5 is_table=true - 103 OpenDup 6 5 0 0 new_cursor=6, original_cursor=5 - 104 OpenDup 7 5 0 0 new_cursor=7, original_cursor=5 - 105 Null 0 60 0 0 r[60]=NULL - 106 InitCoroutine 1 0 2 0 - 107 Yield 1 139 0 0 - 108 NotNull 60 116 0 0 r[60]!=NULL -> goto 116 - 109 Null 0 50 50 0 r[50..50]=NULL; reset accumulator registers - 110 MakeRecord 7 8 63 0 r[63]=mkrec(r[7..14]) - 111 NewRowid 6 60 0 0 r[60]=rowid - 112 Insert 6 63 60 buffer_table_window_0 2 intkey=r[60] data=r[63] - 113 Rewind 5 115 0 0 Rewind ephemeral(buffer_table_window_0) - 114 Rewind 7 115 0 0 Rewind ephemeral(buffer_table_window_0) - 115 Goto 0 138 0 0 - 116 MakeRecord 7 8 64 0 r[64]=mkrec(r[7..14]) - 117 NewRowid 6 60 0 0 r[60]=rowid - 118 Insert 6 64 60 buffer_table_window_0 2 intkey=r[60] data=r[64] - 119 Goto 0 138 0 0 - 120 AggStep 0 65 50 count 0 accum=r[50] step(r[65]) - 121 Next 7 123 0 0 - 122 Goto 0 124 0 0 - 123 Goto 0 120 0 0 - 124 AggValue 0 50 51 count 0 accum=r[50] dest=r[51] - 125 Column 5 0 52 0 r[52]=ephemeral(buffer_table_window_0).emp_id - 126 Column 5 1 53 0 r[53]=ephemeral(buffer_table_window_0).name - 127 Column 5 2 54 0 r[54]=ephemeral(buffer_table_window_0).department - 128 Column 5 3 55 0 r[55]=ephemeral(buffer_table_window_0).salary - 129 Column 5 4 56 0 r[56]=ephemeral(buffer_table_window_0).column 4 - 130 Column 5 5 57 0 r[57]=ephemeral(buffer_table_window_0).column 5 - 131 Column 5 6 58 0 r[58]=ephemeral(buffer_table_window_0).column 6 - 132 Column 5 7 59 0 r[59]=ephemeral(buffer_table_window_0).column 7 - 133 Gosub 62 163 0 0 - 134 Delete 5 0 0 buffer_table_window_0 0 - 135 Next 5 137 0 0 - 136 Goto 0 138 0 0 - 137 Goto 0 125 0 0 - 138 Goto 0 107 0 0 - 139 Null 0 61 0 0 r[61]=NULL; return remaining buffered rows - 140 Rewind 6 160 0 0 Rewind ephemeral(buffer_table_window_0) - 141 AggStep 0 66 50 count 0 accum=r[50] step(r[66]) - 142 Next 7 144 0 0 - 143 Goto 0 145 0 0 - 144 Goto 0 141 0 0 - 145 AggValue 0 50 51 count 0 accum=r[50] dest=r[51] - 146 Column 5 0 52 0 r[52]=ephemeral(buffer_table_window_0).emp_id - 147 Column 5 1 53 0 r[53]=ephemeral(buffer_table_window_0).name - 148 Column 5 2 54 0 r[54]=ephemeral(buffer_table_window_0).department - 149 Column 5 3 55 0 r[55]=ephemeral(buffer_table_window_0).salary - 150 Column 5 4 56 0 r[56]=ephemeral(buffer_table_window_0).column 4 - 151 Column 5 5 57 0 r[57]=ephemeral(buffer_table_window_0).column 5 - 152 Column 5 6 58 0 r[58]=ephemeral(buffer_table_window_0).column 6 - 153 Column 5 7 59 0 r[59]=ephemeral(buffer_table_window_0).column 7 - 154 Gosub 62 163 0 0 - 155 Delete 5 0 0 buffer_table_window_0 0 - 156 Next 5 158 0 0 - 157 Goto 0 160 0 0 - 158 Goto 0 146 0 0 - 159 Goto 0 145 0 0 - 160 ResetSorter 5 0 0 0 cursor=5 - 161 Return 61 0 1 0 - 162 Goto 0 174 0 0 - 163 Copy 52 41 0 0 r[41]=r[52] - 164 Copy 53 42 0 0 r[42]=r[53] - 165 Copy 54 43 0 0 r[43]=r[54] - 166 Copy 55 44 0 0 r[44]=r[55] - 167 Copy 51 45 0 0 r[45]=r[51] - 168 Copy 56 46 0 0 r[46]=r[56] - 169 Copy 57 47 0 0 r[47]=r[57] - 170 Copy 58 48 0 0 r[48]=r[58] - 171 Copy 59 49 0 0 r[49]=r[59] - 172 ResultRow 41 9 0 0 output=r[41..49] - 173 Return 62 0 0 0 - 174 Halt 0 0 0 0 - 175 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 176 Integer 1 33 0 0 r[33]=1 - 177 Integer 1 37 0 0 r[37]=1 - 178 Integer 1 65 0 0 r[65]=1 - 179 Integer 1 66 0 0 r[66]=1 - 180 Goto 0 1 0 0 + 52 ColumnRange 2 1 23 2 0 r[23..24]=ephemeral(buffer_table_window_1).emp_id..name + 53 Column 2 0 25 0 r[25]=ephemeral(buffer_table_window_1).department + 54 Column 2 3 26 0 r[26]=ephemeral(buffer_table_window_1).salary + 55 Gosub 30 89 0 0 + 56 Delete 2 0 0 buffer_table_window_1 0 + 57 Next 2 59 0 0 + 58 Goto 0 60 0 0 + 59 Goto 0 52 0 0 + 60 Goto 0 20 0 0 + 61 Null 0 29 0 0 r[29]=NULL; return remaining buffered rows + 62 Rewind 3 86 0 0 Rewind ephemeral(buffer_table_window_1) + 63 AggStep 0 37 15 count 0 accum=r[15] step(r[37]) + 64 Column 4 3 38 0 r[38]=ephemeral(buffer_table_window_1).salary + 65 AggStep 0 38 16 min(Binary) 0 accum=r[16] step(r[38]) + 66 Column 4 3 39 0 r[39]=ephemeral(buffer_table_window_1).salary + 67 AggStep 0 39 17 max(Binary) 0 accum=r[17] step(r[39]) + 68 Column 4 3 40 0 r[40]=ephemeral(buffer_table_window_1).salary + 69 AggStep 0 40 18 avg 0 accum=r[18] step(r[40]) + 70 Next 4 72 0 0 + 71 Goto 0 73 0 0 + 72 Goto 0 63 0 0 + 73 AggValue 0 15 19 count 0 accum=r[15] dest=r[19] + 74 AggValue 0 16 20 min 0 accum=r[16] dest=r[20] + 75 AggValue 0 17 21 max 0 accum=r[17] dest=r[21] + 76 AggValue 0 18 22 avg 0 accum=r[18] dest=r[22] + 77 ColumnRange 2 1 23 2 0 r[23..24]=ephemeral(buffer_table_window_1).emp_id..name + 78 Column 2 0 25 0 r[25]=ephemeral(buffer_table_window_1).department + 79 Column 2 3 26 0 r[26]=ephemeral(buffer_table_window_1).salary + 80 Gosub 30 89 0 0 + 81 Delete 2 0 0 buffer_table_window_1 0 + 82 Next 2 84 0 0 + 83 Goto 0 86 0 0 + 84 Goto 0 77 0 0 + 85 Goto 0 73 0 0 + 86 ResetSorter 2 0 0 0 cursor=2 + 87 Return 29 0 1 0 + 88 Goto 0 99 0 0 + 89 Copy 23 7 0 0 r[7]=r[23] + 90 Copy 24 8 0 0 r[8]=r[24] + 91 Copy 25 9 0 0 r[9]=r[25] + 92 Copy 26 10 0 0 r[10]=r[26] + 93 Copy 19 11 0 0 r[11]=r[19] + 94 Copy 20 12 0 0 r[12]=r[20] + 95 Copy 21 13 0 0 r[13]=r[21] + 96 Copy 22 14 0 0 r[14]=r[22] + 97 Yield 1 0 0 0 + 98 Return 30 0 0 0 + 99 EndCoroutine 1 0 0 0 + 100 OpenEphemeral 5 1 0 0 cursor=5 is_table=true + 101 OpenDup 6 5 0 0 new_cursor=6, original_cursor=5 + 102 OpenDup 7 5 0 0 new_cursor=7, original_cursor=5 + 103 Null 0 60 0 0 r[60]=NULL + 104 InitCoroutine 1 0 2 0 + 105 Yield 1 130 0 0 + 106 NotNull 60 114 0 0 r[60]!=NULL -> goto 114 + 107 Null 0 50 50 0 r[50..50]=NULL; reset accumulator registers + 108 MakeRecord 7 8 63 0 r[63]=mkrec(r[7..14]) + 109 NewRowid 6 60 0 0 r[60]=rowid + 110 Insert 6 63 60 buffer_table_window_0 2 intkey=r[60] data=r[63] + 111 Rewind 5 113 0 0 Rewind ephemeral(buffer_table_window_0) + 112 Rewind 7 113 0 0 Rewind ephemeral(buffer_table_window_0) + 113 Goto 0 129 0 0 + 114 MakeRecord 7 8 64 0 r[64]=mkrec(r[7..14]) + 115 NewRowid 6 60 0 0 r[60]=rowid + 116 Insert 6 64 60 buffer_table_window_0 2 intkey=r[60] data=r[64] + 117 Goto 0 129 0 0 ; peer of previous row: buffer only, handle at group end + 118 AggStep 0 65 50 count 0 accum=r[50] step(r[65]) + 119 Next 7 121 0 0 + 120 Goto 0 122 0 0 + 121 Goto 0 118 0 0 + 122 AggValue 0 50 51 count 0 accum=r[50] dest=r[51] + 123 ColumnRange 5 0 52 8 0 r[52..59]=ephemeral(buffer_table_window_0).emp_id..column 7 + 124 Gosub 62 147 0 0 + 125 Delete 5 0 0 buffer_table_window_0 0 + 126 Next 5 128 0 0 + 127 Goto 0 129 0 0 + 128 Goto 0 123 0 0 + 129 Goto 0 105 0 0 + 130 Null 0 61 0 0 r[61]=NULL; return remaining buffered rows + 131 Rewind 6 144 0 0 Rewind ephemeral(buffer_table_window_0) + 132 AggStep 0 66 50 count 0 accum=r[50] step(r[66]) + 133 Next 7 135 0 0 + 134 Goto 0 136 0 0 + 135 Goto 0 132 0 0 + 136 AggValue 0 50 51 count 0 accum=r[50] dest=r[51] + 137 ColumnRange 5 0 52 8 0 r[52..59]=ephemeral(buffer_table_window_0).emp_id..column 7 + 138 Gosub 62 147 0 0 + 139 Delete 5 0 0 buffer_table_window_0 0 + 140 Next 5 142 0 0 + 141 Goto 0 144 0 0 + 142 Goto 0 137 0 0 + 143 Goto 0 136 0 0 + 144 ResetSorter 5 0 0 0 cursor=5 + 145 Return 61 0 1 0 + 146 Goto 0 158 0 0 + 147 Copy 52 41 0 0 r[41]=r[52] + 148 Copy 53 42 0 0 r[42]=r[53] + 149 Copy 54 43 0 0 r[43]=r[54] + 150 Copy 55 44 0 0 r[44]=r[55] + 151 Copy 51 45 0 0 r[45]=r[51] + 152 Copy 56 46 0 0 r[46]=r[56] + 153 Copy 57 47 0 0 r[47]=r[57] + 154 Copy 58 48 0 0 r[48]=r[58] + 155 Copy 59 49 0 0 r[49]=r[59] + 156 ResultRow 41 9 0 0 output=r[41..49] + 157 Return 62 0 0 0 + 158 Halt 0 0 0 0 + 159 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 160 Integer 1 33 0 0 r[33]=1 + 161 Integer 1 37 0 0 r[37]=1 + 162 Integer 1 65 0 0 r[65]=1 + 163 Integer 1 66 0 0 r[66]=1 + 164 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__multiple-windows.snap b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__multiple-windows.snap index 939775a9..705bbe89 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__multiple-windows.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__multiple-windows.snap @@ -37,10 +37,10 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 366 0 0 Start at 366 - 1 InitCoroutine 1 291 2 0 - 2 InitCoroutine 2 203 3 0 - 3 InitCoroutine 3 110 4 0 + 0 Init 0 311 0 0 Start at 311 + 1 InitCoroutine 1 248 2 0 + 2 InitCoroutine 2 176 3 0 + 3 InitCoroutine 3 98 4 0 4 InitCoroutine 4 21 5 0 5 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=employees, root=2, iDb=0 6 OpenRead 1 3 0 k(5,B,B,B,B,B) 0 table=sales, root=3, iDb=0 @@ -56,7 +56,7 @@ addr opcode p1 p2 p3 p4 p5 16 RowId 0 9 0 0 r[9]=employees.rowid 17 Column 0 1 10 0 r[10]=employees.name 18 Yield 4 0 0 0 - 19 Next 2 9 0 0 + 19 Next 2 9 0 1 20 EndCoroutine 4 0 0 0 21 SorterOpen 3 1 0 k(1,B) 0 cursor=3 22 OpenEphemeral 4 1 0 0 cursor=4 is_table=true @@ -65,10 +65,10 @@ addr opcode p1 p2 p3 p4 p5 25 Null 0 28 0 0 r[28]=NULL 26 Null 0 29 0 0 r[29]=NULL 27 InitCoroutine 4 0 5 0 - 28 Yield 4 64 0 0 + 28 Yield 4 61 0 0 29 Compare 5 29 1 k(1, Binary) 0 r[5..5]==r[29..29]; compare partition keys to detect new partition 30 Jump 31 34 31 0 - 31 Gosub 30 65 0 0 ; detected new partition + 31 Gosub 30 62 0 0 ; detected new partition 32 Null 0 28 0 0 r[28]=NULL 33 Copy 5 29 0 0 r[29]=r[5] 34 NotNull 28 42 0 0 r[28]!=NULL -> goto 42 @@ -78,11 +78,11 @@ addr opcode p1 p2 p3 p4 p5 38 Insert 5 32 28 buffer_table_window_3 2 intkey=r[28] data=r[32] 39 Rewind 4 41 0 0 Rewind ephemeral(buffer_table_window_3) 40 Rewind 6 41 0 0 Rewind ephemeral(buffer_table_window_3) - 41 Goto 0 63 0 0 + 41 Goto 0 60 0 0 42 MakeRecord 5 6 33 0 r[33]=mkrec(r[5..10]) 43 NewRowid 5 28 0 0 r[28]=rowid 44 Insert 5 33 28 buffer_table_window_3 2 intkey=r[28] data=r[33] - 45 Goto 0 63 0 0 + 45 Goto 0 60 0 0 ; peer of previous row: buffer only, handle at group end 46 Column 6 3 34 0 r[34]=ephemeral(buffer_table_window_3).salary 47 AggStep 0 34 20 avg 0 accum=r[20] step(r[34]) 48 Next 6 50 0 0 @@ -91,317 +91,262 @@ addr opcode p1 p2 p3 p4 p5 51 AggValue 0 20 21 avg 0 accum=r[20] dest=r[21] 52 Column 4 1 22 0 r[22]=ephemeral(buffer_table_window_3).department 53 Column 4 0 23 0 r[23]=ephemeral(buffer_table_window_3).region - 54 Column 4 2 24 0 r[24]=ephemeral(buffer_table_window_3).amount - 55 Column 4 3 25 0 r[25]=ephemeral(buffer_table_window_3).salary - 56 Column 4 4 26 0 r[26]=ephemeral(buffer_table_window_3).emp_id - 57 Column 4 5 27 0 r[27]=ephemeral(buffer_table_window_3).name - 58 Gosub 31 87 0 0 - 59 Delete 4 0 0 buffer_table_window_3 0 - 60 Next 4 62 0 0 - 61 Goto 0 63 0 0 - 62 Goto 0 52 0 0 - 63 Goto 0 28 0 0 - 64 Null 0 30 0 0 r[30]=NULL; return remaining buffered rows - 65 Rewind 5 84 0 0 Rewind ephemeral(buffer_table_window_3) - 66 Column 6 3 35 0 r[35]=ephemeral(buffer_table_window_3).salary - 67 AggStep 0 35 20 avg 0 accum=r[20] step(r[35]) - 68 Next 6 70 0 0 - 69 Goto 0 71 0 0 - 70 Goto 0 66 0 0 - 71 AggValue 0 20 21 avg 0 accum=r[20] dest=r[21] - 72 Column 4 1 22 0 r[22]=ephemeral(buffer_table_window_3).department - 73 Column 4 0 23 0 r[23]=ephemeral(buffer_table_window_3).region - 74 Column 4 2 24 0 r[24]=ephemeral(buffer_table_window_3).amount - 75 Column 4 3 25 0 r[25]=ephemeral(buffer_table_window_3).salary - 76 Column 4 4 26 0 r[26]=ephemeral(buffer_table_window_3).emp_id - 77 Column 4 5 27 0 r[27]=ephemeral(buffer_table_window_3).name - 78 Gosub 31 87 0 0 - 79 Delete 4 0 0 buffer_table_window_3 0 - 80 Next 4 82 0 0 - 81 Goto 0 84 0 0 - 82 Goto 0 72 0 0 - 83 Goto 0 71 0 0 - 84 ResetSorter 4 0 0 0 cursor=4 - 85 Return 30 0 1 0 - 86 Goto 0 97 0 0 - 87 Copy 22 36 0 0 r[36]=r[22] - 88 Copy 23 37 0 0 r[37]=r[23] - 89 Copy 24 38 0 0 r[38]=r[24] - 90 Copy 25 39 0 0 r[39]=r[25] - 91 Copy 26 40 0 0 r[40]=r[26] - 92 Copy 27 41 0 0 r[41]=r[27] - 93 Copy 21 42 0 0 r[42]=r[21] - 94 MakeRecord 36 7 19 0 r[19]=mkrec(r[36..42]) - 95 SorterInsert 3 19 0 0 0 key=r[19] - 96 Return 31 0 0 0 - 97 OpenPseudo 7 19 7 0 7 columns in r[19] - 98 SorterSort 3 109 0 0 - 99 SorterData 3 19 7 0 r[19]=data - 100 Column 7 0 12 0 r[12]=pseudo.column 0 - 101 Column 7 1 13 0 r[13]=pseudo.column 1 - 102 Column 7 2 14 0 r[14]=pseudo.column 2 - 103 Column 7 3 15 0 r[15]=pseudo.column 3 - 104 Column 7 4 16 0 r[16]=pseudo.column 4 - 105 Column 7 5 17 0 r[17]=pseudo.column 5 - 106 Column 7 6 18 0 r[18]=pseudo.column 6 - 107 Yield 3 0 0 0 - 108 SorterNext 3 99 0 0 - 109 EndCoroutine 3 0 0 0 - 110 SorterOpen 8 2 0 k(2,B,-B) 0 cursor=8 - 111 OpenEphemeral 9 1 0 0 cursor=9 is_table=true - 112 OpenDup 10 9 0 0 new_cursor=10, original_cursor=9 - 113 OpenDup 11 9 0 0 new_cursor=11, original_cursor=9 - 114 Null 0 61 0 0 r[61]=NULL - 115 Null 0 62 0 0 r[62]=NULL - 116 InitCoroutine 3 0 4 0 - 117 Yield 3 154 0 0 - 118 Compare 12 62 1 k(1, Binary) 0 r[12..12]==r[62..62]; compare partition keys to detect new partition - 119 Jump 120 123 120 0 - 120 Gosub 63 155 0 0 ; detected new partition - 121 Null 0 61 0 0 r[61]=NULL - 122 Copy 12 62 0 0 r[62]=r[12] - 123 NotNull 61 131 0 0 r[61]!=NULL -> goto 131 - 124 Null 0 52 52 0 r[52..52]=NULL; reset accumulator registers - 125 MakeRecord 12 7 65 0 r[65]=mkrec(r[12..18]) - 126 NewRowid 10 61 0 0 r[61]=rowid - 127 Insert 10 65 61 buffer_table_window_2 2 intkey=r[61] data=r[65] - 128 Rewind 9 130 0 0 Rewind ephemeral(buffer_table_window_2) - 129 Rewind 11 130 0 0 Rewind ephemeral(buffer_table_window_2) - 130 Goto 0 153 0 0 - 131 MakeRecord 12 7 66 0 r[66]=mkrec(r[12..18]) - 132 NewRowid 10 61 0 0 r[61]=rowid - 133 Insert 10 66 61 buffer_table_window_2 2 intkey=r[61] data=r[66] - 134 Goto 0 153 0 0 - 135 Column 11 2 67 0 r[67]=ephemeral(buffer_table_window_2).amount - 136 AggStep 0 67 52 sum 0 accum=r[52] step(r[67]) - 137 Next 11 139 0 0 - 138 Goto 0 140 0 0 - 139 Goto 0 135 0 0 - 140 AggValue 0 52 53 sum 0 accum=r[52] dest=r[53] - 141 Column 9 1 54 0 r[54]=ephemeral(buffer_table_window_2).region - 142 Column 9 2 55 0 r[55]=ephemeral(buffer_table_window_2).amount - 143 Column 9 0 56 0 r[56]=ephemeral(buffer_table_window_2).department - 144 Column 9 3 57 0 r[57]=ephemeral(buffer_table_window_2).salary - 145 Column 9 4 58 0 r[58]=ephemeral(buffer_table_window_2).emp_id - 146 Column 9 5 59 0 r[59]=ephemeral(buffer_table_window_2).name - 147 Column 9 6 60 0 r[60]=ephemeral(buffer_table_window_2).column 6 - 148 Gosub 64 178 0 0 - 149 Delete 9 0 0 buffer_table_window_2 0 - 150 Next 9 152 0 0 - 151 Goto 0 153 0 0 - 152 Goto 0 141 0 0 - 153 Goto 0 117 0 0 - 154 Null 0 63 0 0 r[63]=NULL; return remaining buffered rows - 155 Rewind 10 175 0 0 Rewind ephemeral(buffer_table_window_2) - 156 Column 11 2 68 0 r[68]=ephemeral(buffer_table_window_2).amount - 157 AggStep 0 68 52 sum 0 accum=r[52] step(r[68]) - 158 Next 11 160 0 0 - 159 Goto 0 161 0 0 - 160 Goto 0 156 0 0 - 161 AggValue 0 52 53 sum 0 accum=r[52] dest=r[53] - 162 Column 9 1 54 0 r[54]=ephemeral(buffer_table_window_2).region - 163 Column 9 2 55 0 r[55]=ephemeral(buffer_table_window_2).amount - 164 Column 9 0 56 0 r[56]=ephemeral(buffer_table_window_2).department - 165 Column 9 3 57 0 r[57]=ephemeral(buffer_table_window_2).salary - 166 Column 9 4 58 0 r[58]=ephemeral(buffer_table_window_2).emp_id - 167 Column 9 5 59 0 r[59]=ephemeral(buffer_table_window_2).name - 168 Column 9 6 60 0 r[60]=ephemeral(buffer_table_window_2).column 6 - 169 Gosub 64 178 0 0 - 170 Delete 9 0 0 buffer_table_window_2 0 - 171 Next 9 173 0 0 - 172 Goto 0 175 0 0 - 173 Goto 0 162 0 0 - 174 Goto 0 161 0 0 - 175 ResetSorter 9 0 0 0 cursor=9 - 176 Return 63 0 1 0 - 177 Goto 0 189 0 0 - 178 Copy 54 69 0 0 r[69]=r[54] - 179 Copy 55 70 0 0 r[70]=r[55] - 180 Copy 56 71 0 0 r[71]=r[56] - 181 Copy 57 72 0 0 r[72]=r[57] - 182 Copy 58 73 0 0 r[73]=r[58] - 183 Copy 59 74 0 0 r[74]=r[59] - 184 Copy 53 75 0 0 r[75]=r[53] - 185 Copy 60 76 0 0 r[76]=r[60] - 186 MakeRecord 69 8 51 0 r[51]=mkrec(r[69..76]) - 187 SorterInsert 8 51 0 0 0 key=r[51] - 188 Return 64 0 0 0 - 189 OpenPseudo 12 51 8 0 8 columns in r[51] - 190 SorterSort 8 202 0 0 - 191 SorterData 8 51 12 0 r[51]=data - 192 Column 12 0 43 0 r[43]=pseudo.column 0 - 193 Column 12 1 44 0 r[44]=pseudo.column 1 - 194 Column 12 2 45 0 r[45]=pseudo.column 2 - 195 Column 12 3 46 0 r[46]=pseudo.column 3 - 196 Column 12 4 47 0 r[47]=pseudo.column 4 - 197 Column 12 5 48 0 r[48]=pseudo.column 5 - 198 Column 12 6 49 0 r[49]=pseudo.column 6 - 199 Column 12 7 50 0 r[50]=pseudo.column 7 - 200 Yield 2 0 0 0 - 201 SorterNext 8 191 0 0 - 202 EndCoroutine 2 0 0 0 - 203 SorterOpen 13 2 0 k(2,B,-B) 0 cursor=13 - 204 OpenEphemeral 14 1 0 0 cursor=14 is_table=true - 205 OpenDup 15 14 0 0 new_cursor=15, original_cursor=14 - 206 OpenDup 16 14 0 0 new_cursor=16, original_cursor=14 - 207 Null 0 97 0 0 r[97]=NULL - 208 Null 0 98 0 0 r[98]=NULL - 209 InitCoroutine 2 0 3 0 - 210 Yield 2 243 0 0 - 211 Copy 44 100 0 0 r[100]=r[44] - 212 Compare 43 98 1 k(1, Binary) 0 r[43..43]==r[98..98]; compare partition keys to detect new partition - 213 Jump 214 217 214 0 - 214 Gosub 99 244 0 0 ; detected new partition - 215 Null 0 97 0 0 r[97]=NULL - 216 Copy 43 98 0 0 r[98]=r[43] - 217 NotNull 97 225 0 0 r[97]!=NULL -> goto 225 - 218 Null 0 87 87 0 r[87..87]=NULL; initialize per-cursor peer-reference registers for new partition - 219 MakeRecord 43 8 102 0 r[102]=mkrec(r[43..50]) - 220 NewRowid 15 97 0 0 r[97]=rowid - 221 Insert 15 102 97 buffer_table_window_1 2 intkey=r[97] data=r[102] - 222 Rewind 14 224 0 0 Rewind ephemeral(buffer_table_window_1) - 223 Rewind 16 224 0 0 Rewind ephemeral(buffer_table_window_1) - 224 Goto 0 242 0 0 - 225 MakeRecord 43 8 103 0 r[103]=mkrec(r[43..50]) - 226 NewRowid 15 97 0 0 r[97]=rowid - 227 Insert 15 103 97 buffer_table_window_1 2 intkey=r[97] data=r[103] - 228 AggStep 0 0 87 row_number 0 accum=r[87] step(r[0]) - 229 Next 16 230 0 0 - 230 AggValue 0 87 88 row_number 0 accum=r[87] dest=r[88] - 231 Column 14 2 89 0 r[89]=ephemeral(buffer_table_window_1).department - 232 Column 14 3 90 0 r[90]=ephemeral(buffer_table_window_1).salary - 233 Column 14 4 91 0 r[91]=ephemeral(buffer_table_window_1).emp_id - 234 Column 14 5 92 0 r[92]=ephemeral(buffer_table_window_1).name - 235 Column 14 1 93 0 r[93]=ephemeral(buffer_table_window_1).amount - 236 Column 14 0 94 0 r[94]=ephemeral(buffer_table_window_1).region - 237 Column 14 6 95 0 r[95]=ephemeral(buffer_table_window_1).column 6 - 238 Column 14 7 96 0 r[96]=ephemeral(buffer_table_window_1).column 7 - 239 Gosub 101 264 0 0 - 240 Delete 14 0 0 buffer_table_window_1 0 - 241 Next 14 242 0 0 - 242 Goto 0 210 0 0 - 243 Null 0 99 0 0 r[99]=NULL; return remaining buffered rows - 244 Rewind 15 261 0 0 Rewind ephemeral(buffer_table_window_1) - 245 AggStep 0 0 87 row_number 0 accum=r[87] step(r[0]) - 246 Next 16 247 0 0 - 247 AggValue 0 87 88 row_number 0 accum=r[87] dest=r[88] - 248 Column 14 2 89 0 r[89]=ephemeral(buffer_table_window_1).department - 249 Column 14 3 90 0 r[90]=ephemeral(buffer_table_window_1).salary - 250 Column 14 4 91 0 r[91]=ephemeral(buffer_table_window_1).emp_id - 251 Column 14 5 92 0 r[92]=ephemeral(buffer_table_window_1).name - 252 Column 14 1 93 0 r[93]=ephemeral(buffer_table_window_1).amount - 253 Column 14 0 94 0 r[94]=ephemeral(buffer_table_window_1).region - 254 Column 14 6 95 0 r[95]=ephemeral(buffer_table_window_1).column 6 - 255 Column 14 7 96 0 r[96]=ephemeral(buffer_table_window_1).column 7 - 256 Gosub 101 264 0 0 - 257 Delete 14 0 0 buffer_table_window_1 0 - 258 Next 14 260 0 0 - 259 Goto 0 261 0 0 - 260 Goto 0 247 0 0 - 261 ResetSorter 14 0 0 0 cursor=14 - 262 Return 99 0 1 0 - 263 Goto 0 276 0 0 - 264 Copy 89 104 0 0 r[104]=r[89] - 265 Copy 90 105 0 0 r[105]=r[90] - 266 Copy 91 106 0 0 r[106]=r[91] - 267 Copy 92 107 0 0 r[107]=r[92] - 268 Copy 93 108 0 0 r[108]=r[93] - 269 Copy 94 109 0 0 r[109]=r[94] - 270 Copy 88 110 0 0 r[110]=r[88] - 271 Copy 95 111 0 0 r[111]=r[95] - 272 Copy 96 112 0 0 r[112]=r[96] - 273 MakeRecord 104 9 86 0 r[86]=mkrec(r[104..112]) - 274 SorterInsert 13 86 0 0 0 key=r[86] - 275 Return 101 0 0 0 - 276 OpenPseudo 17 86 9 0 9 columns in r[86] - 277 SorterSort 13 290 0 0 - 278 SorterData 13 86 17 0 r[86]=data - 279 Column 17 0 77 0 r[77]=pseudo.column 0 - 280 Column 17 1 78 0 r[78]=pseudo.column 1 - 281 Column 17 2 79 0 r[79]=pseudo.column 2 - 282 Column 17 3 80 0 r[80]=pseudo.column 3 - 283 Column 17 4 81 0 r[81]=pseudo.column 4 - 284 Column 17 5 82 0 r[82]=pseudo.column 5 - 285 Column 17 6 83 0 r[83]=pseudo.column 6 - 286 Column 17 7 84 0 r[84]=pseudo.column 7 - 287 Column 17 8 85 0 r[85]=pseudo.column 8 - 288 Yield 1 0 0 0 - 289 SorterNext 13 278 0 0 - 290 EndCoroutine 1 0 0 0 - 291 OpenEphemeral 18 1 0 0 cursor=18 is_table=true - 292 OpenDup 19 18 0 0 new_cursor=19, original_cursor=18 - 293 OpenDup 20 18 0 0 new_cursor=20, original_cursor=18 - 294 Null 0 134 0 0 r[134]=NULL - 295 Null 0 135 0 0 r[135]=NULL - 296 InitCoroutine 1 0 2 0 - 297 Yield 1 331 0 0 - 298 Copy 78 137 0 0 r[137]=r[78] - 299 Compare 77 135 1 k(1, Binary) 0 r[77..77]==r[135..135]; compare partition keys to detect new partition - 300 Jump 301 304 301 0 - 301 Gosub 136 332 0 0 ; detected new partition - 302 Null 0 134 0 0 r[134]=NULL - 303 Copy 77 135 0 0 r[135]=r[77] - 304 NotNull 134 312 0 0 r[134]!=NULL -> goto 312 - 305 Null 0 123 123 0 r[123..123]=NULL; initialize per-cursor peer-reference registers for new partition - 306 MakeRecord 77 9 139 0 r[139]=mkrec(r[77..85]) - 307 NewRowid 19 134 0 0 r[134]=rowid - 308 Insert 19 139 134 buffer_table_window_0 2 intkey=r[134] data=r[139] - 309 Rewind 18 311 0 0 Rewind ephemeral(buffer_table_window_0) - 310 Rewind 20 311 0 0 Rewind ephemeral(buffer_table_window_0) - 311 Goto 0 330 0 0 - 312 MakeRecord 77 9 140 0 r[140]=mkrec(r[77..85]) - 313 NewRowid 19 134 0 0 r[134]=rowid - 314 Insert 19 140 134 buffer_table_window_0 2 intkey=r[134] data=r[140] - 315 AggStep 0 0 123 row_number 0 accum=r[123] step(r[0]) - 316 Next 20 317 0 0 - 317 AggValue 0 123 124 row_number 0 accum=r[123] dest=r[124] - 318 Column 18 2 125 0 r[125]=ephemeral(buffer_table_window_0).emp_id - 319 Column 18 3 126 0 r[126]=ephemeral(buffer_table_window_0).name - 320 Column 18 0 127 0 r[127]=ephemeral(buffer_table_window_0).department - 321 Column 18 1 128 0 r[128]=ephemeral(buffer_table_window_0).salary - 322 Column 18 4 129 0 r[129]=ephemeral(buffer_table_window_0).amount - 323 Column 18 5 130 0 r[130]=ephemeral(buffer_table_window_0).region - 324 Column 18 6 131 0 r[131]=ephemeral(buffer_table_window_0).column 6 - 325 Column 18 7 132 0 r[132]=ephemeral(buffer_table_window_0).column 7 - 326 Column 18 8 133 0 r[133]=ephemeral(buffer_table_window_0).column 8 - 327 Gosub 138 353 0 0 - 328 Delete 18 0 0 buffer_table_window_0 0 - 329 Next 18 330 0 0 - 330 Goto 0 297 0 0 - 331 Null 0 136 0 0 r[136]=NULL; return remaining buffered rows - 332 Rewind 19 350 0 0 Rewind ephemeral(buffer_table_window_0) - 333 AggStep 0 0 123 row_number 0 accum=r[123] step(r[0]) - 334 Next 20 335 0 0 - 335 AggValue 0 123 124 row_number 0 accum=r[123] dest=r[124] - 336 Column 18 2 125 0 r[125]=ephemeral(buffer_table_window_0).emp_id - 337 Column 18 3 126 0 r[126]=ephemeral(buffer_table_window_0).name - 338 Column 18 0 127 0 r[127]=ephemeral(buffer_table_window_0).department - 339 Column 18 1 128 0 r[128]=ephemeral(buffer_table_window_0).salary - 340 Column 18 4 129 0 r[129]=ephemeral(buffer_table_window_0).amount - 341 Column 18 5 130 0 r[130]=ephemeral(buffer_table_window_0).region - 342 Column 18 6 131 0 r[131]=ephemeral(buffer_table_window_0).column 6 - 343 Column 18 7 132 0 r[132]=ephemeral(buffer_table_window_0).column 7 - 344 Column 18 8 133 0 r[133]=ephemeral(buffer_table_window_0).column 8 - 345 Gosub 138 353 0 0 - 346 Delete 18 0 0 buffer_table_window_0 0 - 347 Next 18 349 0 0 - 348 Goto 0 350 0 0 - 349 Goto 0 335 0 0 - 350 ResetSorter 18 0 0 0 cursor=18 - 351 Return 136 0 1 0 - 352 Goto 0 365 0 0 - 353 Copy 125 113 0 0 r[113]=r[125] - 354 Copy 126 114 0 0 r[114]=r[126] - 355 Copy 127 115 0 0 r[115]=r[127] - 356 Copy 128 116 0 0 r[116]=r[128] - 357 Copy 129 117 0 0 r[117]=r[129] - 358 Copy 130 118 0 0 r[118]=r[130] - 359 Copy 124 119 0 0 r[119]=r[124] - 360 Copy 131 120 0 0 r[120]=r[131] - 361 Copy 132 121 0 0 r[121]=r[132] - 362 Copy 133 122 0 0 r[122]=r[133] - 363 ResultRow 113 10 0 0 output=r[113..122] - 364 Return 138 0 0 0 - 365 Halt 0 0 0 0 - 366 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 367 Goto 0 1 0 0 + 54 ColumnRange 4 2 24 4 0 r[24..27]=ephemeral(buffer_table_window_3).amount..name + 55 Gosub 31 81 0 0 + 56 Delete 4 0 0 buffer_table_window_3 0 + 57 Next 4 59 0 0 + 58 Goto 0 60 0 0 + 59 Goto 0 52 0 0 + 60 Goto 0 28 0 0 + 61 Null 0 30 0 0 r[30]=NULL; return remaining buffered rows + 62 Rewind 5 78 0 0 Rewind ephemeral(buffer_table_window_3) + 63 Column 6 3 35 0 r[35]=ephemeral(buffer_table_window_3).salary + 64 AggStep 0 35 20 avg 0 accum=r[20] step(r[35]) + 65 Next 6 67 0 0 + 66 Goto 0 68 0 0 + 67 Goto 0 63 0 0 + 68 AggValue 0 20 21 avg 0 accum=r[20] dest=r[21] + 69 Column 4 1 22 0 r[22]=ephemeral(buffer_table_window_3).department + 70 Column 4 0 23 0 r[23]=ephemeral(buffer_table_window_3).region + 71 ColumnRange 4 2 24 4 0 r[24..27]=ephemeral(buffer_table_window_3).amount..name + 72 Gosub 31 81 0 0 + 73 Delete 4 0 0 buffer_table_window_3 0 + 74 Next 4 76 0 0 + 75 Goto 0 78 0 0 + 76 Goto 0 69 0 0 + 77 Goto 0 68 0 0 + 78 ResetSorter 4 0 0 0 cursor=4 + 79 Return 30 0 1 0 + 80 Goto 0 91 0 0 + 81 Copy 22 36 0 0 r[36]=r[22] + 82 Copy 23 37 0 0 r[37]=r[23] + 83 Copy 24 38 0 0 r[38]=r[24] + 84 Copy 25 39 0 0 r[39]=r[25] + 85 Copy 26 40 0 0 r[40]=r[26] + 86 Copy 27 41 0 0 r[41]=r[27] + 87 Copy 21 42 0 0 r[42]=r[21] + 88 MakeRecord 36 7 19 0 r[19]=mkrec(r[36..42]) + 89 SorterInsert 3 19 0 0 0 key=r[19] + 90 Return 31 0 0 0 + 91 OpenPseudo 7 19 7 0 7 columns in r[19] + 92 SorterSort 3 97 0 0 + 93 SorterData 3 19 7 0 r[19]=data + 94 ColumnRange 7 0 12 7 0 r[12..18]=pseudo.column 0..column 6 + 95 Yield 3 0 0 0 + 96 SorterNext 3 93 0 0 + 97 EndCoroutine 3 0 0 0 + 98 SorterOpen 8 2 0 k(2,B,-B) 0 cursor=8 + 99 OpenEphemeral 9 1 0 0 cursor=9 is_table=true + 100 OpenDup 10 9 0 0 new_cursor=10, original_cursor=9 + 101 OpenDup 11 9 0 0 new_cursor=11, original_cursor=9 + 102 Null 0 61 0 0 r[61]=NULL + 103 Null 0 62 0 0 r[62]=NULL + 104 InitCoroutine 3 0 4 0 + 105 Yield 3 138 0 0 + 106 Compare 12 62 1 k(1, Binary) 0 r[12..12]==r[62..62]; compare partition keys to detect new partition + 107 Jump 108 111 108 0 + 108 Gosub 63 139 0 0 ; detected new partition + 109 Null 0 61 0 0 r[61]=NULL + 110 Copy 12 62 0 0 r[62]=r[12] + 111 NotNull 61 119 0 0 r[61]!=NULL -> goto 119 + 112 Null 0 52 52 0 r[52..52]=NULL; reset accumulator registers + 113 MakeRecord 12 7 65 0 r[65]=mkrec(r[12..18]) + 114 NewRowid 10 61 0 0 r[61]=rowid + 115 Insert 10 65 61 buffer_table_window_2 2 intkey=r[61] data=r[65] + 116 Rewind 9 118 0 0 Rewind ephemeral(buffer_table_window_2) + 117 Rewind 11 118 0 0 Rewind ephemeral(buffer_table_window_2) + 118 Goto 0 137 0 0 + 119 MakeRecord 12 7 66 0 r[66]=mkrec(r[12..18]) + 120 NewRowid 10 61 0 0 r[61]=rowid + 121 Insert 10 66 61 buffer_table_window_2 2 intkey=r[61] data=r[66] + 122 Goto 0 137 0 0 ; peer of previous row: buffer only, handle at group end + 123 Column 11 2 67 0 r[67]=ephemeral(buffer_table_window_2).amount + 124 AggStep 0 67 52 sum 0 accum=r[52] step(r[67]) + 125 Next 11 127 0 0 + 126 Goto 0 128 0 0 + 127 Goto 0 123 0 0 + 128 AggValue 0 52 53 sum 0 accum=r[52] dest=r[53] + 129 ColumnRange 9 1 54 2 0 r[54..55]=ephemeral(buffer_table_window_2).region..amount + 130 Column 9 0 56 0 r[56]=ephemeral(buffer_table_window_2).department + 131 ColumnRange 9 3 57 4 0 r[57..60]=ephemeral(buffer_table_window_2).salary..column 6 + 132 Gosub 64 158 0 0 + 133 Delete 9 0 0 buffer_table_window_2 0 + 134 Next 9 136 0 0 + 135 Goto 0 137 0 0 + 136 Goto 0 129 0 0 + 137 Goto 0 105 0 0 + 138 Null 0 63 0 0 r[63]=NULL; return remaining buffered rows + 139 Rewind 10 155 0 0 Rewind ephemeral(buffer_table_window_2) + 140 Column 11 2 68 0 r[68]=ephemeral(buffer_table_window_2).amount + 141 AggStep 0 68 52 sum 0 accum=r[52] step(r[68]) + 142 Next 11 144 0 0 + 143 Goto 0 145 0 0 + 144 Goto 0 140 0 0 + 145 AggValue 0 52 53 sum 0 accum=r[52] dest=r[53] + 146 ColumnRange 9 1 54 2 0 r[54..55]=ephemeral(buffer_table_window_2).region..amount + 147 Column 9 0 56 0 r[56]=ephemeral(buffer_table_window_2).department + 148 ColumnRange 9 3 57 4 0 r[57..60]=ephemeral(buffer_table_window_2).salary..column 6 + 149 Gosub 64 158 0 0 + 150 Delete 9 0 0 buffer_table_window_2 0 + 151 Next 9 153 0 0 + 152 Goto 0 155 0 0 + 153 Goto 0 146 0 0 + 154 Goto 0 145 0 0 + 155 ResetSorter 9 0 0 0 cursor=9 + 156 Return 63 0 1 0 + 157 Goto 0 169 0 0 + 158 Copy 54 69 0 0 r[69]=r[54] + 159 Copy 55 70 0 0 r[70]=r[55] + 160 Copy 56 71 0 0 r[71]=r[56] + 161 Copy 57 72 0 0 r[72]=r[57] + 162 Copy 58 73 0 0 r[73]=r[58] + 163 Copy 59 74 0 0 r[74]=r[59] + 164 Copy 53 75 0 0 r[75]=r[53] + 165 Copy 60 76 0 0 r[76]=r[60] + 166 MakeRecord 69 8 51 0 r[51]=mkrec(r[69..76]) + 167 SorterInsert 8 51 0 0 0 key=r[51] + 168 Return 64 0 0 0 + 169 OpenPseudo 12 51 8 0 8 columns in r[51] + 170 SorterSort 8 175 0 0 + 171 SorterData 8 51 12 0 r[51]=data + 172 ColumnRange 12 0 43 8 0 r[43..50]=pseudo.column 0..column 7 + 173 Yield 2 0 0 0 + 174 SorterNext 8 171 0 0 + 175 EndCoroutine 2 0 0 0 + 176 SorterOpen 13 2 0 k(2,B,-B) 0 cursor=13 + 177 OpenEphemeral 14 1 0 0 cursor=14 is_table=true + 178 OpenDup 15 14 0 0 new_cursor=15, original_cursor=14 + 179 OpenDup 16 14 0 0 new_cursor=16, original_cursor=14 + 180 Null 0 97 0 0 r[97]=NULL + 181 Null 0 98 0 0 r[98]=NULL + 182 InitCoroutine 2 0 3 0 + 183 Yield 2 212 0 0 + 184 Copy 44 100 0 0 r[100]=r[44] + 185 Compare 43 98 1 k(1, Binary) 0 r[43..43]==r[98..98]; compare partition keys to detect new partition + 186 Jump 187 190 187 0 + 187 Gosub 99 213 0 0 ; detected new partition + 188 Null 0 97 0 0 r[97]=NULL + 189 Copy 43 98 0 0 r[98]=r[43] + 190 NotNull 97 198 0 0 r[97]!=NULL -> goto 198 + 191 Null 0 87 87 0 r[87..87]=NULL; reset accumulator registers + 192 MakeRecord 43 8 102 0 r[102]=mkrec(r[43..50]) + 193 NewRowid 15 97 0 0 r[97]=rowid + 194 Insert 15 102 97 buffer_table_window_1 2 intkey=r[97] data=r[102] + 195 Rewind 14 197 0 0 Rewind ephemeral(buffer_table_window_1) + 196 Rewind 16 197 0 0 Rewind ephemeral(buffer_table_window_1) + 197 Goto 0 211 0 0 + 198 MakeRecord 43 8 103 0 r[103]=mkrec(r[43..50]) + 199 NewRowid 15 97 0 0 r[97]=rowid + 200 Insert 15 103 97 buffer_table_window_1 2 intkey=r[97] data=r[103] + 201 AggStep 0 0 87 row_number 0 accum=r[87] step(r[0]) + 202 Next 16 203 0 0 + 203 AggValue 0 87 88 row_number 0 accum=r[87] dest=r[88] + 204 ColumnRange 14 2 89 4 0 r[89..92]=ephemeral(buffer_table_window_1).department..name + 205 Column 14 1 93 0 r[93]=ephemeral(buffer_table_window_1).amount + 206 Column 14 0 94 0 r[94]=ephemeral(buffer_table_window_1).region + 207 ColumnRange 14 6 95 2 0 r[95..96]=ephemeral(buffer_table_window_1).column 6..column 7 + 208 Gosub 101 229 0 0 + 209 Delete 14 0 0 buffer_table_window_1 0 + 210 Next 14 211 0 0 + 211 Goto 0 183 0 0 + 212 Null 0 99 0 0 r[99]=NULL; return remaining buffered rows + 213 Rewind 15 226 0 0 Rewind ephemeral(buffer_table_window_1) + 214 AggStep 0 0 87 row_number 0 accum=r[87] step(r[0]) + 215 Next 16 216 0 0 + 216 AggValue 0 87 88 row_number 0 accum=r[87] dest=r[88] + 217 ColumnRange 14 2 89 4 0 r[89..92]=ephemeral(buffer_table_window_1).department..name + 218 Column 14 1 93 0 r[93]=ephemeral(buffer_table_window_1).amount + 219 Column 14 0 94 0 r[94]=ephemeral(buffer_table_window_1).region + 220 ColumnRange 14 6 95 2 0 r[95..96]=ephemeral(buffer_table_window_1).column 6..column 7 + 221 Gosub 101 229 0 0 + 222 Delete 14 0 0 buffer_table_window_1 0 + 223 Next 14 225 0 0 + 224 Goto 0 226 0 0 + 225 Goto 0 216 0 0 + 226 ResetSorter 14 0 0 0 cursor=14 + 227 Return 99 0 1 0 + 228 Goto 0 241 0 0 + 229 Copy 89 104 0 0 r[104]=r[89] + 230 Copy 90 105 0 0 r[105]=r[90] + 231 Copy 91 106 0 0 r[106]=r[91] + 232 Copy 92 107 0 0 r[107]=r[92] + 233 Copy 93 108 0 0 r[108]=r[93] + 234 Copy 94 109 0 0 r[109]=r[94] + 235 Copy 88 110 0 0 r[110]=r[88] + 236 Copy 95 111 0 0 r[111]=r[95] + 237 Copy 96 112 0 0 r[112]=r[96] + 238 MakeRecord 104 9 86 0 r[86]=mkrec(r[104..112]) + 239 SorterInsert 13 86 0 0 0 key=r[86] + 240 Return 101 0 0 0 + 241 OpenPseudo 17 86 9 0 9 columns in r[86] + 242 SorterSort 13 247 0 0 + 243 SorterData 13 86 17 0 r[86]=data + 244 ColumnRange 17 0 77 9 0 r[77..85]=pseudo.column 0..column 8 + 245 Yield 1 0 0 0 + 246 SorterNext 13 243 0 0 + 247 EndCoroutine 1 0 0 0 + 248 OpenEphemeral 18 1 0 0 cursor=18 is_table=true + 249 OpenDup 19 18 0 0 new_cursor=19, original_cursor=18 + 250 OpenDup 20 18 0 0 new_cursor=20, original_cursor=18 + 251 Null 0 134 0 0 r[134]=NULL + 252 Null 0 135 0 0 r[135]=NULL + 253 InitCoroutine 1 0 2 0 + 254 Yield 1 282 0 0 + 255 Copy 78 137 0 0 r[137]=r[78] + 256 Compare 77 135 1 k(1, Binary) 0 r[77..77]==r[135..135]; compare partition keys to detect new partition + 257 Jump 258 261 258 0 + 258 Gosub 136 283 0 0 ; detected new partition + 259 Null 0 134 0 0 r[134]=NULL + 260 Copy 77 135 0 0 r[135]=r[77] + 261 NotNull 134 269 0 0 r[134]!=NULL -> goto 269 + 262 Null 0 123 123 0 r[123..123]=NULL; reset accumulator registers + 263 MakeRecord 77 9 139 0 r[139]=mkrec(r[77..85]) + 264 NewRowid 19 134 0 0 r[134]=rowid + 265 Insert 19 139 134 buffer_table_window_0 2 intkey=r[134] data=r[139] + 266 Rewind 18 268 0 0 Rewind ephemeral(buffer_table_window_0) + 267 Rewind 20 268 0 0 Rewind ephemeral(buffer_table_window_0) + 268 Goto 0 281 0 0 + 269 MakeRecord 77 9 140 0 r[140]=mkrec(r[77..85]) + 270 NewRowid 19 134 0 0 r[134]=rowid + 271 Insert 19 140 134 buffer_table_window_0 2 intkey=r[134] data=r[140] + 272 AggStep 0 0 123 row_number 0 accum=r[123] step(r[0]) + 273 Next 20 274 0 0 + 274 AggValue 0 123 124 row_number 0 accum=r[123] dest=r[124] + 275 ColumnRange 18 2 125 2 0 r[125..126]=ephemeral(buffer_table_window_0).emp_id..name + 276 ColumnRange 18 0 127 2 0 r[127..128]=ephemeral(buffer_table_window_0).department..salary + 277 ColumnRange 18 4 129 5 0 r[129..133]=ephemeral(buffer_table_window_0).amount..column 8 + 278 Gosub 138 298 0 0 + 279 Delete 18 0 0 buffer_table_window_0 0 + 280 Next 18 281 0 0 + 281 Goto 0 254 0 0 + 282 Null 0 136 0 0 r[136]=NULL; return remaining buffered rows + 283 Rewind 19 295 0 0 Rewind ephemeral(buffer_table_window_0) + 284 AggStep 0 0 123 row_number 0 accum=r[123] step(r[0]) + 285 Next 20 286 0 0 + 286 AggValue 0 123 124 row_number 0 accum=r[123] dest=r[124] + 287 ColumnRange 18 2 125 2 0 r[125..126]=ephemeral(buffer_table_window_0).emp_id..name + 288 ColumnRange 18 0 127 2 0 r[127..128]=ephemeral(buffer_table_window_0).department..salary + 289 ColumnRange 18 4 129 5 0 r[129..133]=ephemeral(buffer_table_window_0).amount..column 8 + 290 Gosub 138 298 0 0 + 291 Delete 18 0 0 buffer_table_window_0 0 + 292 Next 18 294 0 0 + 293 Goto 0 295 0 0 + 294 Goto 0 286 0 0 + 295 ResetSorter 18 0 0 0 cursor=18 + 296 Return 136 0 1 0 + 297 Goto 0 310 0 0 + 298 Copy 125 113 0 0 r[113]=r[125] + 299 Copy 126 114 0 0 r[114]=r[126] + 300 Copy 127 115 0 0 r[115]=r[127] + 301 Copy 128 116 0 0 r[116]=r[128] + 302 Copy 129 117 0 0 r[117]=r[129] + 303 Copy 130 118 0 0 r[118]=r[130] + 304 Copy 124 119 0 0 r[119]=r[124] + 305 Copy 131 120 0 0 r[120]=r[131] + 306 Copy 132 121 0 0 r[121]=r[132] + 307 Copy 133 122 0 0 r[122]=r[133] + 308 ResultRow 113 10 0 0 output=r[113..122] + 309 Return 138 0 0 0 + 310 Halt 0 0 0 0 + 311 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 312 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__percent-of-total.snap b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__percent-of-total.snap index e45ad59e..1eb5c0da 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__percent-of-total.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__percent-of-total.snap @@ -24,8 +24,8 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 157 0 0 Start at 157 - 1 InitCoroutine 1 83 2 0 + 0 Init 0 147 0 0 Start at 147 + 1 InitCoroutine 1 81 2 0 2 InitCoroutine 2 14 3 0 3 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=employees, root=2, iDb=0 4 OpenRead 1 4 0 k(2,B) 0 index=idx_employees_department, root=4, iDb=0 @@ -36,7 +36,7 @@ addr opcode p1 p2 p3 p4 p5 comment 9 Column 0 1 5 0 r[5]=employees.name 10 Column 0 3 6 0 r[6]=employees.salary 11 Yield 2 0 0 0 - 12 Next 1 6 0 0 + 12 Next 1 6 0 1 13 EndCoroutine 2 0 0 0 14 OpenEphemeral 2 1 0 0 cursor=2 is_table=true 15 OpenDup 3 2 0 0 new_cursor=3, original_cursor=2 @@ -44,10 +44,10 @@ addr opcode p1 p2 p3 p4 p5 comment 17 Null 0 18 0 0 r[18]=NULL 18 Null 0 19 0 0 r[19]=NULL 19 InitCoroutine 2 0 3 0 - 20 Yield 2 54 0 0 + 20 Yield 2 53 0 0 21 Compare 3 19 1 k(1, Binary) 0 r[3..3]==r[19..19]; compare partition keys to detect new partition 22 Jump 23 26 23 0 - 23 Gosub 20 55 0 0 ; detected new partition + 23 Gosub 20 54 0 0 ; detected new partition 24 Null 0 18 0 0 r[18]=NULL 25 Copy 3 19 0 0 r[19]=r[3] 26 NotNull 18 34 0 0 r[18]!=NULL -> goto 34 @@ -57,131 +57,121 @@ addr opcode p1 p2 p3 p4 p5 comment 30 Insert 3 22 18 buffer_table_window_1 2 intkey=r[18] data=r[22] 31 Rewind 2 33 0 0 Rewind ephemeral(buffer_table_window_1) 32 Rewind 4 33 0 0 Rewind ephemeral(buffer_table_window_1) - 33 Goto 0 53 0 0 + 33 Goto 0 52 0 0 34 MakeRecord 3 4 23 0 r[23]=mkrec(r[3..6]) 35 NewRowid 3 18 0 0 r[18]=rowid 36 Insert 3 23 18 buffer_table_window_1 2 intkey=r[18] data=r[23] - 37 Goto 0 53 0 0 + 37 Goto 0 52 0 0 ; peer of previous row: buffer only, handle at group end 38 Column 4 3 24 0 r[24]=ephemeral(buffer_table_window_1).salary 39 AggStep 0 24 12 sum 0 accum=r[12] step(r[24]) 40 Next 4 42 0 0 41 Goto 0 43 0 0 42 Goto 0 38 0 0 43 AggValue 0 12 13 sum 0 accum=r[12] dest=r[13] - 44 Column 2 1 14 0 r[14]=ephemeral(buffer_table_window_1).emp_id - 45 Column 2 2 15 0 r[15]=ephemeral(buffer_table_window_1).name - 46 Column 2 0 16 0 r[16]=ephemeral(buffer_table_window_1).department - 47 Column 2 3 17 0 r[17]=ephemeral(buffer_table_window_1).salary - 48 Gosub 21 75 0 0 - 49 Delete 2 0 0 buffer_table_window_1 0 - 50 Next 2 52 0 0 - 51 Goto 0 53 0 0 - 52 Goto 0 44 0 0 - 53 Goto 0 20 0 0 - 54 Null 0 20 0 0 r[20]=NULL; return remaining buffered rows - 55 Rewind 3 72 0 0 Rewind ephemeral(buffer_table_window_1) - 56 Column 4 3 25 0 r[25]=ephemeral(buffer_table_window_1).salary - 57 AggStep 0 25 12 sum 0 accum=r[12] step(r[25]) - 58 Next 4 60 0 0 - 59 Goto 0 61 0 0 - 60 Goto 0 56 0 0 - 61 AggValue 0 12 13 sum 0 accum=r[12] dest=r[13] - 62 Column 2 1 14 0 r[14]=ephemeral(buffer_table_window_1).emp_id - 63 Column 2 2 15 0 r[15]=ephemeral(buffer_table_window_1).name - 64 Column 2 0 16 0 r[16]=ephemeral(buffer_table_window_1).department - 65 Column 2 3 17 0 r[17]=ephemeral(buffer_table_window_1).salary - 66 Gosub 21 75 0 0 - 67 Delete 2 0 0 buffer_table_window_1 0 - 68 Next 2 70 0 0 - 69 Goto 0 72 0 0 - 70 Goto 0 62 0 0 - 71 Goto 0 61 0 0 - 72 ResetSorter 2 0 0 0 cursor=2 - 73 Return 20 0 1 0 - 74 Goto 0 82 0 0 - 75 Copy 14 7 0 0 r[7]=r[14] - 76 Copy 15 8 0 0 r[8]=r[15] - 77 Copy 16 9 0 0 r[9]=r[16] - 78 Copy 17 10 0 0 r[10]=r[17] - 79 Copy 13 11 0 0 r[11]=r[13] - 80 Yield 1 0 0 0 - 81 Return 21 0 0 0 - 82 EndCoroutine 1 0 0 0 - 83 OpenEphemeral 5 1 0 0 cursor=5 is_table=true - 84 OpenDup 6 5 0 0 new_cursor=6, original_cursor=5 - 85 OpenDup 7 5 0 0 new_cursor=7, original_cursor=5 - 86 Null 0 39 0 0 r[39]=NULL - 87 InitCoroutine 1 0 2 0 - 88 Yield 1 118 0 0 - 89 NotNull 39 97 0 0 r[39]!=NULL -> goto 97 - 90 Null 0 32 32 0 r[32..32]=NULL; reset accumulator registers - 91 MakeRecord 7 5 42 0 r[42]=mkrec(r[7..11]) - 92 NewRowid 6 39 0 0 r[39]=rowid - 93 Insert 6 42 39 buffer_table_window_0 2 intkey=r[39] data=r[42] - 94 Rewind 5 96 0 0 Rewind ephemeral(buffer_table_window_0) - 95 Rewind 7 96 0 0 Rewind ephemeral(buffer_table_window_0) - 96 Goto 0 117 0 0 - 97 MakeRecord 7 5 43 0 r[43]=mkrec(r[7..11]) - 98 NewRowid 6 39 0 0 r[39]=rowid - 99 Insert 6 43 39 buffer_table_window_0 2 intkey=r[39] data=r[43] - 100 Goto 0 117 0 0 - 101 Column 7 3 44 0 r[44]=ephemeral(buffer_table_window_0).salary - 102 AggStep 0 44 32 sum 0 accum=r[32] step(r[44]) - 103 Next 7 105 0 0 - 104 Goto 0 106 0 0 - 105 Goto 0 101 0 0 - 106 AggValue 0 32 33 sum 0 accum=r[32] dest=r[33] - 107 Column 5 0 34 0 r[34]=ephemeral(buffer_table_window_0).emp_id - 108 Column 5 1 35 0 r[35]=ephemeral(buffer_table_window_0).name - 109 Column 5 2 36 0 r[36]=ephemeral(buffer_table_window_0).department - 110 Column 5 3 37 0 r[37]=ephemeral(buffer_table_window_0).salary - 111 Column 5 4 38 0 r[38]=ephemeral(buffer_table_window_0).column 4 - 112 Gosub 41 140 0 0 - 113 Delete 5 0 0 buffer_table_window_0 0 - 114 Next 5 116 0 0 - 115 Goto 0 117 0 0 - 116 Goto 0 107 0 0 - 117 Goto 0 88 0 0 - 118 Null 0 40 0 0 r[40]=NULL; return remaining buffered rows - 119 Rewind 6 137 0 0 Rewind ephemeral(buffer_table_window_0) - 120 Column 7 3 45 0 r[45]=ephemeral(buffer_table_window_0).salary - 121 AggStep 0 45 32 sum 0 accum=r[32] step(r[45]) - 122 Next 7 124 0 0 - 123 Goto 0 125 0 0 - 124 Goto 0 120 0 0 - 125 AggValue 0 32 33 sum 0 accum=r[32] dest=r[33] - 126 Column 5 0 34 0 r[34]=ephemeral(buffer_table_window_0).emp_id - 127 Column 5 1 35 0 r[35]=ephemeral(buffer_table_window_0).name - 128 Column 5 2 36 0 r[36]=ephemeral(buffer_table_window_0).department - 129 Column 5 3 37 0 r[37]=ephemeral(buffer_table_window_0).salary - 130 Column 5 4 38 0 r[38]=ephemeral(buffer_table_window_0).column 4 - 131 Gosub 41 140 0 0 - 132 Delete 5 0 0 buffer_table_window_0 0 - 133 Next 5 135 0 0 - 134 Goto 0 137 0 0 - 135 Goto 0 126 0 0 - 136 Goto 0 125 0 0 - 137 ResetSorter 5 0 0 0 cursor=5 - 138 Return 40 0 1 0 - 139 Goto 0 156 0 0 - 140 Copy 34 26 0 0 r[26]=r[34] - 141 Copy 35 27 0 0 r[27]=r[35] - 142 Copy 36 28 0 0 r[28]=r[36] - 143 Copy 37 29 0 0 r[29]=r[37] - 144 Copy 37 48 0 0 r[48]=r[37] - 145 Cast 48 0 0 0 affinity(r[48]=Real) - 146 Multiply 48 49 46 0 r[46]=r[48]*r[49] - 147 Copy 33 47 0 0 r[47]=r[33] - 148 Divide 46 47 30 0 r[30]=r[46]/r[47] - 149 Copy 37 52 0 0 r[52]=r[37] - 150 Cast 52 0 0 0 affinity(r[52]=Real) - 151 Multiply 52 53 50 0 r[50]=r[52]*r[53] - 152 Copy 38 51 0 0 r[51]=r[38] - 153 Divide 50 51 31 0 r[31]=r[50]/r[51] - 154 ResultRow 26 6 0 0 output=r[26..31] - 155 Return 41 0 0 0 - 156 Halt 0 0 0 0 - 157 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 158 Real 0 49 0 100.0 0 r[49]=100 - 159 Real 0 53 0 100.0 0 r[53]=100 - 160 Goto 0 1 0 0 + 44 ColumnRange 2 1 14 2 0 r[14..15]=ephemeral(buffer_table_window_1).emp_id..name + 45 Column 2 0 16 0 r[16]=ephemeral(buffer_table_window_1).department + 46 Column 2 3 17 0 r[17]=ephemeral(buffer_table_window_1).salary + 47 Gosub 21 73 0 0 + 48 Delete 2 0 0 buffer_table_window_1 0 + 49 Next 2 51 0 0 + 50 Goto 0 52 0 0 + 51 Goto 0 44 0 0 + 52 Goto 0 20 0 0 + 53 Null 0 20 0 0 r[20]=NULL; return remaining buffered rows + 54 Rewind 3 70 0 0 Rewind ephemeral(buffer_table_window_1) + 55 Column 4 3 25 0 r[25]=ephemeral(buffer_table_window_1).salary + 56 AggStep 0 25 12 sum 0 accum=r[12] step(r[25]) + 57 Next 4 59 0 0 + 58 Goto 0 60 0 0 + 59 Goto 0 55 0 0 + 60 AggValue 0 12 13 sum 0 accum=r[12] dest=r[13] + 61 ColumnRange 2 1 14 2 0 r[14..15]=ephemeral(buffer_table_window_1).emp_id..name + 62 Column 2 0 16 0 r[16]=ephemeral(buffer_table_window_1).department + 63 Column 2 3 17 0 r[17]=ephemeral(buffer_table_window_1).salary + 64 Gosub 21 73 0 0 + 65 Delete 2 0 0 buffer_table_window_1 0 + 66 Next 2 68 0 0 + 67 Goto 0 70 0 0 + 68 Goto 0 61 0 0 + 69 Goto 0 60 0 0 + 70 ResetSorter 2 0 0 0 cursor=2 + 71 Return 20 0 1 0 + 72 Goto 0 80 0 0 + 73 Copy 14 7 0 0 r[7]=r[14] + 74 Copy 15 8 0 0 r[8]=r[15] + 75 Copy 16 9 0 0 r[9]=r[16] + 76 Copy 17 10 0 0 r[10]=r[17] + 77 Copy 13 11 0 0 r[11]=r[13] + 78 Yield 1 0 0 0 + 79 Return 21 0 0 0 + 80 EndCoroutine 1 0 0 0 + 81 OpenEphemeral 5 1 0 0 cursor=5 is_table=true + 82 OpenDup 6 5 0 0 new_cursor=6, original_cursor=5 + 83 OpenDup 7 5 0 0 new_cursor=7, original_cursor=5 + 84 Null 0 39 0 0 r[39]=NULL + 85 InitCoroutine 1 0 2 0 + 86 Yield 1 112 0 0 + 87 NotNull 39 95 0 0 r[39]!=NULL -> goto 95 + 88 Null 0 32 32 0 r[32..32]=NULL; reset accumulator registers + 89 MakeRecord 7 5 42 0 r[42]=mkrec(r[7..11]) + 90 NewRowid 6 39 0 0 r[39]=rowid + 91 Insert 6 42 39 buffer_table_window_0 2 intkey=r[39] data=r[42] + 92 Rewind 5 94 0 0 Rewind ephemeral(buffer_table_window_0) + 93 Rewind 7 94 0 0 Rewind ephemeral(buffer_table_window_0) + 94 Goto 0 111 0 0 + 95 MakeRecord 7 5 43 0 r[43]=mkrec(r[7..11]) + 96 NewRowid 6 39 0 0 r[39]=rowid + 97 Insert 6 43 39 buffer_table_window_0 2 intkey=r[39] data=r[43] + 98 Goto 0 111 0 0 ; peer of previous row: buffer only, handle at group end + 99 Column 7 3 44 0 r[44]=ephemeral(buffer_table_window_0).salary + 100 AggStep 0 44 32 sum 0 accum=r[32] step(r[44]) + 101 Next 7 103 0 0 + 102 Goto 0 104 0 0 + 103 Goto 0 99 0 0 + 104 AggValue 0 32 33 sum 0 accum=r[32] dest=r[33] + 105 ColumnRange 5 0 34 5 0 r[34..38]=ephemeral(buffer_table_window_0).emp_id..column 4 + 106 Gosub 41 130 0 0 + 107 Delete 5 0 0 buffer_table_window_0 0 + 108 Next 5 110 0 0 + 109 Goto 0 111 0 0 + 110 Goto 0 105 0 0 + 111 Goto 0 86 0 0 + 112 Null 0 40 0 0 r[40]=NULL; return remaining buffered rows + 113 Rewind 6 127 0 0 Rewind ephemeral(buffer_table_window_0) + 114 Column 7 3 45 0 r[45]=ephemeral(buffer_table_window_0).salary + 115 AggStep 0 45 32 sum 0 accum=r[32] step(r[45]) + 116 Next 7 118 0 0 + 117 Goto 0 119 0 0 + 118 Goto 0 114 0 0 + 119 AggValue 0 32 33 sum 0 accum=r[32] dest=r[33] + 120 ColumnRange 5 0 34 5 0 r[34..38]=ephemeral(buffer_table_window_0).emp_id..column 4 + 121 Gosub 41 130 0 0 + 122 Delete 5 0 0 buffer_table_window_0 0 + 123 Next 5 125 0 0 + 124 Goto 0 127 0 0 + 125 Goto 0 120 0 0 + 126 Goto 0 119 0 0 + 127 ResetSorter 5 0 0 0 cursor=5 + 128 Return 40 0 1 0 + 129 Goto 0 146 0 0 + 130 Copy 34 26 0 0 r[26]=r[34] + 131 Copy 35 27 0 0 r[27]=r[35] + 132 Copy 36 28 0 0 r[28]=r[36] + 133 Copy 37 29 0 0 r[29]=r[37] + 134 Copy 37 48 0 0 r[48]=r[37] + 135 Cast 48 0 0 0 affinity(r[48]=Real) + 136 Multiply 48 49 46 0 r[46]=r[48]*r[49] + 137 Copy 33 47 0 0 r[47]=r[33] + 138 Divide 46 47 30 0 r[30]=r[46]/r[47] + 139 Copy 37 52 0 0 r[52]=r[37] + 140 Cast 52 0 0 0 affinity(r[52]=Real) + 141 Multiply 52 53 50 0 r[50]=r[52]*r[53] + 142 Copy 38 51 0 0 r[51]=r[38] + 143 Divide 50 51 31 0 r[31]=r[50]/r[51] + 144 ResultRow 26 6 0 0 output=r[26..31] + 145 Return 41 0 0 0 + 146 Halt 0 0 0 0 + 147 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 148 Real 0 49 0 100.0 0 r[49]=100 + 149 Real 0 53 0 100.0 0 r[53]=100 + 150 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__row-number-basic.snap b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__row-number-basic.snap index 0f4d2bf5..fd9fdeb6 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__row-number-basic.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__row-number-basic.snap @@ -22,72 +22,67 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 67 0 0 Start at 67 - 1 InitCoroutine 1 13 2 0 + 0 Init 0 62 0 0 Start at 62 + 1 InitCoroutine 1 12 2 0 2 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=employees, root=2, iDb=0 3 OpenRead 1 5 0 k(2,B) 0 index=idx_employees_salary, root=5, iDb=0 - 4 Last 1 12 0 0 + 4 Last 1 11 0 0 5 DeferredSeek 1 0 0 0 6 Column 1 0 2 0 r[2]=idx_employees_salary.salary 7 IdxRowId 1 3 0 0 r[3]=cursor 1 for index idx_employees_salary.rowid - 8 Column 0 1 4 0 r[4]=employees.name - 9 Column 0 2 5 0 r[5]=employees.department - 10 Yield 1 0 0 0 - 11 Prev 1 5 0 0 - 12 EndCoroutine 1 0 0 0 - 13 OpenEphemeral 2 1 0 0 cursor=2 is_table=true - 14 OpenDup 3 2 0 0 new_cursor=3, original_cursor=2 - 15 OpenDup 4 2 0 0 new_cursor=4, original_cursor=2 - 16 Null 0 17 0 0 r[17]=NULL - 17 InitCoroutine 1 0 2 0 - 18 Yield 1 42 0 0 - 19 Copy 2 19 0 0 r[19]=r[2] - 20 NotNull 17 28 0 0 r[17]!=NULL -> goto 28 - 21 Null 0 11 11 0 r[11..11]=NULL; initialize per-cursor peer-reference registers for new partition - 22 MakeRecord 2 4 21 0 r[21]=mkrec(r[2..5]) - 23 NewRowid 3 17 0 0 r[17]=rowid - 24 Insert 3 21 17 buffer_table_window_0 2 intkey=r[17] data=r[21] - 25 Rewind 2 27 0 0 Rewind ephemeral(buffer_table_window_0) - 26 Rewind 4 27 0 0 Rewind ephemeral(buffer_table_window_0) - 27 Goto 0 41 0 0 - 28 MakeRecord 2 4 22 0 r[22]=mkrec(r[2..5]) - 29 NewRowid 3 17 0 0 r[17]=rowid - 30 Insert 3 22 17 buffer_table_window_0 2 intkey=r[17] data=r[22] - 31 AggStep 0 0 11 row_number 0 accum=r[11] step(r[0]) - 32 Next 4 33 0 0 - 33 AggValue 0 11 12 row_number 0 accum=r[11] dest=r[12] - 34 Column 2 1 13 0 r[13]=ephemeral(buffer_table_window_0).emp_id - 35 Column 2 2 14 0 r[14]=ephemeral(buffer_table_window_0).name - 36 Column 2 3 15 0 r[15]=ephemeral(buffer_table_window_0).department - 37 Column 2 0 16 0 r[16]=ephemeral(buffer_table_window_0).salary - 38 Gosub 20 59 0 0 - 39 Delete 2 0 0 buffer_table_window_0 0 - 40 Next 2 41 0 0 - 41 Goto 0 18 0 0 - 42 Null 0 18 0 0 r[18]=NULL; return remaining buffered rows - 43 Rewind 3 56 0 0 Rewind ephemeral(buffer_table_window_0) - 44 AggStep 0 0 11 row_number 0 accum=r[11] step(r[0]) - 45 Next 4 46 0 0 - 46 AggValue 0 11 12 row_number 0 accum=r[11] dest=r[12] - 47 Column 2 1 13 0 r[13]=ephemeral(buffer_table_window_0).emp_id - 48 Column 2 2 14 0 r[14]=ephemeral(buffer_table_window_0).name - 49 Column 2 3 15 0 r[15]=ephemeral(buffer_table_window_0).department - 50 Column 2 0 16 0 r[16]=ephemeral(buffer_table_window_0).salary - 51 Gosub 20 59 0 0 - 52 Delete 2 0 0 buffer_table_window_0 0 - 53 Next 2 55 0 0 - 54 Goto 0 56 0 0 - 55 Goto 0 46 0 0 - 56 ResetSorter 2 0 0 0 cursor=2 - 57 Return 18 0 1 0 - 58 Goto 0 66 0 0 - 59 Copy 13 6 0 0 r[6]=r[13] - 60 Copy 14 7 0 0 r[7]=r[14] - 61 Copy 15 8 0 0 r[8]=r[15] - 62 Copy 16 9 0 0 r[9]=r[16] - 63 Copy 12 10 0 0 r[10]=r[12] - 64 ResultRow 6 5 0 0 output=r[6..10] - 65 Return 20 0 0 0 - 66 Halt 0 0 0 0 - 67 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 68 Goto 0 1 0 0 + 8 ColumnRange 0 1 4 2 0 r[4..5]=employees.name..department + 9 Yield 1 0 0 0 + 10 Prev 1 5 0 1 + 11 EndCoroutine 1 0 0 0 + 12 OpenEphemeral 2 1 0 0 cursor=2 is_table=true + 13 OpenDup 3 2 0 0 new_cursor=3, original_cursor=2 + 14 OpenDup 4 2 0 0 new_cursor=4, original_cursor=2 + 15 Null 0 17 0 0 r[17]=NULL + 16 InitCoroutine 1 0 2 0 + 17 Yield 1 39 0 0 + 18 Copy 2 19 0 0 r[19]=r[2] + 19 NotNull 17 27 0 0 r[17]!=NULL -> goto 27 + 20 Null 0 11 11 0 r[11..11]=NULL; reset accumulator registers + 21 MakeRecord 2 4 21 0 r[21]=mkrec(r[2..5]) + 22 NewRowid 3 17 0 0 r[17]=rowid + 23 Insert 3 21 17 buffer_table_window_0 2 intkey=r[17] data=r[21] + 24 Rewind 2 26 0 0 Rewind ephemeral(buffer_table_window_0) + 25 Rewind 4 26 0 0 Rewind ephemeral(buffer_table_window_0) + 26 Goto 0 38 0 0 + 27 MakeRecord 2 4 22 0 r[22]=mkrec(r[2..5]) + 28 NewRowid 3 17 0 0 r[17]=rowid + 29 Insert 3 22 17 buffer_table_window_0 2 intkey=r[17] data=r[22] + 30 AggStep 0 0 11 row_number 0 accum=r[11] step(r[0]) + 31 Next 4 32 0 0 + 32 AggValue 0 11 12 row_number 0 accum=r[11] dest=r[12] + 33 ColumnRange 2 1 13 3 0 r[13..15]=ephemeral(buffer_table_window_0).emp_id..department + 34 Column 2 0 16 0 r[16]=ephemeral(buffer_table_window_0).salary + 35 Gosub 20 54 0 0 + 36 Delete 2 0 0 buffer_table_window_0 0 + 37 Next 2 38 0 0 + 38 Goto 0 17 0 0 + 39 Null 0 18 0 0 r[18]=NULL; return remaining buffered rows + 40 Rewind 3 51 0 0 Rewind ephemeral(buffer_table_window_0) + 41 AggStep 0 0 11 row_number 0 accum=r[11] step(r[0]) + 42 Next 4 43 0 0 + 43 AggValue 0 11 12 row_number 0 accum=r[11] dest=r[12] + 44 ColumnRange 2 1 13 3 0 r[13..15]=ephemeral(buffer_table_window_0).emp_id..department + 45 Column 2 0 16 0 r[16]=ephemeral(buffer_table_window_0).salary + 46 Gosub 20 54 0 0 + 47 Delete 2 0 0 buffer_table_window_0 0 + 48 Next 2 50 0 0 + 49 Goto 0 51 0 0 + 50 Goto 0 43 0 0 + 51 ResetSorter 2 0 0 0 cursor=2 + 52 Return 18 0 1 0 + 53 Goto 0 61 0 0 + 54 Copy 13 6 0 0 r[6]=r[13] + 55 Copy 14 7 0 0 r[7]=r[14] + 56 Copy 15 8 0 0 r[8]=r[15] + 57 Copy 16 9 0 0 r[9]=r[16] + 58 Copy 12 10 0 0 r[10]=r[12] + 59 ResultRow 6 5 0 0 output=r[6..10] + 60 Return 20 0 0 0 + 61 Halt 0 0 0 0 + 62 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 63 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__running-total.snap b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__running-total.snap index 7eb0e44c..252e381e 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__running-total.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__running-total.snap @@ -29,207 +29,197 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 202 0 0 Start at 202 - 1 InitCoroutine 1 120 2 0 - 2 InitCoroutine 2 23 3 0 + 0 Init 0 192 0 0 Start at 192 + 1 InitCoroutine 1 112 2 0 + 2 InitCoroutine 2 19 3 0 3 SorterOpen 0 2 0 k(2,B,B) 0 cursor=0 4 OpenRead 1 3 0 k(5,B,B,B,B,B) 0 table=sales, root=3, iDb=0 - 5 Rewind 1 13 0 0 Rewind table sales - 6 Column 1 1 8 0 r[8]=sales.emp_id - 7 Column 1 2 9 0 r[9]=sales.sale_date - 8 RowId 1 10 0 0 r[10]=sales.rowid - 9 Column 1 3 11 0 r[11]=sales.amount - 10 MakeRecord 8 4 7 0 r[7]=mkrec(r[8..11]) - 11 SorterInsert 0 7 0 0 0 key=r[7] - 12 Next 1 6 0 0 - 13 OpenPseudo 2 7 4 0 4 columns in r[7] - 14 SorterSort 0 22 0 0 - 15 SorterData 0 7 2 0 r[7]=data - 16 Column 2 0 3 0 r[3]=pseudo.column 0 - 17 Column 2 1 4 0 r[4]=pseudo.column 1 - 18 Column 2 2 5 0 r[5]=pseudo.column 2 - 19 Column 2 3 6 0 r[6]=pseudo.column 3 - 20 Yield 2 0 0 0 - 21 SorterNext 0 15 0 0 - 22 EndCoroutine 2 0 0 0 - 23 SorterOpen 3 1 0 k(1,B) 0 cursor=3 - 24 OpenEphemeral 4 1 0 0 cursor=4 is_table=true - 25 OpenDup 5 4 0 0 new_cursor=5, original_cursor=4 - 26 OpenDup 6 4 0 0 new_cursor=6, original_cursor=4 - 27 Null 0 24 0 0 r[24]=NULL - 28 Null 0 25 0 0 r[25]=NULL - 29 InitCoroutine 2 0 3 0 - 30 Yield 2 74 0 0 - 31 Copy 4 27 0 0 r[27]=r[4] - 32 Compare 3 25 1 k(1, Binary) 0 r[3..3]==r[25..25]; compare partition keys to detect new partition - 33 Jump 34 37 34 0 - 34 Gosub 26 75 0 0 ; detected new partition - 35 Null 0 24 0 0 r[24]=NULL - 36 Copy 3 25 0 0 r[25]=r[3] - 37 NotNull 24 47 0 0 r[24]!=NULL -> goto 47 - 38 Copy 27 28 0 0 r[28]=r[27]; initialize per-cursor peer-reference registers for new partition - 39 Copy 27 29 0 0 r[29]=r[27] - 40 Null 0 18 18 0 r[18..18]=NULL; reset accumulator registers - 41 MakeRecord 3 4 31 0 r[31]=mkrec(r[3..6]) - 42 NewRowid 5 24 0 0 r[24]=rowid - 43 Insert 5 31 24 buffer_table_window_1 2 intkey=r[24] data=r[31] - 44 Rewind 4 46 0 0 Rewind ephemeral(buffer_table_window_1) - 45 Rewind 6 46 0 0 Rewind ephemeral(buffer_table_window_1) - 46 Goto 0 73 0 0 - 47 MakeRecord 3 4 32 0 r[32]=mkrec(r[3..6]) - 48 NewRowid 5 24 0 0 r[24]=rowid - 49 Insert 5 32 24 buffer_table_window_1 2 intkey=r[24] data=r[32] - 50 Compare 27 29 1 k(1, Binary) 0 r[27..27]==r[29..29]; compare ORDER BY columns to detect peer - 51 Jump 52 73 52 0 - 52 Column 6 3 33 0 r[33]=ephemeral(buffer_table_window_1).amount - 53 AggStep 0 33 18 sum 0 accum=r[18] step(r[33]) - 54 Next 6 56 0 0 - 55 Goto 0 60 0 0 - 56 Column 6 1 34 0 r[34]=ephemeral(buffer_table_window_1).sale_date - 57 Compare 29 34 1 k(1, Binary) 0 r[29..29]==r[34..34] - 58 Jump 59 52 59 0 - 59 Copy 34 29 0 0 r[29]=r[34] - 60 AggValue 0 18 19 sum 0 accum=r[18] dest=r[19] - 61 Column 4 1 20 0 r[20]=ephemeral(buffer_table_window_1).sale_date - 62 Column 4 2 21 0 r[21]=ephemeral(buffer_table_window_1).sale_id - 63 Column 4 0 22 0 r[22]=ephemeral(buffer_table_window_1).emp_id - 64 Column 4 3 23 0 r[23]=ephemeral(buffer_table_window_1).amount - 65 Gosub 30 101 0 0 - 66 Delete 4 0 0 buffer_table_window_1 0 - 67 Next 4 69 0 0 - 68 Goto 0 73 0 0 - 69 Column 4 1 35 0 r[35]=ephemeral(buffer_table_window_1).sale_date - 70 Compare 28 35 1 k(1, Binary) 0 r[28..28]==r[35..35] - 71 Jump 72 61 72 0 - 72 Copy 35 28 0 0 r[28]=r[35] - 73 Goto 0 30 0 0 - 74 Null 0 26 0 0 r[26]=NULL; return remaining buffered rows - 75 Rewind 5 98 0 0 Rewind ephemeral(buffer_table_window_1) - 76 Column 6 3 36 0 r[36]=ephemeral(buffer_table_window_1).amount - 77 AggStep 0 36 18 sum 0 accum=r[18] step(r[36]) - 78 Next 6 80 0 0 - 79 Goto 0 84 0 0 - 80 Column 6 1 37 0 r[37]=ephemeral(buffer_table_window_1).sale_date - 81 Compare 29 37 1 k(1, Binary) 0 r[29..29]==r[37..37] - 82 Jump 83 76 83 0 - 83 Copy 37 29 0 0 r[29]=r[37] - 84 AggValue 0 18 19 sum 0 accum=r[18] dest=r[19] - 85 Column 4 1 20 0 r[20]=ephemeral(buffer_table_window_1).sale_date - 86 Column 4 2 21 0 r[21]=ephemeral(buffer_table_window_1).sale_id - 87 Column 4 0 22 0 r[22]=ephemeral(buffer_table_window_1).emp_id - 88 Column 4 3 23 0 r[23]=ephemeral(buffer_table_window_1).amount - 89 Gosub 30 101 0 0 - 90 Delete 4 0 0 buffer_table_window_1 0 - 91 Next 4 93 0 0 - 92 Goto 0 98 0 0 - 93 Column 4 1 38 0 r[38]=ephemeral(buffer_table_window_1).sale_date - 94 Compare 28 38 1 k(1, Binary) 0 r[28..28]==r[38..38] - 95 Jump 96 85 96 0 - 96 Copy 38 28 0 0 r[28]=r[38] - 97 Goto 0 84 0 0 - 98 ResetSorter 4 0 0 0 cursor=4 - 99 Return 26 0 1 0 - 100 Goto 0 109 0 0 - 101 Copy 20 39 0 0 r[39]=r[20] - 102 Copy 21 40 0 0 r[40]=r[21] - 103 Copy 22 41 0 0 r[41]=r[22] - 104 Copy 23 42 0 0 r[42]=r[23] - 105 Copy 19 43 0 0 r[43]=r[19] - 106 MakeRecord 39 5 17 0 r[17]=mkrec(r[39..43]) - 107 SorterInsert 3 17 0 0 0 key=r[17] - 108 Return 30 0 0 0 - 109 OpenPseudo 7 17 5 0 5 columns in r[17] - 110 SorterSort 3 119 0 0 - 111 SorterData 3 17 7 0 r[17]=data - 112 Column 7 0 12 0 r[12]=pseudo.column 0 - 113 Column 7 1 13 0 r[13]=pseudo.column 1 - 114 Column 7 2 14 0 r[14]=pseudo.column 2 - 115 Column 7 3 15 0 r[15]=pseudo.column 3 - 116 Column 7 4 16 0 r[16]=pseudo.column 4 - 117 Yield 1 0 0 0 - 118 SorterNext 3 111 0 0 - 119 EndCoroutine 1 0 0 0 - 120 OpenEphemeral 8 1 0 0 cursor=8 is_table=true - 121 OpenDup 9 8 0 0 new_cursor=9, original_cursor=8 - 122 OpenDup 10 8 0 0 new_cursor=10, original_cursor=8 - 123 Null 0 57 0 0 r[57]=NULL - 124 InitCoroutine 1 0 2 0 - 125 Yield 1 165 0 0 - 126 Copy 12 59 0 0 r[59]=r[12] - 127 NotNull 57 137 0 0 r[57]!=NULL -> goto 137 - 128 Copy 59 60 0 0 r[60]=r[59]; initialize per-cursor peer-reference registers for new partition - 129 Copy 59 61 0 0 r[61]=r[59] - 130 Null 0 50 50 0 r[50..50]=NULL; reset accumulator registers - 131 MakeRecord 12 5 63 0 r[63]=mkrec(r[12..16]) - 132 NewRowid 9 57 0 0 r[57]=rowid - 133 Insert 9 63 57 buffer_table_window_0 2 intkey=r[57] data=r[63] - 134 Rewind 8 136 0 0 Rewind ephemeral(buffer_table_window_0) - 135 Rewind 10 136 0 0 Rewind ephemeral(buffer_table_window_0) - 136 Goto 0 164 0 0 - 137 MakeRecord 12 5 64 0 r[64]=mkrec(r[12..16]) - 138 NewRowid 9 57 0 0 r[57]=rowid - 139 Insert 9 64 57 buffer_table_window_0 2 intkey=r[57] data=r[64] - 140 Compare 59 61 1 k(1, Binary) 0 r[59..59]==r[61..61]; compare ORDER BY columns to detect peer - 141 Jump 142 164 142 0 - 142 Column 10 3 65 0 r[65]=ephemeral(buffer_table_window_0).amount - 143 AggStep 0 65 50 sum 0 accum=r[50] step(r[65]) - 144 Next 10 146 0 0 - 145 Goto 0 150 0 0 - 146 Column 10 0 66 0 r[66]=ephemeral(buffer_table_window_0).sale_date - 147 Compare 61 66 1 k(1, Binary) 0 r[61..61]==r[66..66] - 148 Jump 149 142 149 0 - 149 Copy 66 61 0 0 r[61]=r[66] - 150 AggValue 0 50 51 sum 0 accum=r[50] dest=r[51] - 151 Column 8 1 52 0 r[52]=ephemeral(buffer_table_window_0).sale_id - 152 Column 8 2 53 0 r[53]=ephemeral(buffer_table_window_0).emp_id - 153 Column 8 0 54 0 r[54]=ephemeral(buffer_table_window_0).sale_date - 154 Column 8 3 55 0 r[55]=ephemeral(buffer_table_window_0).amount - 155 Column 8 4 56 0 r[56]=ephemeral(buffer_table_window_0).column 4 - 156 Gosub 62 193 0 0 - 157 Delete 8 0 0 buffer_table_window_0 0 - 158 Next 8 160 0 0 - 159 Goto 0 164 0 0 - 160 Column 8 0 67 0 r[67]=ephemeral(buffer_table_window_0).sale_date - 161 Compare 60 67 1 k(1, Binary) 0 r[60..60]==r[67..67] - 162 Jump 163 151 163 0 - 163 Copy 67 60 0 0 r[60]=r[67] - 164 Goto 0 125 0 0 - 165 Null 0 58 0 0 r[58]=NULL; return remaining buffered rows - 166 Rewind 9 190 0 0 Rewind ephemeral(buffer_table_window_0) - 167 Column 10 3 68 0 r[68]=ephemeral(buffer_table_window_0).amount - 168 AggStep 0 68 50 sum 0 accum=r[50] step(r[68]) - 169 Next 10 171 0 0 - 170 Goto 0 175 0 0 - 171 Column 10 0 69 0 r[69]=ephemeral(buffer_table_window_0).sale_date - 172 Compare 61 69 1 k(1, Binary) 0 r[61..61]==r[69..69] - 173 Jump 174 167 174 0 - 174 Copy 69 61 0 0 r[61]=r[69] - 175 AggValue 0 50 51 sum 0 accum=r[50] dest=r[51] - 176 Column 8 1 52 0 r[52]=ephemeral(buffer_table_window_0).sale_id - 177 Column 8 2 53 0 r[53]=ephemeral(buffer_table_window_0).emp_id - 178 Column 8 0 54 0 r[54]=ephemeral(buffer_table_window_0).sale_date - 179 Column 8 3 55 0 r[55]=ephemeral(buffer_table_window_0).amount - 180 Column 8 4 56 0 r[56]=ephemeral(buffer_table_window_0).column 4 - 181 Gosub 62 193 0 0 - 182 Delete 8 0 0 buffer_table_window_0 0 - 183 Next 8 185 0 0 - 184 Goto 0 190 0 0 - 185 Column 8 0 70 0 r[70]=ephemeral(buffer_table_window_0).sale_date - 186 Compare 60 70 1 k(1, Binary) 0 r[60..60]==r[70..70] - 187 Jump 188 176 188 0 - 188 Copy 70 60 0 0 r[60]=r[70] - 189 Goto 0 175 0 0 - 190 ResetSorter 8 0 0 0 cursor=8 - 191 Return 58 0 1 0 - 192 Goto 0 201 0 0 - 193 Copy 52 44 0 0 r[44]=r[52] - 194 Copy 53 45 0 0 r[45]=r[53] - 195 Copy 54 46 0 0 r[46]=r[54] - 196 Copy 55 47 0 0 r[47]=r[55] - 197 Copy 51 48 0 0 r[48]=r[51] - 198 Copy 56 49 0 0 r[49]=r[56] - 199 ResultRow 44 6 0 0 output=r[44..49] - 200 Return 62 0 0 0 - 201 Halt 0 0 0 0 - 202 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 203 Goto 0 1 0 0 + 5 Rewind 1 12 0 0 Rewind table sales + 6 ColumnRange 1 1 8 2 0 r[8..9]=sales.emp_id..sale_date + 7 RowId 1 10 0 0 r[10]=sales.rowid + 8 Column 1 3 11 0 r[11]=sales.amount + 9 MakeRecord 8 4 7 0 r[7]=mkrec(r[8..11]) + 10 SorterInsert 0 7 0 0 0 key=r[7] + 11 Next 1 6 0 1 + 12 OpenPseudo 2 7 4 0 4 columns in r[7] + 13 SorterSort 0 18 0 0 + 14 SorterData 0 7 2 0 r[7]=data + 15 ColumnRange 2 0 3 4 0 r[3..6]=pseudo.column 0..column 3 + 16 Yield 2 0 0 0 + 17 SorterNext 0 14 0 0 + 18 EndCoroutine 2 0 0 0 + 19 SorterOpen 3 1 0 k(1,B) 0 cursor=3 + 20 OpenEphemeral 4 1 0 0 cursor=4 is_table=true + 21 OpenDup 5 4 0 0 new_cursor=5, original_cursor=4 + 22 OpenDup 6 4 0 0 new_cursor=6, original_cursor=4 + 23 Null 0 24 0 0 r[24]=NULL + 24 Null 0 25 0 0 r[25]=NULL + 25 InitCoroutine 2 0 3 0 + 26 Yield 2 71 0 0 + 27 Copy 4 27 0 0 r[27]=r[4] + 28 Compare 3 25 1 k(1, Binary) 0 r[3..3]==r[25..25]; compare partition keys to detect new partition + 29 Jump 30 33 30 0 + 30 Gosub 26 72 0 0 ; detected new partition + 31 Null 0 24 0 0 r[24]=NULL + 32 Copy 3 25 0 0 r[25]=r[3] + 33 NotNull 24 44 0 0 r[24]!=NULL -> goto 44 + 34 Copy 27 28 0 0 r[28]=r[27]; initialize peer-reference registers for new partition + 35 Copy 28 29 0 0 r[29]=r[28] + 36 Copy 28 30 0 0 r[30]=r[28] + 37 Null 0 18 18 0 r[18..18]=NULL; reset accumulator registers + 38 MakeRecord 3 4 32 0 r[32]=mkrec(r[3..6]) + 39 NewRowid 5 24 0 0 r[24]=rowid + 40 Insert 5 32 24 buffer_table_window_1 2 intkey=r[24] data=r[32] + 41 Rewind 4 43 0 0 Rewind ephemeral(buffer_table_window_1) + 42 Rewind 6 43 0 0 Rewind ephemeral(buffer_table_window_1) + 43 Goto 0 70 0 0 + 44 MakeRecord 3 4 33 0 r[33]=mkrec(r[3..6]) + 45 NewRowid 5 24 0 0 r[24]=rowid + 46 Insert 5 33 24 buffer_table_window_1 2 intkey=r[24] data=r[33] + 47 Compare 27 28 1 k(1, Binary) 0 r[27..27]==r[28..28]; peer of previous row: buffer only, handle at group end + 48 Jump 49 70 49 0 + 49 Copy 27 28 0 0 r[28]=r[27] + 50 Column 6 3 34 0 r[34]=ephemeral(buffer_table_window_1).amount + 51 AggStep 0 34 18 sum 0 accum=r[18] step(r[34]) + 52 Next 6 54 0 0 + 53 Goto 0 58 0 0 + 54 Column 6 1 35 0 r[35]=ephemeral(buffer_table_window_1).sale_date + 55 Compare 30 35 1 k(1, Binary) 0 r[30..30]==r[35..35] + 56 Jump 57 50 57 0 + 57 Copy 35 30 0 0 r[30]=r[35] + 58 AggValue 0 18 19 sum 0 accum=r[18] dest=r[19] + 59 ColumnRange 4 1 20 2 0 r[20..21]=ephemeral(buffer_table_window_1).sale_date..sale_id + 60 Column 4 0 22 0 r[22]=ephemeral(buffer_table_window_1).emp_id + 61 Column 4 3 23 0 r[23]=ephemeral(buffer_table_window_1).amount + 62 Gosub 31 97 0 0 + 63 Delete 4 0 0 buffer_table_window_1 0 + 64 Next 4 66 0 0 + 65 Goto 0 70 0 0 + 66 Column 4 1 36 0 r[36]=ephemeral(buffer_table_window_1).sale_date + 67 Compare 29 36 1 k(1, Binary) 0 r[29..29]==r[36..36] + 68 Jump 69 59 69 0 + 69 Copy 36 29 0 0 r[29]=r[36] + 70 Goto 0 26 0 0 + 71 Null 0 26 0 0 r[26]=NULL; return remaining buffered rows + 72 Rewind 5 94 0 0 Rewind ephemeral(buffer_table_window_1) + 73 Column 6 3 37 0 r[37]=ephemeral(buffer_table_window_1).amount + 74 AggStep 0 37 18 sum 0 accum=r[18] step(r[37]) + 75 Next 6 77 0 0 + 76 Goto 0 81 0 0 + 77 Column 6 1 38 0 r[38]=ephemeral(buffer_table_window_1).sale_date + 78 Compare 30 38 1 k(1, Binary) 0 r[30..30]==r[38..38] + 79 Jump 80 73 80 0 + 80 Copy 38 30 0 0 r[30]=r[38] + 81 AggValue 0 18 19 sum 0 accum=r[18] dest=r[19] + 82 ColumnRange 4 1 20 2 0 r[20..21]=ephemeral(buffer_table_window_1).sale_date..sale_id + 83 Column 4 0 22 0 r[22]=ephemeral(buffer_table_window_1).emp_id + 84 Column 4 3 23 0 r[23]=ephemeral(buffer_table_window_1).amount + 85 Gosub 31 97 0 0 + 86 Delete 4 0 0 buffer_table_window_1 0 + 87 Next 4 89 0 0 + 88 Goto 0 94 0 0 + 89 Column 4 1 39 0 r[39]=ephemeral(buffer_table_window_1).sale_date + 90 Compare 29 39 1 k(1, Binary) 0 r[29..29]==r[39..39] + 91 Jump 92 82 92 0 + 92 Copy 39 29 0 0 r[29]=r[39] + 93 Goto 0 81 0 0 + 94 ResetSorter 4 0 0 0 cursor=4 + 95 Return 26 0 1 0 + 96 Goto 0 105 0 0 + 97 Copy 20 40 0 0 r[40]=r[20] + 98 Copy 21 41 0 0 r[41]=r[21] + 99 Copy 22 42 0 0 r[42]=r[22] + 100 Copy 23 43 0 0 r[43]=r[23] + 101 Copy 19 44 0 0 r[44]=r[19] + 102 MakeRecord 40 5 17 0 r[17]=mkrec(r[40..44]) + 103 SorterInsert 3 17 0 0 0 key=r[17] + 104 Return 31 0 0 0 + 105 OpenPseudo 7 17 5 0 5 columns in r[17] + 106 SorterSort 3 111 0 0 + 107 SorterData 3 17 7 0 r[17]=data + 108 ColumnRange 7 0 12 5 0 r[12..16]=pseudo.column 0..column 4 + 109 Yield 1 0 0 0 + 110 SorterNext 3 107 0 0 + 111 EndCoroutine 1 0 0 0 + 112 OpenEphemeral 8 1 0 0 cursor=8 is_table=true + 113 OpenDup 9 8 0 0 new_cursor=9, original_cursor=8 + 114 OpenDup 10 8 0 0 new_cursor=10, original_cursor=8 + 115 Null 0 58 0 0 r[58]=NULL + 116 InitCoroutine 1 0 2 0 + 117 Yield 1 157 0 0 + 118 Copy 12 60 0 0 r[60]=r[12] + 119 NotNull 58 130 0 0 r[58]!=NULL -> goto 130 + 120 Copy 60 61 0 0 r[61]=r[60]; initialize peer-reference registers for new partition + 121 Copy 61 62 0 0 r[62]=r[61] + 122 Copy 61 63 0 0 r[63]=r[61] + 123 Null 0 51 51 0 r[51..51]=NULL; reset accumulator registers + 124 MakeRecord 12 5 65 0 r[65]=mkrec(r[12..16]) + 125 NewRowid 9 58 0 0 r[58]=rowid + 126 Insert 9 65 58 buffer_table_window_0 2 intkey=r[58] data=r[65] + 127 Rewind 8 129 0 0 Rewind ephemeral(buffer_table_window_0) + 128 Rewind 10 129 0 0 Rewind ephemeral(buffer_table_window_0) + 129 Goto 0 156 0 0 + 130 MakeRecord 12 5 66 0 r[66]=mkrec(r[12..16]) + 131 NewRowid 9 58 0 0 r[58]=rowid + 132 Insert 9 66 58 buffer_table_window_0 2 intkey=r[58] data=r[66] + 133 Compare 60 61 1 k(1, Binary) 0 r[60..60]==r[61..61]; peer of previous row: buffer only, handle at group end + 134 Jump 135 156 135 0 + 135 Copy 60 61 0 0 r[61]=r[60] + 136 Column 10 3 67 0 r[67]=ephemeral(buffer_table_window_0).amount + 137 AggStep 0 67 51 sum 0 accum=r[51] step(r[67]) + 138 Next 10 140 0 0 + 139 Goto 0 144 0 0 + 140 Column 10 0 68 0 r[68]=ephemeral(buffer_table_window_0).sale_date + 141 Compare 63 68 1 k(1, Binary) 0 r[63..63]==r[68..68] + 142 Jump 143 136 143 0 + 143 Copy 68 63 0 0 r[63]=r[68] + 144 AggValue 0 51 52 sum 0 accum=r[51] dest=r[52] + 145 ColumnRange 8 1 53 2 0 r[53..54]=ephemeral(buffer_table_window_0).sale_id..emp_id + 146 Column 8 0 55 0 r[55]=ephemeral(buffer_table_window_0).sale_date + 147 ColumnRange 8 3 56 2 0 r[56..57]=ephemeral(buffer_table_window_0).amount..column 4 + 148 Gosub 64 183 0 0 + 149 Delete 8 0 0 buffer_table_window_0 0 + 150 Next 8 152 0 0 + 151 Goto 0 156 0 0 + 152 Column 8 0 69 0 r[69]=ephemeral(buffer_table_window_0).sale_date + 153 Compare 62 69 1 k(1, Binary) 0 r[62..62]==r[69..69] + 154 Jump 155 145 155 0 + 155 Copy 69 62 0 0 r[62]=r[69] + 156 Goto 0 117 0 0 + 157 Null 0 59 0 0 r[59]=NULL; return remaining buffered rows + 158 Rewind 9 180 0 0 Rewind ephemeral(buffer_table_window_0) + 159 Column 10 3 70 0 r[70]=ephemeral(buffer_table_window_0).amount + 160 AggStep 0 70 51 sum 0 accum=r[51] step(r[70]) + 161 Next 10 163 0 0 + 162 Goto 0 167 0 0 + 163 Column 10 0 71 0 r[71]=ephemeral(buffer_table_window_0).sale_date + 164 Compare 63 71 1 k(1, Binary) 0 r[63..63]==r[71..71] + 165 Jump 166 159 166 0 + 166 Copy 71 63 0 0 r[63]=r[71] + 167 AggValue 0 51 52 sum 0 accum=r[51] dest=r[52] + 168 ColumnRange 8 1 53 2 0 r[53..54]=ephemeral(buffer_table_window_0).sale_id..emp_id + 169 Column 8 0 55 0 r[55]=ephemeral(buffer_table_window_0).sale_date + 170 ColumnRange 8 3 56 2 0 r[56..57]=ephemeral(buffer_table_window_0).amount..column 4 + 171 Gosub 64 183 0 0 + 172 Delete 8 0 0 buffer_table_window_0 0 + 173 Next 8 175 0 0 + 174 Goto 0 180 0 0 + 175 Column 8 0 72 0 r[72]=ephemeral(buffer_table_window_0).sale_date + 176 Compare 62 72 1 k(1, Binary) 0 r[62..62]==r[72..72] + 177 Jump 178 168 178 0 + 178 Copy 72 62 0 0 r[62]=r[72] + 179 Goto 0 167 0 0 + 180 ResetSorter 8 0 0 0 cursor=8 + 181 Return 59 0 1 0 + 182 Goto 0 191 0 0 + 183 Copy 53 45 0 0 r[45]=r[53] + 184 Copy 54 46 0 0 r[46]=r[54] + 185 Copy 55 47 0 0 r[47]=r[55] + 186 Copy 56 48 0 0 r[48]=r[56] + 187 Copy 52 49 0 0 r[49]=r[52] + 188 Copy 57 50 0 0 r[50]=r[57] + 189 ResultRow 45 6 0 0 output=r[45..50] + 190 Return 64 0 0 0 + 191 Halt 0 0 0 0 + 192 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 193 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-order-by.snap b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-order-by.snap index aa4944dd..6c13cec1 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-order-by.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-order-by.snap @@ -24,87 +24,78 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 82 0 0 Start at 82 - 1 InitCoroutine 1 24 2 0 + 0 Init 0 73 0 0 Start at 73 + 1 InitCoroutine 1 19 2 0 2 SorterOpen 0 2 0 k(2,B,-B) 0 cursor=0 3 OpenRead 1 3 0 k(5,B,B,B,B,B) 0 table=sales, root=3, iDb=0 - 4 Rewind 1 13 0 0 Rewind table sales - 5 Column 1 2 8 0 r[8]=sales.sale_date - 6 Column 1 3 9 0 r[9]=sales.amount - 7 RowId 1 10 0 0 r[10]=sales.rowid - 8 Column 1 1 11 0 r[11]=sales.emp_id - 9 Column 1 4 12 0 r[12]=sales.region - 10 MakeRecord 8 5 7 0 r[7]=mkrec(r[8..12]) - 11 SorterInsert 0 7 0 0 0 key=r[7] - 12 Next 1 5 0 0 - 13 OpenPseudo 2 7 5 0 5 columns in r[7] - 14 SorterSort 0 23 0 0 - 15 SorterData 0 7 2 0 r[7]=data - 16 Column 2 0 2 0 r[2]=pseudo.column 0 - 17 Column 2 1 3 0 r[3]=pseudo.column 1 - 18 Column 2 2 4 0 r[4]=pseudo.column 2 - 19 Column 2 3 5 0 r[5]=pseudo.column 3 - 20 Column 2 4 6 0 r[6]=pseudo.column 4 - 21 Yield 1 0 0 0 - 22 SorterNext 0 15 0 0 - 23 EndCoroutine 1 0 0 0 - 24 OpenEphemeral 3 1 0 0 cursor=3 is_table=true - 25 OpenDup 4 3 0 0 new_cursor=4, original_cursor=3 - 26 OpenDup 5 3 0 0 new_cursor=5, original_cursor=3 - 27 Null 0 26 0 0 r[26]=NULL - 28 InitCoroutine 1 0 2 0 - 29 Yield 1 55 0 0 - 30 Copy 2 28 0 0 r[28]=r[2] - 31 Copy 3 29 0 0 r[29]=r[3] - 32 NotNull 26 40 0 0 r[26]!=NULL -> goto 40 - 33 Null 0 19 19 0 r[19..19]=NULL; initialize per-cursor peer-reference registers for new partition - 34 MakeRecord 2 5 31 0 r[31]=mkrec(r[2..6]) - 35 NewRowid 4 26 0 0 r[26]=rowid - 36 Insert 4 31 26 buffer_table_window_0 2 intkey=r[26] data=r[31] - 37 Rewind 3 39 0 0 Rewind ephemeral(buffer_table_window_0) - 38 Rewind 5 39 0 0 Rewind ephemeral(buffer_table_window_0) - 39 Goto 0 54 0 0 - 40 MakeRecord 2 5 32 0 r[32]=mkrec(r[2..6]) - 41 NewRowid 4 26 0 0 r[26]=rowid - 42 Insert 4 32 26 buffer_table_window_0 2 intkey=r[26] data=r[32] - 43 AggStep 0 0 19 row_number 0 accum=r[19] step(r[0]) - 44 Next 5 45 0 0 - 45 AggValue 0 19 20 row_number 0 accum=r[19] dest=r[20] - 46 Column 3 2 21 0 r[21]=ephemeral(buffer_table_window_0).sale_id - 47 Column 3 3 22 0 r[22]=ephemeral(buffer_table_window_0).emp_id - 48 Column 3 0 23 0 r[23]=ephemeral(buffer_table_window_0).sale_date - 49 Column 3 1 24 0 r[24]=ephemeral(buffer_table_window_0).amount - 50 Column 3 4 25 0 r[25]=ephemeral(buffer_table_window_0).region - 51 Gosub 30 73 0 0 - 52 Delete 3 0 0 buffer_table_window_0 0 - 53 Next 3 54 0 0 - 54 Goto 0 29 0 0 - 55 Null 0 27 0 0 r[27]=NULL; return remaining buffered rows - 56 Rewind 4 70 0 0 Rewind ephemeral(buffer_table_window_0) - 57 AggStep 0 0 19 row_number 0 accum=r[19] step(r[0]) - 58 Next 5 59 0 0 - 59 AggValue 0 19 20 row_number 0 accum=r[19] dest=r[20] - 60 Column 3 2 21 0 r[21]=ephemeral(buffer_table_window_0).sale_id - 61 Column 3 3 22 0 r[22]=ephemeral(buffer_table_window_0).emp_id - 62 Column 3 0 23 0 r[23]=ephemeral(buffer_table_window_0).sale_date - 63 Column 3 1 24 0 r[24]=ephemeral(buffer_table_window_0).amount - 64 Column 3 4 25 0 r[25]=ephemeral(buffer_table_window_0).region - 65 Gosub 30 73 0 0 - 66 Delete 3 0 0 buffer_table_window_0 0 - 67 Next 3 69 0 0 - 68 Goto 0 70 0 0 - 69 Goto 0 59 0 0 - 70 ResetSorter 3 0 0 0 cursor=3 - 71 Return 27 0 1 0 - 72 Goto 0 81 0 0 - 73 Copy 21 13 0 0 r[13]=r[21] - 74 Copy 22 14 0 0 r[14]=r[22] - 75 Copy 23 15 0 0 r[15]=r[23] - 76 Copy 24 16 0 0 r[16]=r[24] - 77 Copy 25 17 0 0 r[17]=r[25] - 78 Copy 20 18 0 0 r[18]=r[20] - 79 ResultRow 13 6 0 0 output=r[13..18] - 80 Return 30 0 0 0 - 81 Halt 0 0 0 0 - 82 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 83 Goto 0 1 0 0 + 4 Rewind 1 12 0 0 Rewind table sales + 5 ColumnRange 1 2 8 2 0 r[8..9]=sales.sale_date..amount + 6 RowId 1 10 0 0 r[10]=sales.rowid + 7 Column 1 1 11 0 r[11]=sales.emp_id + 8 Column 1 4 12 0 r[12]=sales.region + 9 MakeRecord 8 5 7 0 r[7]=mkrec(r[8..12]) + 10 SorterInsert 0 7 0 0 0 key=r[7] + 11 Next 1 5 0 1 + 12 OpenPseudo 2 7 5 0 5 columns in r[7] + 13 SorterSort 0 18 0 0 + 14 SorterData 0 7 2 0 r[7]=data + 15 ColumnRange 2 0 2 5 0 r[2..6]=pseudo.column 0..column 4 + 16 Yield 1 0 0 0 + 17 SorterNext 0 14 0 0 + 18 EndCoroutine 1 0 0 0 + 19 OpenEphemeral 3 1 0 0 cursor=3 is_table=true + 20 OpenDup 4 3 0 0 new_cursor=4, original_cursor=3 + 21 OpenDup 5 3 0 0 new_cursor=5, original_cursor=3 + 22 Null 0 26 0 0 r[26]=NULL + 23 InitCoroutine 1 0 2 0 + 24 Yield 1 48 0 0 + 25 Copy 2 28 0 0 r[28]=r[2] + 26 Copy 3 29 0 0 r[29]=r[3] + 27 NotNull 26 35 0 0 r[26]!=NULL -> goto 35 + 28 Null 0 19 19 0 r[19..19]=NULL; reset accumulator registers + 29 MakeRecord 2 5 31 0 r[31]=mkrec(r[2..6]) + 30 NewRowid 4 26 0 0 r[26]=rowid + 31 Insert 4 31 26 buffer_table_window_0 2 intkey=r[26] data=r[31] + 32 Rewind 3 34 0 0 Rewind ephemeral(buffer_table_window_0) + 33 Rewind 5 34 0 0 Rewind ephemeral(buffer_table_window_0) + 34 Goto 0 47 0 0 + 35 MakeRecord 2 5 32 0 r[32]=mkrec(r[2..6]) + 36 NewRowid 4 26 0 0 r[26]=rowid + 37 Insert 4 32 26 buffer_table_window_0 2 intkey=r[26] data=r[32] + 38 AggStep 0 0 19 row_number 0 accum=r[19] step(r[0]) + 39 Next 5 40 0 0 + 40 AggValue 0 19 20 row_number 0 accum=r[19] dest=r[20] + 41 ColumnRange 3 2 21 2 0 r[21..22]=ephemeral(buffer_table_window_0).sale_id..emp_id + 42 ColumnRange 3 0 23 2 0 r[23..24]=ephemeral(buffer_table_window_0).sale_date..amount + 43 Column 3 4 25 0 r[25]=ephemeral(buffer_table_window_0).region + 44 Gosub 30 64 0 0 + 45 Delete 3 0 0 buffer_table_window_0 0 + 46 Next 3 47 0 0 + 47 Goto 0 24 0 0 + 48 Null 0 27 0 0 r[27]=NULL; return remaining buffered rows + 49 Rewind 4 61 0 0 Rewind ephemeral(buffer_table_window_0) + 50 AggStep 0 0 19 row_number 0 accum=r[19] step(r[0]) + 51 Next 5 52 0 0 + 52 AggValue 0 19 20 row_number 0 accum=r[19] dest=r[20] + 53 ColumnRange 3 2 21 2 0 r[21..22]=ephemeral(buffer_table_window_0).sale_id..emp_id + 54 ColumnRange 3 0 23 2 0 r[23..24]=ephemeral(buffer_table_window_0).sale_date..amount + 55 Column 3 4 25 0 r[25]=ephemeral(buffer_table_window_0).region + 56 Gosub 30 64 0 0 + 57 Delete 3 0 0 buffer_table_window_0 0 + 58 Next 3 60 0 0 + 59 Goto 0 61 0 0 + 60 Goto 0 52 0 0 + 61 ResetSorter 3 0 0 0 cursor=3 + 62 Return 27 0 1 0 + 63 Goto 0 72 0 0 + 64 Copy 21 13 0 0 r[13]=r[21] + 65 Copy 22 14 0 0 r[14]=r[22] + 66 Copy 23 15 0 0 r[15]=r[23] + 67 Copy 24 16 0 0 r[16]=r[24] + 68 Copy 25 17 0 0 r[17]=r[25] + 69 Copy 20 18 0 0 r[18]=r[20] + 70 ResultRow 13 6 0 0 output=r[13..18] + 71 Return 30 0 0 0 + 72 Halt 0 0 0 0 + 73 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 74 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-over-grouped.snap b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-over-grouped.snap index db63e9ee..081930aa 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-over-grouped.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-over-grouped.snap @@ -22,109 +22,105 @@ QUERY PLAN `--USE SORTER FOR ORDER BY BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 102 0 0 Start at 102 - 1 InitCoroutine 1 51 2 0 - 2 SorterOpen 0 1 0 k(1,-B) 0 cursor=0 - 3 Null 0 11 12 0 r[11..12]=NULL - 4 Integer 0 8 0 0 r[8]=0; clear group by abort flag - 5 Null 0 9 0 0 r[9]=NULL; initialize group by comparison registers to NULL - 6 Gosub 16 39 0 0 ; go to clear accumulator subroutine - 7 OpenRead 1 2 0 k(5,B,B,B,B,B) 0 table=employees, root=2, iDb=0 - 8 OpenRead 2 4 0 k(2,B) 0 index=idx_employees_department, root=4, iDb=0 - 9 Rewind 2 25 0 0 Rewind index idx_employees_department - 10 DeferredSeek 2 1 0 0 - 11 Column 2 0 14 0 r[14]=idx_employees_department.department - 12 Column 1 3 15 0 r[15]=employees.salary - 13 Compare 9 14 1 k(1, Binary) 0 r[9..9]==r[14..14] - 14 Jump 15 19 15 0 ; start new group if comparison is not equal - 15 Gosub 6 29 0 0 ; check if ended group had data, and output if so - 16 Move 14 9 1 0 r[9..9]=r[14..14] - 17 IfPos 8 42 0 0 r[8]>0 -> r[8]-=0, goto 42; check abort flag - 18 Gosub 16 39 0 0 ; goto clear accumulator subroutine - 19 AggStep 0 15 11 sum 0 accum=r[11] step(r[15]) - 20 AggStep 0 17 12 count 0 accum=r[12] step(r[17]) - 21 If 7 23 0 0 if r[7] goto 23; don't emit group columns if continuing existing group - 22 Column 2 0 10 0 r[10]=idx_employees_department.department - 23 Integer 1 7 0 0 r[7]=1; indicate data in accumulator - 24 Next 2 10 0 0 - 25 Gosub 6 29 0 0 ; emit row for final group - 26 Goto 0 42 0 0 ; group by finished - 27 Integer 1 8 0 0 r[8]=1 - 28 Return 6 0 0 0 - 29 IfPos 7 31 0 0 r[7]>0 -> r[7]-=0, goto 31; output group by row subroutine start - 30 Return 6 0 0 0 - 31 AggFinal 0 11 0 sum 0 accum=r[11] - 32 AggFinal 0 12 0 count 0 accum=r[12] - 33 Copy 11 18 0 0 r[18]=r[11] - 34 Copy 10 19 0 0 r[19]=r[10] - 35 Copy 12 20 0 0 r[20]=r[12] - 36 MakeRecord 18 3 5 0 r[5]=mkrec(r[18..20]) - 37 SorterInsert 0 5 0 0 0 key=r[5] - 38 Return 6 0 0 0 - 39 Null 0 10 12 0 r[10..12]=NULL; clear accumulator subroutine start - 40 Integer 0 7 0 0 r[7]=0 - 41 Return 16 0 0 0 - 42 OpenPseudo 3 5 3 0 3 columns in r[5] - 43 SorterSort 0 50 0 0 - 44 SorterData 0 5 3 0 r[5]=data - 45 Column 3 0 2 0 r[2]=pseudo.column 0 - 46 Column 3 1 3 0 r[3]=pseudo.column 1 - 47 Column 3 2 4 0 r[4]=pseudo.column 2 - 48 Yield 1 0 0 0 - 49 SorterNext 0 44 0 0 - 50 EndCoroutine 1 0 0 0 - 51 OpenEphemeral 4 1 0 0 cursor=4 is_table=true - 52 OpenDup 5 4 0 0 new_cursor=5, original_cursor=4 - 53 OpenDup 6 4 0 0 new_cursor=6, original_cursor=4 - 54 Null 0 30 0 0 r[30]=NULL - 55 InitCoroutine 1 0 2 0 - 56 Yield 1 79 0 0 - 57 Copy 2 32 0 0 r[32]=r[2] - 58 NotNull 30 66 0 0 r[30]!=NULL -> goto 66 - 59 Null 0 25 25 0 r[25..25]=NULL; initialize per-cursor peer-reference registers for new partition - 60 MakeRecord 2 3 34 0 r[34]=mkrec(r[2..4]) - 61 NewRowid 5 30 0 0 r[30]=rowid - 62 Insert 5 34 30 buffer_table_window_0 2 intkey=r[30] data=r[34] - 63 Rewind 4 65 0 0 Rewind ephemeral(buffer_table_window_0) - 64 Rewind 6 65 0 0 Rewind ephemeral(buffer_table_window_0) - 65 Goto 0 78 0 0 - 66 MakeRecord 2 3 35 0 r[35]=mkrec(r[2..4]) - 67 NewRowid 5 30 0 0 r[30]=rowid - 68 Insert 5 35 30 buffer_table_window_0 2 intkey=r[30] data=r[35] - 69 AggStep 0 0 25 row_number 0 accum=r[25] step(r[0]) - 70 Next 6 71 0 0 - 71 AggValue 0 25 26 row_number 0 accum=r[25] dest=r[26] - 72 Column 4 1 27 0 r[27]=ephemeral(buffer_table_window_0).department - 73 Column 4 2 28 0 r[28]=ephemeral(buffer_table_window_0).column 2 - 74 Column 4 0 29 0 r[29]=ephemeral(buffer_table_window_0).column 0 - 75 Gosub 33 95 0 0 - 76 Delete 4 0 0 buffer_table_window_0 0 - 77 Next 4 78 0 0 - 78 Goto 0 56 0 0 - 79 Null 0 31 0 0 r[31]=NULL; return remaining buffered rows - 80 Rewind 5 92 0 0 Rewind ephemeral(buffer_table_window_0) - 81 AggStep 0 0 25 row_number 0 accum=r[25] step(r[0]) - 82 Next 6 83 0 0 - 83 AggValue 0 25 26 row_number 0 accum=r[25] dest=r[26] - 84 Column 4 1 27 0 r[27]=ephemeral(buffer_table_window_0).department - 85 Column 4 2 28 0 r[28]=ephemeral(buffer_table_window_0).column 2 - 86 Column 4 0 29 0 r[29]=ephemeral(buffer_table_window_0).column 0 - 87 Gosub 33 95 0 0 - 88 Delete 4 0 0 buffer_table_window_0 0 - 89 Next 4 91 0 0 - 90 Goto 0 92 0 0 - 91 Goto 0 83 0 0 - 92 ResetSorter 4 0 0 0 cursor=4 - 93 Return 31 0 1 0 - 94 Goto 0 101 0 0 - 95 Copy 27 21 0 0 r[21]=r[27] - 96 Copy 28 22 0 0 r[22]=r[28] - 97 Copy 29 23 0 0 r[23]=r[29] - 98 Copy 26 24 0 0 r[24]=r[26] - 99 ResultRow 21 4 0 0 output=r[21..24] - 100 Return 33 0 0 0 - 101 Halt 0 0 0 0 - 102 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 103 Integer 1 17 0 0 r[17]=1 - 104 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 98 0 0 Start at 98 + 1 InitCoroutine 1 49 2 0 + 2 SorterOpen 0 1 0 k(1,-B) 0 cursor=0 + 3 Null 0 11 12 0 r[11..12]=NULL + 4 Integer 0 8 0 0 r[8]=0; clear group by abort flag + 5 Null 0 9 0 0 r[9]=NULL; initialize group by comparison registers to NULL + 6 Gosub 16 39 0 0 ; go to clear accumulator subroutine + 7 OpenRead 1 2 0 k(5,B,B,B,B,B) 0 table=employees, root=2, iDb=0 + 8 OpenRead 2 4 0 k(2,B) 0 index=idx_employees_department, root=4, iDb=0 + 9 Rewind 2 25 0 0 Rewind index idx_employees_department + 10 DeferredSeek 2 1 0 0 + 11 Column 2 0 14 0 r[14]=idx_employees_department.department + 12 Column 1 3 15 0 r[15]=employees.salary + 13 Compare 9 14 1 k(1, Binary) 0 r[9..9]==r[14..14] + 14 Jump 15 19 15 0 ; start new group if comparison is not equal + 15 Gosub 6 29 0 0 ; check if ended group had data, and output if so + 16 Move 14 9 1 0 r[9..9]=r[14..14] + 17 IfPos 8 42 0 0 r[8]>0 -> r[8]-=0, goto 42; check abort flag + 18 Gosub 16 39 0 0 ; goto clear accumulator subroutine + 19 AggStep 0 15 11 sum 0 accum=r[11] step(r[15]) + 20 AggStep 0 17 12 count 0 accum=r[12] step(r[17]) + 21 If 7 23 0 0 if r[7] goto 23; don't emit group columns if continuing existing group + 22 Column 2 0 10 0 r[10]=idx_employees_department.department + 23 Integer 1 7 0 0 r[7]=1; indicate data in accumulator + 24 Next 2 10 0 1 + 25 Gosub 6 29 0 0 ; emit row for final group + 26 Goto 0 42 0 0 ; group by finished + 27 Integer 1 8 0 0 r[8]=1 + 28 Return 6 0 0 0 + 29 IfPos 7 31 0 0 r[7]>0 -> r[7]-=0, goto 31; output group by row subroutine start + 30 Return 6 0 0 0 + 31 AggFinal 0 11 0 sum 0 accum=r[11] + 32 AggFinal 0 12 0 count 0 accum=r[12] + 33 Copy 11 18 0 0 r[18]=r[11] + 34 Copy 10 19 0 0 r[19]=r[10] + 35 Copy 12 20 0 0 r[20]=r[12] + 36 MakeRecord 18 3 5 0 r[5]=mkrec(r[18..20]) + 37 SorterInsert 0 5 0 0 0 key=r[5] + 38 Return 6 0 0 0 + 39 Null 0 10 12 0 r[10..12]=NULL; clear accumulator subroutine start + 40 Integer 0 7 0 0 r[7]=0 + 41 Return 16 0 0 0 + 42 OpenPseudo 3 5 3 0 3 columns in r[5] + 43 SorterSort 0 48 0 0 + 44 SorterData 0 5 3 0 r[5]=data + 45 ColumnRange 3 0 2 3 0 r[2..4]=pseudo.column 0..column 2 + 46 Yield 1 0 0 0 + 47 SorterNext 0 44 0 0 + 48 EndCoroutine 1 0 0 0 + 49 OpenEphemeral 4 1 0 0 cursor=4 is_table=true + 50 OpenDup 5 4 0 0 new_cursor=5, original_cursor=4 + 51 OpenDup 6 4 0 0 new_cursor=6, original_cursor=4 + 52 Null 0 30 0 0 r[30]=NULL + 53 InitCoroutine 1 0 2 0 + 54 Yield 1 76 0 0 + 55 Copy 2 32 0 0 r[32]=r[2] + 56 NotNull 30 64 0 0 r[30]!=NULL -> goto 64 + 57 Null 0 25 25 0 r[25..25]=NULL; reset accumulator registers + 58 MakeRecord 2 3 34 0 r[34]=mkrec(r[2..4]) + 59 NewRowid 5 30 0 0 r[30]=rowid + 60 Insert 5 34 30 buffer_table_window_0 2 intkey=r[30] data=r[34] + 61 Rewind 4 63 0 0 Rewind ephemeral(buffer_table_window_0) + 62 Rewind 6 63 0 0 Rewind ephemeral(buffer_table_window_0) + 63 Goto 0 75 0 0 + 64 MakeRecord 2 3 35 0 r[35]=mkrec(r[2..4]) + 65 NewRowid 5 30 0 0 r[30]=rowid + 66 Insert 5 35 30 buffer_table_window_0 2 intkey=r[30] data=r[35] + 67 AggStep 0 0 25 row_number 0 accum=r[25] step(r[0]) + 68 Next 6 69 0 0 + 69 AggValue 0 25 26 row_number 0 accum=r[25] dest=r[26] + 70 ColumnRange 4 1 27 2 0 r[27..28]=ephemeral(buffer_table_window_0).department..column 2 + 71 Column 4 0 29 0 r[29]=ephemeral(buffer_table_window_0).column 0 + 72 Gosub 33 91 0 0 + 73 Delete 4 0 0 buffer_table_window_0 0 + 74 Next 4 75 0 0 + 75 Goto 0 54 0 0 + 76 Null 0 31 0 0 r[31]=NULL; return remaining buffered rows + 77 Rewind 5 88 0 0 Rewind ephemeral(buffer_table_window_0) + 78 AggStep 0 0 25 row_number 0 accum=r[25] step(r[0]) + 79 Next 6 80 0 0 + 80 AggValue 0 25 26 row_number 0 accum=r[25] dest=r[26] + 81 ColumnRange 4 1 27 2 0 r[27..28]=ephemeral(buffer_table_window_0).department..column 2 + 82 Column 4 0 29 0 r[29]=ephemeral(buffer_table_window_0).column 0 + 83 Gosub 33 91 0 0 + 84 Delete 4 0 0 buffer_table_window_0 0 + 85 Next 4 87 0 0 + 86 Goto 0 88 0 0 + 87 Goto 0 80 0 0 + 88 ResetSorter 4 0 0 0 cursor=4 + 89 Return 31 0 1 0 + 90 Goto 0 97 0 0 + 91 Copy 27 21 0 0 r[21]=r[27] + 92 Copy 28 22 0 0 r[22]=r[28] + 93 Copy 29 23 0 0 r[23]=r[29] + 94 Copy 26 24 0 0 r[24]=r[26] + 95 ResultRow 21 4 0 0 output=r[21..24] + 96 Return 33 0 0 0 + 97 Halt 0 0 0 0 + 98 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 99 Integer 1 17 0 0 r[17]=1 + 100 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-with-subquery.snap b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-with-subquery.snap index 569bb96b..941c03a7 100644 --- a/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-with-subquery.snap +++ b/conformance/sqlite-sqltests/snapshot_tests/windows/snapshots/windows__window-with-subquery.snap @@ -35,9 +35,9 @@ QUERY PLAN BYTECODE addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 170 0 0 Start at 170 - 1 InitCoroutine 1 110 2 0 - 2 InitCoroutine 2 84 3 0 + 0 Init 0 155 0 0 Start at 155 + 1 InitCoroutine 1 103 2 0 + 2 InitCoroutine 2 82 3 0 3 InitCoroutine 3 15 4 0 4 OpenRead 0 2 0 k(5,B,B,B,B,B) 0 table=employees, root=2, iDb=0 5 OpenRead 1 4 0 k(2,B) 0 index=idx_employees_department, root=4, iDb=0 @@ -48,7 +48,7 @@ addr opcode p1 p2 p3 p4 p5 comment 10 Column 0 1 6 0 r[6]=employees.name 11 Column 0 3 7 0 r[7]=employees.salary 12 Yield 3 0 0 0 - 13 Next 1 7 0 0 + 13 Next 1 7 0 1 14 EndCoroutine 3 0 0 0 15 OpenEphemeral 2 1 0 0 cursor=2 is_table=true 16 OpenDup 3 2 0 0 new_cursor=3, original_cursor=2 @@ -56,10 +56,10 @@ addr opcode p1 p2 p3 p4 p5 comment 18 Null 0 19 0 0 r[19]=NULL 19 Null 0 20 0 0 r[20]=NULL 20 InitCoroutine 3 0 4 0 - 21 Yield 3 55 0 0 + 21 Yield 3 54 0 0 22 Compare 4 20 1 k(1, Binary) 0 r[4..4]==r[20..20]; compare partition keys to detect new partition 23 Jump 24 27 24 0 - 24 Gosub 21 56 0 0 ; detected new partition + 24 Gosub 21 55 0 0 ; detected new partition 25 Null 0 19 0 0 r[19]=NULL 26 Copy 4 20 0 0 r[20]=r[4] 27 NotNull 19 35 0 0 r[19]!=NULL -> goto 35 @@ -69,141 +69,126 @@ addr opcode p1 p2 p3 p4 p5 comment 31 Insert 3 23 19 buffer_table_window_0 2 intkey=r[19] data=r[23] 32 Rewind 2 34 0 0 Rewind ephemeral(buffer_table_window_0) 33 Rewind 4 34 0 0 Rewind ephemeral(buffer_table_window_0) - 34 Goto 0 54 0 0 + 34 Goto 0 53 0 0 35 MakeRecord 4 4 24 0 r[24]=mkrec(r[4..7]) 36 NewRowid 3 19 0 0 r[19]=rowid 37 Insert 3 24 19 buffer_table_window_0 2 intkey=r[19] data=r[24] - 38 Goto 0 54 0 0 + 38 Goto 0 53 0 0 ; peer of previous row: buffer only, handle at group end 39 Column 4 3 25 0 r[25]=ephemeral(buffer_table_window_0).salary 40 AggStep 0 25 13 avg 0 accum=r[13] step(r[25]) 41 Next 4 43 0 0 42 Goto 0 44 0 0 43 Goto 0 39 0 0 44 AggValue 0 13 14 avg 0 accum=r[13] dest=r[14] - 45 Column 2 1 15 0 r[15]=ephemeral(buffer_table_window_0).emp_id - 46 Column 2 2 16 0 r[16]=ephemeral(buffer_table_window_0).name - 47 Column 2 0 17 0 r[17]=ephemeral(buffer_table_window_0).department - 48 Column 2 3 18 0 r[18]=ephemeral(buffer_table_window_0).salary - 49 Gosub 22 76 0 0 - 50 Delete 2 0 0 buffer_table_window_0 0 - 51 Next 2 53 0 0 - 52 Goto 0 54 0 0 - 53 Goto 0 45 0 0 - 54 Goto 0 21 0 0 - 55 Null 0 21 0 0 r[21]=NULL; return remaining buffered rows - 56 Rewind 3 73 0 0 Rewind ephemeral(buffer_table_window_0) - 57 Column 4 3 26 0 r[26]=ephemeral(buffer_table_window_0).salary - 58 AggStep 0 26 13 avg 0 accum=r[13] step(r[26]) - 59 Next 4 61 0 0 - 60 Goto 0 62 0 0 - 61 Goto 0 57 0 0 - 62 AggValue 0 13 14 avg 0 accum=r[13] dest=r[14] - 63 Column 2 1 15 0 r[15]=ephemeral(buffer_table_window_0).emp_id - 64 Column 2 2 16 0 r[16]=ephemeral(buffer_table_window_0).name - 65 Column 2 0 17 0 r[17]=ephemeral(buffer_table_window_0).department - 66 Column 2 3 18 0 r[18]=ephemeral(buffer_table_window_0).salary - 67 Gosub 22 76 0 0 - 68 Delete 2 0 0 buffer_table_window_0 0 - 69 Next 2 71 0 0 - 70 Goto 0 73 0 0 - 71 Goto 0 63 0 0 - 72 Goto 0 62 0 0 - 73 ResetSorter 2 0 0 0 cursor=2 - 74 Return 21 0 1 0 - 75 Goto 0 83 0 0 - 76 Copy 15 8 0 0 r[8]=r[15] - 77 Copy 16 9 0 0 r[9]=r[16] - 78 Copy 17 10 0 0 r[10]=r[17] - 79 Copy 18 11 0 0 r[11]=r[18] - 80 Copy 14 12 0 0 r[12]=r[14] - 81 Yield 2 0 0 0 - 82 Return 22 0 0 0 - 83 EndCoroutine 2 0 0 0 - 84 SorterOpen 5 1 0 k(1,-B) 0 cursor=5 - 85 InitCoroutine 2 0 3 0 - 86 Yield 2 98 0 0 - 87 Copy 11 40 0 0 r[40]=r[11] - 88 Copy 12 41 0 0 r[41]=r[12] - 89 Subtract 40 41 34 0 r[34]=r[40]-r[41] - 90 Copy 8 35 0 0 r[35]=r[8] - 91 Copy 9 36 0 0 r[36]=r[9] - 92 Copy 10 37 0 0 r[37]=r[10] - 93 Copy 11 38 0 0 r[38]=r[11] - 94 Copy 12 39 0 0 r[39]=r[12] - 95 MakeRecord 34 6 33 0 r[33]=mkrec(r[34..39]) - 96 SorterInsert 5 33 0 0 0 key=r[33] - 97 Goto 0 86 0 0 - 98 OpenPseudo 6 33 6 0 6 columns in r[33] - 99 SorterSort 5 109 0 0 - 100 SorterData 5 33 6 0 r[33]=data - 101 Column 6 0 27 0 r[27]=pseudo.column 0 - 102 Column 6 1 28 0 r[28]=pseudo.column 1 - 103 Column 6 2 29 0 r[29]=pseudo.column 2 - 104 Column 6 3 30 0 r[30]=pseudo.column 3 - 105 Column 6 4 31 0 r[31]=pseudo.column 4 - 106 Column 6 5 32 0 r[32]=pseudo.column 5 - 107 Yield 1 0 0 0 - 108 SorterNext 5 100 0 0 - 109 EndCoroutine 1 0 0 0 - 110 OpenEphemeral 7 1 0 0 cursor=7 is_table=true - 111 OpenDup 8 7 0 0 new_cursor=8, original_cursor=7 - 112 OpenDup 9 7 0 0 new_cursor=9, original_cursor=7 - 113 Null 0 56 0 0 r[56]=NULL - 114 InitCoroutine 1 0 2 0 - 115 Yield 1 140 0 0 - 116 Copy 27 58 0 0 r[58]=r[27] - 117 NotNull 56 125 0 0 r[56]!=NULL -> goto 125 - 118 Null 0 49 49 0 r[49..49]=NULL; initialize per-cursor peer-reference registers for new partition - 119 MakeRecord 27 6 60 0 r[60]=mkrec(r[27..32]) - 120 NewRowid 8 56 0 0 r[56]=rowid - 121 Insert 8 60 56 buffer_table_window_0 2 intkey=r[56] data=r[60] - 122 Rewind 7 124 0 0 Rewind ephemeral(buffer_table_window_0) - 123 Rewind 9 124 0 0 Rewind ephemeral(buffer_table_window_0) - 124 Goto 0 139 0 0 - 125 MakeRecord 27 6 61 0 r[61]=mkrec(r[27..32]) - 126 NewRowid 8 56 0 0 r[56]=rowid - 127 Insert 8 61 56 buffer_table_window_0 2 intkey=r[56] data=r[61] - 128 AggStep 0 0 49 row_number 0 accum=r[49] step(r[0]) - 129 Next 9 130 0 0 - 130 AggValue 0 49 50 row_number 0 accum=r[49] dest=r[50] - 131 Column 7 1 51 0 r[51]=ephemeral(buffer_table_window_0).emp_id - 132 Column 7 2 52 0 r[52]=ephemeral(buffer_table_window_0).name - 133 Column 7 3 53 0 r[53]=ephemeral(buffer_table_window_0).department - 134 Column 7 4 54 0 r[54]=ephemeral(buffer_table_window_0).salary - 135 Column 7 5 55 0 r[55]=ephemeral(buffer_table_window_0).dept_avg - 136 Gosub 59 158 0 0 - 137 Delete 7 0 0 buffer_table_window_0 0 - 138 Next 7 139 0 0 - 139 Goto 0 115 0 0 - 140 Null 0 57 0 0 r[57]=NULL; return remaining buffered rows - 141 Rewind 8 155 0 0 Rewind ephemeral(buffer_table_window_0) - 142 AggStep 0 0 49 row_number 0 accum=r[49] step(r[0]) - 143 Next 9 144 0 0 - 144 AggValue 0 49 50 row_number 0 accum=r[49] dest=r[50] - 145 Column 7 1 51 0 r[51]=ephemeral(buffer_table_window_0).emp_id - 146 Column 7 2 52 0 r[52]=ephemeral(buffer_table_window_0).name - 147 Column 7 3 53 0 r[53]=ephemeral(buffer_table_window_0).department - 148 Column 7 4 54 0 r[54]=ephemeral(buffer_table_window_0).salary - 149 Column 7 5 55 0 r[55]=ephemeral(buffer_table_window_0).dept_avg - 150 Gosub 59 158 0 0 - 151 Delete 7 0 0 buffer_table_window_0 0 - 152 Next 7 154 0 0 - 153 Goto 0 155 0 0 - 154 Goto 0 144 0 0 - 155 ResetSorter 7 0 0 0 cursor=7 - 156 Return 57 0 1 0 - 157 Goto 0 169 0 0 - 158 Copy 51 42 0 0 r[42]=r[51] - 159 Copy 52 43 0 0 r[43]=r[52] - 160 Copy 53 44 0 0 r[44]=r[53] - 161 Copy 54 45 0 0 r[45]=r[54] - 162 Copy 55 46 0 0 r[46]=r[55] - 163 Copy 54 62 0 0 r[62]=r[54] - 164 Copy 55 63 0 0 r[63]=r[55] - 165 Subtract 62 63 47 0 r[47]=r[62]-r[63] - 166 Copy 50 48 0 0 r[48]=r[50] - 167 ResultRow 42 7 0 0 output=r[42..48] - 168 Return 59 0 0 0 - 169 Halt 0 0 0 0 - 170 Transaction 0 1 7 0 iDb=0 tx_mode=Read - 171 Goto 0 1 0 0 + 45 ColumnRange 2 1 15 2 0 r[15..16]=ephemeral(buffer_table_window_0).emp_id..name + 46 Column 2 0 17 0 r[17]=ephemeral(buffer_table_window_0).department + 47 Column 2 3 18 0 r[18]=ephemeral(buffer_table_window_0).salary + 48 Gosub 22 74 0 0 + 49 Delete 2 0 0 buffer_table_window_0 0 + 50 Next 2 52 0 0 + 51 Goto 0 53 0 0 + 52 Goto 0 45 0 0 + 53 Goto 0 21 0 0 + 54 Null 0 21 0 0 r[21]=NULL; return remaining buffered rows + 55 Rewind 3 71 0 0 Rewind ephemeral(buffer_table_window_0) + 56 Column 4 3 26 0 r[26]=ephemeral(buffer_table_window_0).salary + 57 AggStep 0 26 13 avg 0 accum=r[13] step(r[26]) + 58 Next 4 60 0 0 + 59 Goto 0 61 0 0 + 60 Goto 0 56 0 0 + 61 AggValue 0 13 14 avg 0 accum=r[13] dest=r[14] + 62 ColumnRange 2 1 15 2 0 r[15..16]=ephemeral(buffer_table_window_0).emp_id..name + 63 Column 2 0 17 0 r[17]=ephemeral(buffer_table_window_0).department + 64 Column 2 3 18 0 r[18]=ephemeral(buffer_table_window_0).salary + 65 Gosub 22 74 0 0 + 66 Delete 2 0 0 buffer_table_window_0 0 + 67 Next 2 69 0 0 + 68 Goto 0 71 0 0 + 69 Goto 0 62 0 0 + 70 Goto 0 61 0 0 + 71 ResetSorter 2 0 0 0 cursor=2 + 72 Return 21 0 1 0 + 73 Goto 0 81 0 0 + 74 Copy 15 8 0 0 r[8]=r[15] + 75 Copy 16 9 0 0 r[9]=r[16] + 76 Copy 17 10 0 0 r[10]=r[17] + 77 Copy 18 11 0 0 r[11]=r[18] + 78 Copy 14 12 0 0 r[12]=r[14] + 79 Yield 2 0 0 0 + 80 Return 22 0 0 0 + 81 EndCoroutine 2 0 0 0 + 82 SorterOpen 5 1 0 k(1,-B) 0 cursor=5 + 83 InitCoroutine 2 0 3 0 + 84 Yield 2 96 0 0 + 85 Copy 11 40 0 0 r[40]=r[11] + 86 Copy 12 41 0 0 r[41]=r[12] + 87 Subtract 40 41 34 0 r[34]=r[40]-r[41] + 88 Copy 8 35 0 0 r[35]=r[8] + 89 Copy 9 36 0 0 r[36]=r[9] + 90 Copy 10 37 0 0 r[37]=r[10] + 91 Copy 11 38 0 0 r[38]=r[11] + 92 Copy 12 39 0 0 r[39]=r[12] + 93 MakeRecord 34 6 33 0 r[33]=mkrec(r[34..39]) + 94 SorterInsert 5 33 0 0 0 key=r[33] + 95 Goto 0 84 0 0 + 96 OpenPseudo 6 33 6 0 6 columns in r[33] + 97 SorterSort 5 102 0 0 + 98 SorterData 5 33 6 0 r[33]=data + 99 ColumnRange 6 0 27 6 0 r[27..32]=pseudo.column 0..column 5 + 100 Yield 1 0 0 0 + 101 SorterNext 5 98 0 0 + 102 EndCoroutine 1 0 0 0 + 103 OpenEphemeral 7 1 0 0 cursor=7 is_table=true + 104 OpenDup 8 7 0 0 new_cursor=8, original_cursor=7 + 105 OpenDup 9 7 0 0 new_cursor=9, original_cursor=7 + 106 Null 0 56 0 0 r[56]=NULL + 107 InitCoroutine 1 0 2 0 + 108 Yield 1 129 0 0 + 109 Copy 27 58 0 0 r[58]=r[27] + 110 NotNull 56 118 0 0 r[56]!=NULL -> goto 118 + 111 Null 0 49 49 0 r[49..49]=NULL; reset accumulator registers + 112 MakeRecord 27 6 60 0 r[60]=mkrec(r[27..32]) + 113 NewRowid 8 56 0 0 r[56]=rowid + 114 Insert 8 60 56 buffer_table_window_0 2 intkey=r[56] data=r[60] + 115 Rewind 7 117 0 0 Rewind ephemeral(buffer_table_window_0) + 116 Rewind 9 117 0 0 Rewind ephemeral(buffer_table_window_0) + 117 Goto 0 128 0 0 + 118 MakeRecord 27 6 61 0 r[61]=mkrec(r[27..32]) + 119 NewRowid 8 56 0 0 r[56]=rowid + 120 Insert 8 61 56 buffer_table_window_0 2 intkey=r[56] data=r[61] + 121 AggStep 0 0 49 row_number 0 accum=r[49] step(r[0]) + 122 Next 9 123 0 0 + 123 AggValue 0 49 50 row_number 0 accum=r[49] dest=r[50] + 124 ColumnRange 7 1 51 5 0 r[51..55]=ephemeral(buffer_table_window_0).emp_id..dept_avg + 125 Gosub 59 143 0 0 + 126 Delete 7 0 0 buffer_table_window_0 0 + 127 Next 7 128 0 0 + 128 Goto 0 108 0 0 + 129 Null 0 57 0 0 r[57]=NULL; return remaining buffered rows + 130 Rewind 8 140 0 0 Rewind ephemeral(buffer_table_window_0) + 131 AggStep 0 0 49 row_number 0 accum=r[49] step(r[0]) + 132 Next 9 133 0 0 + 133 AggValue 0 49 50 row_number 0 accum=r[49] dest=r[50] + 134 ColumnRange 7 1 51 5 0 r[51..55]=ephemeral(buffer_table_window_0).emp_id..dept_avg + 135 Gosub 59 143 0 0 + 136 Delete 7 0 0 buffer_table_window_0 0 + 137 Next 7 139 0 0 + 138 Goto 0 140 0 0 + 139 Goto 0 133 0 0 + 140 ResetSorter 7 0 0 0 cursor=7 + 141 Return 57 0 1 0 + 142 Goto 0 154 0 0 + 143 Copy 51 42 0 0 r[42]=r[51] + 144 Copy 52 43 0 0 r[43]=r[52] + 145 Copy 53 44 0 0 r[44]=r[53] + 146 Copy 54 45 0 0 r[45]=r[54] + 147 Copy 55 46 0 0 r[46]=r[55] + 148 Copy 54 62 0 0 r[62]=r[54] + 149 Copy 55 63 0 0 r[63]=r[55] + 150 Subtract 62 63 47 0 r[47]=r[62]-r[63] + 151 Copy 50 48 0 0 r[48]=r[50] + 152 ResultRow 42 7 0 0 output=r[42..48] + 153 Return 59 0 0 0 + 154 Halt 0 0 0 0 + 155 Transaction 0 1 7 0 iDb=0 tx_mode=Read + 156 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/cte_cardinality__small-cte-materialized-big-cte-outer.snap b/conformance/sqlite-sqltests/snapshots/cte_cardinality__small-cte-materialized-big-cte-outer.snap index a7787c48..59f49813 100644 --- a/conformance/sqlite-sqltests/snapshots/cte_cardinality__small-cte-materialized-big-cte-outer.snap +++ b/conformance/sqlite-sqltests/snapshots/cte_cardinality__small-cte-materialized-big-cte-outer.snap @@ -22,7 +22,7 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN small AS s +|--SCAN big AS b | `--SCAN items -`--SEARCH b USING INDEX ephemeral_subquery_t4 (id=?) +`--SEARCH s USING INDEX ephemeral_subquery_t2 (id=?) `--SCAN items diff --git a/conformance/sqlite-sqltests/snapshots/cte_cardinality__three-ctes-big-outer-others-materialized.snap b/conformance/sqlite-sqltests/snapshots/cte_cardinality__three-ctes-big-outer-others-materialized.snap index 94aca42e..5e6f7f8d 100644 --- a/conformance/sqlite-sqltests/snapshots/cte_cardinality__three-ctes-big-outer-others-materialized.snap +++ b/conformance/sqlite-sqltests/snapshots/cte_cardinality__three-ctes-big-outer-others-materialized.snap @@ -28,9 +28,9 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN tiny AS t +|--SCAN big AS b | `--SCAN items |--SEARCH m USING INDEX ephemeral_subquery_t4 (id=?) | `--SCAN items -`--SEARCH b USING INDEX ephemeral_subquery_t6 (id=?) +`--SEARCH t USING INDEX ephemeral_subquery_t2 (id=?) `--SCAN items diff --git a/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-column-no-child-points-at.snap b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-column-no-child-points-at.snap new file mode 100644 index 00000000..5ebd555c --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-column-no-child-points-at.snap @@ -0,0 +1,34 @@ +--- +source: foreign_keys.sqltest +expression: UPDATE parent SET v = v + 1 WHERE id = 1; +info: + statement_type: UPDATE + tables: + - parent + setup_blocks: + - fk_bytecode_no_action_child + database: ':memory:' +--- +QUERY PLAN +`--SEARCH parent USING INTEGER PRIMARY KEY (rowid=?) + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 14 0 0 Start at 14 + 1 OpenWrite 0 2 0 0 root=2; iDb=0 + 2 SeekRowid 0 1 13 0 if (r[1]!=cursor 0 for table parent.rowid) goto 13 + 3 RowId 0 2 0 0 r[2]=parent.rowid + 4 IsNull 2 13 0 0 if (r[2]==NULL) goto 13 + 5 Null 0 3 0 0 r[3]=NULL + 6 Column 0 1 4 0 r[4]=parent.k + 7 Column 0 2 6 0 r[6]=parent.v + 8 Add 6 7 5 0 r[5]=r[6]+r[7] + 9 Affinity 3 3 0 0 r[3..6] = D, D, D + 10 Copy 2 3 0 0 r[3]=r[2] + 11 MakeRecord 3 3 8 0 r[8]=mkrec(r[3..5]) + 12 Insert 0 8 2 parent 8 intkey=r[2] data=r[8] + 13 Halt 0 0 0 0 + 14 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 15 Integer 1 1 0 0 r[1]=1 + 16 Integer 1 7 0 0 r[7]=1 + 17 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-cascade-child.snap b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-cascade-child.snap new file mode 100644 index 00000000..169f06dd --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-cascade-child.snap @@ -0,0 +1,88 @@ +--- +source: foreign_keys.sqltest +expression: UPDATE parent SET k = k + 1 WHERE id = 1; +info: + statement_type: UPDATE + tables: + - parent + setup_blocks: + - fk_bytecode_cascade_child + database: ':memory:' +--- +QUERY PLAN +`--SEARCH parent USING INTEGER PRIMARY KEY (rowid=?) + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 43 0 0 Start at 43 + 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true + 2 OpenRead 1 2 0 k(3,B,B,B) 0 table=parent, root=2, iDb=0 + 3 SeekRowid 1 2 7 0 if (r[2]!=cursor 1 for table parent.rowid) goto 7 + 4 RowId 1 1 0 0 r[1]=parent.rowid + 5 MakeRecord 1 1 3 0 r[3]=mkrec(r[1..1]); for ephemeral_scratch + 6 Insert 0 3 1 ephemeral_scratch 6 intkey=r[1] data=r[3] + 7 OpenWrite 1 2 0 0 root=2; iDb=0 + 8 OpenWrite 2 3 0 0 root=3; iDb=0 + 9 Rewind 0 42 0 0 Rewind table ephemeral(ephemeral_scratch) + 10 RowId 0 5 0 0 r[5]=ephemeral(ephemeral_scratch).rowid + 11 NotExists 1 41 5 0 + 12 Null 0 6 0 0 r[6]=NULL + 13 Column 1 1 9 0 r[9]=parent.k + 14 Add 9 10 7 0 r[7]=r[9]+r[10] + 15 Column 1 2 8 0 r[8]=parent.v + 16 Affinity 6 3 0 0 r[6..9] = D, D, D + 17 RowId 1 11 0 0 r[11]=parent.rowid + 18 ColumnRange 1 1 12 2 0 r[12..13]=parent.k..v + 19 Copy 5 6 0 0 r[6]=r[5] + 20 Copy 7 14 0 0 r[14]=r[7] + 21 Copy 5 15 0 0 r[15]=r[5] + 22 Affinity 14 1 0 0 r[14..15] = D + 23 MakeRecord 14 2 4 0 r[4]=mkrec(r[14..15]); for sqlite_autoindex_parent_1 + 24 NoConflict 2 28 14 1 0 key=r[14..14] + 25 IdxRowId 2 16 0 0 r[16]=cursor 2.rowid + 26 Eq 5 16 28 Binary 0 if r[5]==r[16] goto 28 + 27 Halt 2067 0 0 parent.k 0 + 28 Column 1 1 17 0 r[17]=parent.k + 29 RowId 1 18 0 0 r[18]=parent.rowid + 30 IdxDelete 2 17 2 1 + 31 IdxInsert 2 4 14 2 key=r[4] + 32 MakeRecord 6 3 19 0 r[19]=mkrec(r[6..8]) + 33 NotExists 1 41 5 0 + 34 Delete 1 0 0 parent 0 + 35 Insert 1 19 5 parent 11 intkey=r[5] data=r[19] + 36 Copy 12 20 0 0 r[20]=r[12] + 37 Copy 7 21 0 0 r[21]=r[7] + 38 Eq 20 21 41 0 if r[20]==r[21] goto 41 + 39 Goto 0 40 0 0 + 40 Program 20 41 2 fk cascade update 0 subprogram=fk cascade update + 41 Next 0 10 0 1 + 42 Halt 0 0 0 0 + 43 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 44 Integer 1 2 0 0 r[2]=1 + 45 Integer 1 10 0 0 r[10]=1 + 46 Goto 0 1 0 0 + 0 Init 0 1 0 0 Start at 1 + 1 OpenWrite 0 4 0 0 root=4; iDb=0 + 2 Rewind 0 24 0 0 Rewind table child + 3 Column 0 1 2 0 r[2]=child.pk + 4 Variable 1 3 0 0 r[3]=parameter(1) + 5 Ne 2 3 23 Binary 0 if r[2]!=r[3] goto 23 + 6 RowId 0 4 0 0 r[4]=child.rowid + 7 IsNull 4 24 0 0 if (r[4]==NULL) goto 24 + 8 Null 0 5 0 0 r[5]=NULL + 9 Variable 2 6 0 0 r[6]=parameter(2) + 10 Affinity 5 2 0 0 r[5..7] = D, D + 11 Copy 4 5 0 0 r[5]=r[4] + 12 IsNull 6 21 0 0 if (r[6]==NULL) goto 21 + 13 OpenRead 1 3 0 k(2,B) 0 =sqlite_autoindex_parent_1, root=3, iDb=0 + 14 Copy 6 7 0 0 r[7]=r[6] + 15 Affinity 7 1 0 0 r[7..8] = D + 16 Found 1 19 7 0 if found goto 19 + 17 FkCounter 1 0 0 0 + 18 Goto 0 19 0 0 + 19 Close 1 0 0 0 + 20 Goto 0 21 0 0 + 21 MakeRecord 5 2 8 0 r[8]=mkrec(r[5..6]) + 22 Insert 0 8 4 child 8 intkey=r[4] data=r[8] + 23 Next 0 3 0 1 + 24 Halt 0 0 0 fk action 0 diff --git a/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-mixed-children.snap b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-mixed-children.snap new file mode 100644 index 00000000..88c9b409 --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-mixed-children.snap @@ -0,0 +1,110 @@ +--- +source: foreign_keys.sqltest +expression: UPDATE parent SET k = k + 1 WHERE id = 1; +info: + statement_type: UPDATE + tables: + - parent + setup_blocks: + - fk_bytecode_mixed_children + database: ':memory:' +--- +QUERY PLAN +`--SEARCH parent USING INTEGER PRIMARY KEY (rowid=?) + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 65 0 0 Start at 65 + 1 OpenEphemeral 0 1 0 0 cursor=0 is_table=true + 2 OpenRead 1 2 0 k(2,B,B) 0 table=parent, root=2, iDb=0 + 3 SeekRowid 1 2 7 0 if (r[2]!=cursor 1 for table parent.rowid) goto 7 + 4 RowId 1 1 0 0 r[1]=parent.rowid + 5 MakeRecord 1 1 3 0 r[3]=mkrec(r[1..1]); for ephemeral_scratch + 6 Insert 0 3 1 ephemeral_scratch 6 intkey=r[1] data=r[3] + 7 OpenWrite 1 2 0 0 root=2; iDb=0 + 8 OpenWrite 2 3 0 0 root=3; iDb=0 + 9 Rewind 0 64 0 0 Rewind table ephemeral(ephemeral_scratch) + 10 RowId 0 5 0 0 r[5]=ephemeral(ephemeral_scratch).rowid + 11 NotExists 1 63 5 0 + 12 Null 0 6 0 0 r[6]=NULL + 13 Column 1 1 8 0 r[8]=parent.k + 14 Add 8 9 7 0 r[7]=r[8]+r[9] + 15 Affinity 6 2 0 0 r[6..8] = D, D + 16 RowId 1 10 0 0 r[10]=parent.rowid + 17 Column 1 1 11 0 r[11]=parent.k + 18 Copy 5 6 0 0 r[6]=r[5] + 19 Copy 7 12 0 0 r[12]=r[7] + 20 Copy 5 13 0 0 r[13]=r[5] + 21 Affinity 12 1 0 0 r[12..13] = D + 22 MakeRecord 12 2 4 0 r[4]=mkrec(r[12..13]); for sqlite_autoindex_parent_1 + 23 NoConflict 2 27 12 1 0 key=r[12..12] + 24 IdxRowId 2 14 0 0 r[14]=cursor 2.rowid + 25 Eq 5 14 27 Binary 0 if r[5]==r[14] goto 27 + 26 Halt 2067 0 0 parent.k 0 + 27 Column 1 1 15 0 r[15]=parent.k + 28 Copy 7 16 0 0 r[16]=r[7] + 29 Eq 15 16 50 0 if r[15]==r[16] goto 50 + 30 Goto 0 31 0 0 + 31 IsNull 15 41 0 0 if (r[15]==NULL) goto 41 + 32 OpenRead 3 4 0 k(2,B,B) 0 =child_no_action, root=4, iDb=0 + 33 Rewind 3 40 0 0 Rewind child_no_action + 34 Column 3 1 17 0 r[17]=child_no_action.pk + 35 IsNull 17 39 0 0 if (r[17]==NULL) goto 39 + 36 Eq 17 15 38 Binary 0 if r[17]==r[15] goto 38 + 37 Goto 0 39 0 0 + 38 FkCounter 1 0 0 0 + 39 Next 3 34 0 0 + 40 Close 3 0 0 0 + 41 IsNull 16 50 0 0 if (r[16]==NULL) goto 50 + 42 OpenRead 4 4 0 k(2,B,B) 0 =child_no_action, root=4, iDb=0 + 43 Rewind 4 49 0 0 Rewind child_no_action + 44 Column 4 1 18 0 r[18]=child_no_action.pk + 45 IsNull 18 48 0 0 if (r[18]==NULL) goto 48 + 46 Eq 18 16 48 Binary 0 if r[18]==r[16] goto 48 + 47 Goto 0 48 0 0 + 48 Next 4 44 0 0 + 49 Close 4 0 0 0 + 50 Column 1 1 19 0 r[19]=parent.k + 51 RowId 1 20 0 0 r[20]=parent.rowid + 52 IdxDelete 2 19 2 1 + 53 IdxInsert 2 4 12 2 key=r[4] + 54 MakeRecord 6 2 21 0 r[21]=mkrec(r[6..7]) + 55 NotExists 1 63 5 0 + 56 Delete 1 0 0 parent 0 + 57 Insert 1 21 5 parent 11 intkey=r[5] data=r[21] + 58 Copy 11 22 0 0 r[22]=r[11] + 59 Copy 7 23 0 0 r[23]=r[7] + 60 Eq 22 23 63 0 if r[22]==r[23] goto 63 + 61 Goto 0 62 0 0 + 62 Program 22 63 2 fk cascade update 0 subprogram=fk cascade update + 63 Next 0 10 0 1 + 64 Halt 0 0 0 0 + 65 Transaction 0 2 3 0 iDb=0 tx_mode=Write + 66 Integer 1 2 0 0 r[2]=1 + 67 Integer 1 9 0 0 r[9]=1 + 68 Goto 0 1 0 0 + 0 Init 0 1 0 0 Start at 1 + 1 OpenWrite 0 5 0 0 root=5; iDb=0 + 2 Rewind 0 24 0 0 Rewind table child_cascade + 3 Column 0 1 2 0 r[2]=child_cascade.pk + 4 Variable 1 3 0 0 r[3]=parameter(1) + 5 Ne 2 3 23 Binary 0 if r[2]!=r[3] goto 23 + 6 RowId 0 4 0 0 r[4]=child_cascade.rowid + 7 IsNull 4 24 0 0 if (r[4]==NULL) goto 24 + 8 Null 0 5 0 0 r[5]=NULL + 9 Variable 2 6 0 0 r[6]=parameter(2) + 10 Affinity 5 2 0 0 r[5..7] = D, D + 11 Copy 4 5 0 0 r[5]=r[4] + 12 IsNull 6 21 0 0 if (r[6]==NULL) goto 21 + 13 OpenRead 1 3 0 k(2,B) 0 =sqlite_autoindex_parent_1, root=3, iDb=0 + 14 Copy 6 7 0 0 r[7]=r[6] + 15 Affinity 7 1 0 0 r[7..8] = D + 16 Found 1 19 7 0 if found goto 19 + 17 FkCounter 1 0 0 0 + 18 Goto 0 19 0 0 + 19 Close 1 0 0 0 + 20 Goto 0 21 0 0 + 21 MakeRecord 5 2 8 0 r[8]=mkrec(r[5..6]) + 22 Insert 0 8 4 child_cascade 8 intkey=r[4] data=r[8] + 23 Next 0 3 0 1 + 24 Halt 0 0 0 fk action 0 diff --git a/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-no-action-child.snap b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-no-action-child.snap new file mode 100644 index 00000000..96c3292a --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-update-parent-key-with-no-action-child.snap @@ -0,0 +1,70 @@ +--- +source: foreign_keys.sqltest +expression: UPDATE parent SET k = k + 1 WHERE id = 1; +info: + statement_type: UPDATE + tables: + - parent + setup_blocks: + - fk_bytecode_no_action_child + database: ':memory:' +--- +QUERY PLAN +`--SEARCH parent USING INTEGER PRIMARY KEY (rowid=?) + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 50 0 0 Start at 50 + 1 OpenWrite 0 2 0 0 root=2; iDb=0 + 2 OpenWrite 1 3 0 0 root=3; iDb=0 + 3 SeekRowid 0 2 49 0 if (r[2]!=cursor 0 for table parent.rowid) goto 49 + 4 RowId 0 3 0 0 r[3]=parent.rowid + 5 IsNull 3 49 0 0 if (r[3]==NULL) goto 49 + 6 Null 0 4 0 0 r[4]=NULL + 7 Column 0 1 7 0 r[7]=parent.k + 8 Add 7 8 5 0 r[5]=r[7]+r[8] + 9 Column 0 2 6 0 r[6]=parent.v + 10 Affinity 4 3 0 0 r[4..7] = D, D, D + 11 Copy 3 4 0 0 r[4]=r[3] + 12 Copy 5 9 0 0 r[9]=r[5] + 13 Copy 3 10 0 0 r[10]=r[3] + 14 Affinity 9 1 0 0 r[9..10] = D + 15 MakeRecord 9 2 1 0 r[1]=mkrec(r[9..10]); for sqlite_autoindex_parent_1 + 16 NoConflict 1 20 9 1 0 key=r[9..9] + 17 IdxRowId 1 11 0 0 r[11]=cursor 1.rowid + 18 Eq 3 11 20 Binary 0 if r[3]==r[11] goto 20 + 19 Halt 2067 0 0 parent.k 0 + 20 Column 0 1 12 0 r[12]=parent.k + 21 Copy 5 13 0 0 r[13]=r[5] + 22 Eq 12 13 43 0 if r[12]==r[13] goto 43 + 23 Goto 0 24 0 0 + 24 IsNull 12 34 0 0 if (r[12]==NULL) goto 34 + 25 OpenRead 2 4 0 k(2,B,B) 0 =child, root=4, iDb=0 + 26 Rewind 2 33 0 0 Rewind child + 27 Column 2 1 14 0 r[14]=child.pk + 28 IsNull 14 32 0 0 if (r[14]==NULL) goto 32 + 29 Eq 14 12 31 Binary 0 if r[14]==r[12] goto 31 + 30 Goto 0 32 0 0 + 31 FkCounter 1 0 0 0 + 32 Next 2 27 0 0 + 33 Close 2 0 0 0 + 34 IsNull 13 43 0 0 if (r[13]==NULL) goto 43 + 35 OpenRead 3 4 0 k(2,B,B) 0 =child, root=4, iDb=0 + 36 Rewind 3 42 0 0 Rewind child + 37 Column 3 1 15 0 r[15]=child.pk + 38 IsNull 15 41 0 0 if (r[15]==NULL) goto 41 + 39 Eq 15 13 41 Binary 0 if r[15]==r[13] goto 41 + 40 Goto 0 41 0 0 + 41 Next 3 37 0 0 + 42 Close 3 0 0 0 + 43 Column 0 1 16 0 r[16]=parent.k + 44 RowId 0 17 0 0 r[17]=parent.rowid + 45 IdxDelete 1 16 2 1 + 46 IdxInsert 1 1 9 2 key=r[1] + 47 MakeRecord 4 3 18 0 r[18]=mkrec(r[4..6]) + 48 Insert 0 18 3 parent 8 intkey=r[3] data=r[18] + 49 Halt 0 0 0 0 + 50 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 51 Integer 1 2 0 0 r[2]=1 + 52 Integer 1 8 0 0 r[8]=1 + 53 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-upsert-column-no-child-points-at.snap b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-upsert-column-no-child-points-at.snap new file mode 100644 index 00000000..40cfa793 --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-upsert-column-no-child-points-at.snap @@ -0,0 +1,59 @@ +--- +source: foreign_keys.sqltest +expression: INSERT INTO parent(id, k, v) VALUES (1, 10, 2) ON CONFLICT(id) DO UPDATE SET v = excluded.v; +info: + statement_type: INSERT + tables: + - conflict + - parent + setup_blocks: + - fk_bytecode_cascade_child + database: ':memory:' +--- +QUERY PLAN + + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 40 0 0 Start at 40 + 1 OpenWrite 1 2 0 0 root=2; iDb=0 + 2 Integer 1 2 0 0 r[2]=1 + 3 SoftNull 3 0 0 0 + 4 Integer 10 4 0 0 r[4]=10 + 5 Integer 2 5 0 0 r[5]=2 + 6 OpenWrite 0 3 0 0 root=3; iDb=0 + 7 Affinity 3 3 0 0 r[3..6] = D, D, D + 8 NotNull 2 10 0 0 r[2]!=NULL -> goto 10 + 9 Goto 0 12 0 0 + 10 MustBeInt 2 0 0 0 + 11 Goto 0 13 0 0 + 12 NewRowid 1 2 0 0 r[2]=rowid + 13 NotExists 1 16 2 0 + 14 Copy 2 1 0 0 r[1]=r[2] + 15 Goto 0 28 0 0 + 16 Copy 4 7 0 0 r[7]=r[4] + 17 Copy 2 8 0 0 r[8]=r[2] + 18 Affinity 7 1 0 0 r[7..8] = D + 19 NoConflict 0 21 7 1 0 key=r[7..7] + 20 Halt 2067 0 0 parent.k 0 + 21 MakeRecord 3 3 6 0 r[6]=mkrec(r[3..5]) + 22 Copy 4 9 0 0 r[9]=r[4] + 23 Copy 2 10 0 0 r[10]=r[2] + 24 MakeRecord 9 2 11 0 r[11]=mkrec(r[9..10]); for sqlite_autoindex_parent_1 + 25 IdxInsert 0 11 9 2 key=r[11] + 26 Insert 1 6 2 parent 0 intkey=r[2] data=r[6] + 27 Goto 0 38 0 0 + 28 SeekRowid 1 1 38 0 if (r[1]!=cursor 1 for table parent.rowid) goto 38 + 29 RowId 1 12 0 0 r[12]=parent.rowid + 30 ColumnRange 1 1 13 2 0 r[13..14]=parent.k..v + 31 Copy 12 15 2 0 r[15]=r[12] + 32 Copy 12 18 2 0 r[18]=r[12] + 33 Copy 5 20 0 0 r[20]=r[5] + 34 Affinity 18 3 0 0 r[18..21] = D, D, D + 35 MakeRecord 18 3 21 0 r[21]=mkrec(r[18..20]) + 36 Insert 1 21 1 parent 8 intkey=r[1] data=r[21] + 37 Goto 0 38 0 0 + 38 Goto 0 39 0 0 + 39 Halt 0 0 0 0 + 40 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 41 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-upsert-parent-key-with-cascade-child.snap b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-upsert-parent-key-with-cascade-child.snap new file mode 100644 index 00000000..f1603a9b --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/foreign_keys__fk-upsert-parent-key-with-cascade-child.snap @@ -0,0 +1,101 @@ +--- +source: foreign_keys.sqltest +expression: INSERT INTO parent(id, k, v) VALUES (1, 10, 2) ON CONFLICT(id) DO UPDATE SET k = excluded.k; +info: + statement_type: INSERT + tables: + - conflict + - parent + setup_blocks: + - fk_bytecode_cascade_child + database: ':memory:' +--- +QUERY PLAN + + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 57 0 0 Start at 57 + 1 OpenWrite 1 2 0 0 root=2; iDb=0 + 2 Integer 1 2 0 0 r[2]=1 + 3 SoftNull 3 0 0 0 + 4 Integer 10 4 0 0 r[4]=10 + 5 Integer 2 5 0 0 r[5]=2 + 6 OpenWrite 0 3 0 0 root=3; iDb=0 + 7 Affinity 3 3 0 0 r[3..6] = D, D, D + 8 NotNull 2 10 0 0 r[2]!=NULL -> goto 10 + 9 Goto 0 12 0 0 + 10 MustBeInt 2 0 0 0 + 11 Goto 0 13 0 0 + 12 NewRowid 1 2 0 0 r[2]=rowid + 13 NotExists 1 16 2 0 + 14 Copy 2 1 0 0 r[1]=r[2] + 15 Goto 0 28 0 0 + 16 Copy 4 7 0 0 r[7]=r[4] + 17 Copy 2 8 0 0 r[8]=r[2] + 18 Affinity 7 1 0 0 r[7..8] = D + 19 NoConflict 0 21 7 1 0 key=r[7..7] + 20 Halt 2067 0 0 parent.k 0 + 21 MakeRecord 3 3 6 0 r[6]=mkrec(r[3..5]) + 22 Copy 4 9 0 0 r[9]=r[4] + 23 Copy 2 10 0 0 r[10]=r[2] + 24 MakeRecord 9 2 11 0 r[11]=mkrec(r[9..10]); for sqlite_autoindex_parent_1 + 25 IdxInsert 0 11 9 2 key=r[11] + 26 Insert 1 6 2 parent 0 intkey=r[2] data=r[6] + 27 Goto 0 55 0 0 + 28 SeekRowid 1 1 55 0 if (r[1]!=cursor 1 for table parent.rowid) goto 55 + 29 RowId 1 12 0 0 r[12]=parent.rowid + 30 ColumnRange 1 1 13 2 0 r[13..14]=parent.k..v + 31 Copy 12 15 2 0 r[15]=r[12] + 32 Copy 12 18 2 0 r[18]=r[12] + 33 Copy 4 19 0 0 r[19]=r[4] + 34 Affinity 18 3 0 0 r[18..21] = D, D, D + 35 Copy 19 21 0 0 r[21]=r[19] + 36 Copy 1 22 0 0 r[22]=r[1] + 37 MakeRecord 21 2 23 0 r[23]=mkrec(r[21..22]); for sqlite_autoindex_parent_1 + 38 Affinity 21 1 0 0 r[21..22] = D + 39 NoConflict 0 43 21 1 0 key=r[21..21] + 40 IdxRowId 0 24 0 0 r[24]=cursor 0.rowid + 41 Eq 1 24 43 0 if r[1]==r[24] goto 43 + 42 Halt 1555 0 0 parent.k 0 + 43 Copy 16 25 0 0 r[25]=r[16] + 44 Copy 1 26 0 0 r[26]=r[1] + 45 IdxDelete 0 25 2 0 + 46 IdxInsert 0 23 21 2 key=r[23] + 47 MakeRecord 18 3 27 0 r[27]=mkrec(r[18..20]) + 48 Insert 1 27 1 parent 8 intkey=r[1] data=r[27] + 49 Copy 13 28 0 0 r[28]=r[13] + 50 Copy 19 29 0 0 r[29]=r[19] + 51 Eq 28 29 54 0 if r[28]==r[29] goto 54 + 52 Goto 0 53 0 0 + 53 Program 28 54 2 fk cascade update 0 subprogram=fk cascade update + 54 Goto 0 55 0 0 + 55 Goto 0 56 0 0 + 56 Halt 0 0 0 0 + 57 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 58 Goto 0 1 0 0 + 0 Init 0 1 0 0 Start at 1 + 1 OpenWrite 0 4 0 0 root=4; iDb=0 + 2 Rewind 0 24 0 0 Rewind table child + 3 Column 0 1 2 0 r[2]=child.pk + 4 Variable 1 3 0 0 r[3]=parameter(1) + 5 Ne 2 3 23 Binary 0 if r[2]!=r[3] goto 23 + 6 RowId 0 4 0 0 r[4]=child.rowid + 7 IsNull 4 24 0 0 if (r[4]==NULL) goto 24 + 8 Null 0 5 0 0 r[5]=NULL + 9 Variable 2 6 0 0 r[6]=parameter(2) + 10 Affinity 5 2 0 0 r[5..7] = D, D + 11 Copy 4 5 0 0 r[5]=r[4] + 12 IsNull 6 21 0 0 if (r[6]==NULL) goto 21 + 13 OpenRead 1 3 0 k(2,B) 0 =sqlite_autoindex_parent_1, root=3, iDb=0 + 14 Copy 6 7 0 0 r[7]=r[6] + 15 Affinity 7 1 0 0 r[7..8] = D + 16 Found 1 19 7 0 if found goto 19 + 17 FkCounter 1 0 0 0 + 18 Goto 0 19 0 0 + 19 Close 1 0 0 0 + 20 Goto 0 21 0 0 + 21 MakeRecord 5 2 8 0 r[8]=mkrec(r[5..6]) + 22 Insert 0 8 4 child 8 intkey=r[4] data=r[8] + 23 Next 0 3 0 1 + 24 Halt 0 0 0 fk action 0 diff --git a/conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-covering-index-scan-tags-next-p5.snap b/conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-covering-index-scan-tags-next-p5.snap new file mode 100644 index 00000000..24766c48 --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-covering-index-scan-tags-next-p5.snap @@ -0,0 +1,25 @@ +--- +source: fullscan_steps.sqltest +expression: SELECT a FROM t ORDER BY a; +info: + statement_type: SELECT + tables: + - t + setup_blocks: + - fullscan_schema + database: ':memory:' +--- +QUERY PLAN +`--SCAN t USING COVERING INDEX ta + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 7 0 0 Start at 7 + 1 OpenRead 0 3 0 k(2,B) 0 index=ta, root=3, iDb=0 + 2 Rewind 0 6 0 0 Rewind index ta + 3 Column 0 0 1 0 r[1]=ta.a + 4 ResultRow 1 1 0 0 output=r[1] + 5 Next 0 3 0 1 + 6 Halt 0 0 0 0 + 7 Transaction 0 1 2 0 iDb=0 tx_mode=Read + 8 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-reverse-table-scan-tags-prev-p5.snap b/conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-reverse-table-scan-tags-prev-p5.snap new file mode 100644 index 00000000..9c454759 --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-reverse-table-scan-tags-prev-p5.snap @@ -0,0 +1,25 @@ +--- +source: fullscan_steps.sqltest +expression: SELECT * FROM t ORDER BY rowid DESC; +info: + statement_type: SELECT + tables: + - t + setup_blocks: + - fullscan_schema + database: ':memory:' +--- +QUERY PLAN +`--SCAN t + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 7 0 0 Start at 7 + 1 OpenRead 0 2 0 k(3,B,B) 0 table=t, root=2, iDb=0 + 2 Last 0 6 0 0 + 3 ColumnRange 0 0 1 2 0 r[1..2]=t.a..b + 4 ResultRow 1 2 0 0 output=r[1..2] + 5 Prev 0 3 0 1 + 6 Halt 0 0 0 0 + 7 Transaction 0 1 2 0 iDb=0 tx_mode=Read + 8 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-table-scan-tags-next-p5.snap b/conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-table-scan-tags-next-p5.snap new file mode 100644 index 00000000..78c71268 --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/fullscan_steps__fullscan-table-scan-tags-next-p5.snap @@ -0,0 +1,25 @@ +--- +source: fullscan_steps.sqltest +expression: SELECT * FROM t; +info: + statement_type: SELECT + tables: + - t + setup_blocks: + - fullscan_schema + database: ':memory:' +--- +QUERY PLAN +`--SCAN t + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 7 0 0 Start at 7 + 1 OpenRead 0 2 0 k(3,B,B) 0 table=t, root=2, iDb=0 + 2 Rewind 0 6 0 0 Rewind table t + 3 ColumnRange 0 0 1 2 0 r[1..2]=t.a..b + 4 ResultRow 1 2 0 0 output=r[1..2] + 5 Next 0 3 0 1 + 6 Halt 0 0 0 0 + 7 Transaction 0 1 2 0 iDb=0 tx_mode=Read + 8 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/fullscan_steps__indexed-range-search-not-tagged-fullscan.snap b/conformance/sqlite-sqltests/snapshots/fullscan_steps__indexed-range-search-not-tagged-fullscan.snap new file mode 100644 index 00000000..2d3e7c56 --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/fullscan_steps__indexed-range-search-not-tagged-fullscan.snap @@ -0,0 +1,29 @@ +--- +source: fullscan_steps.sqltest +expression: SELECT * FROM t WHERE a > 5; +info: + statement_type: SELECT + tables: + - t + setup_blocks: + - fullscan_schema + database: ':memory:' +--- +QUERY PLAN +`--SEARCH t USING INDEX ta (a>?) + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 11 0 0 Start at 11 + 1 OpenRead 0 2 0 k(3,B,B) 0 table=t, root=2, iDb=0 + 2 OpenRead 1 3 0 k(2,B) 0 index=ta, root=3, iDb=0 + 3 Integer 5 3 0 0 r[3]=5 + 4 SeekGT 1 10 3 0 key=[3..3] + 5 DeferredSeek 1 0 0 0 + 6 Column 1 0 1 0 r[1]=ta.a + 7 Column 0 1 2 0 r[2]=t.b + 8 ResultRow 1 2 0 0 output=r[1..2] + 9 Next 1 5 0 0 + 10 Halt 0 0 0 0 + 11 Transaction 0 1 2 0 iDb=0 tx_mode=Read + 12 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/gencol__gencol_drop_virtual_column_no_row_rewrite.snap b/conformance/sqlite-sqltests/snapshots/gencol__gencol_drop_virtual_column_no_row_rewrite.snap index 5584ef07..2a74bc7c 100644 --- a/conformance/sqlite-sqltests/snapshots/gencol__gencol_drop_virtual_column_no_row_rewrite.snap +++ b/conformance/sqlite-sqltests/snapshots/gencol__gencol_drop_virtual_column_no_row_rewrite.snap @@ -13,29 +13,26 @@ QUERY PLAN `--SCAN sqlite_schema BYTECODE -addr opcode p1 p2 p3 p4 p5 comment - 0 Init 0 20 0 0 Start at 20 - 1 OpenWrite 0 1 0 0 root=1; iDb=0 - 2 Rewind 0 17 0 0 Rewind table sqlite_schema - 3 Column 0 1 2 0 r[2]=sqlite_schema.name - 4 Ne 2 3 16 NoCase 0 if r[2]!=r[3] goto 16 - 5 Column 0 0 5 0 r[5]=sqlite_schema.type - 6 Ne 5 6 16 Binary 0 if r[5]!=r[6] goto 16 - 7 RowId 0 7 0 0 r[7]=sqlite_schema.rowid - 8 IsNull 7 17 0 0 if (r[7]==NULL) goto 17 - 9 Column 0 0 8 0 r[8]=sqlite_schema.type - 10 Column 0 1 9 0 r[9]=sqlite_schema.name - 11 Column 0 2 10 0 r[10]=sqlite_schema.tbl_name - 12 Column 0 3 11 0 r[11]=sqlite_schema.rootpage - 13 Affinity 8 5 0 0 r[8..13] = B, B, B, D, B - 14 MakeRecord 8 5 13 0 r[13]=mkrec(r[8..12]) - 15 Insert 0 13 7 sqlite_schema 8 intkey=r[7] data=r[13] - 16 Next 0 3 0 0 - 17 SetCookie 0 1 3 0 - 18 DropColumn 0 0 0 0 drop_column(t, 1) - 19 Halt 0 0 0 0 - 20 Transaction 0 2 2 0 iDb=0 tx_mode=Write - 21 String8 0 3 0 t 0 r[3]='t' - 22 String8 0 6 0 table 0 r[6]='table' - 23 String8 0 12 0 CREATE TABLE t (a) 0 r[12]='CREATE TABLE t (a)' - 24 Goto 0 1 0 0 +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 17 0 0 Start at 17 + 1 OpenWrite 0 1 0 0 root=1; iDb=0 + 2 Rewind 0 14 0 0 Rewind table sqlite_schema + 3 Column 0 1 2 0 r[2]=sqlite_schema.name + 4 Ne 2 3 13 NoCase 0 if r[2]!=r[3] goto 13 + 5 Column 0 0 5 0 r[5]=sqlite_schema.type + 6 Ne 5 6 13 Binary 0 if r[5]!=r[6] goto 13 + 7 RowId 0 7 0 0 r[7]=sqlite_schema.rowid + 8 IsNull 7 14 0 0 if (r[7]==NULL) goto 14 + 9 ColumnRange 0 0 8 4 0 r[8..11]=sqlite_schema.type..rootpage + 10 Affinity 8 5 0 0 r[8..13] = B, B, B, D, B + 11 MakeRecord 8 5 13 0 r[13]=mkrec(r[8..12]) + 12 Insert 0 13 7 sqlite_schema 8 intkey=r[7] data=r[13] + 13 Next 0 3 0 1 + 14 SetCookie 0 1 3 0 + 15 DropColumn 0 0 0 0 drop_column(t, 1) + 16 Halt 0 0 0 0 + 17 Transaction 0 2 2 0 iDb=0 tx_mode=Write + 18 String8 0 3 0 t 0 r[3]='t' + 19 String8 0 6 0 table 0 r[6]='table' + 20 String8 0 12 0 CREATE TABLE t (a) 0 r[12]='CREATE TABLE t (a)' + 21 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/gencol__strict_update_reuses_generated_columns_for_checks.snap b/conformance/sqlite-sqltests/snapshots/gencol__strict_update_reuses_generated_columns_for_checks.snap new file mode 100644 index 00000000..13eec596 --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/gencol__strict_update_reuses_generated_columns_for_checks.snap @@ -0,0 +1,35 @@ +--- +source: gencol.sqltest +expression: UPDATE t SET c = '42'; +info: + statement_type: UPDATE + tables: + - t + setup_blocks: + - strict_update_generated_columns + database: ':memory:' +--- +QUERY PLAN +`--SCAN t + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 15 0 0 Start at 15 + 1 OpenWrite 0 2 0 0 root=2; iDb=0 + 2 Rewind 0 14 0 0 Rewind table t + 3 RowId 0 1 0 0 r[1]=t.rowid + 4 IsNull 1 14 0 0 if (r[1]==NULL) goto 14 + 5 Column 0 0 2 0 r[2]=t.a + 6 TypeCheck 2 2 0 0 + 7 Copy 2 5 0 0 r[5]=r[2] + 8 Add 5 6 4 0 r[4]=r[5]+r[6] + 9 Affinity 4 1 0 0 r[4..5] = D + 10 TypeCheck 2 3 1 0 + 11 MakeRecord 2 2 7 0 r[7]=mkrec(r[2..3]) + 12 Insert 0 7 1 t 8 intkey=r[1] data=r[7] + 13 Next 0 3 0 1 + 14 Halt 0 0 0 0 + 15 Transaction 0 2 1 0 iDb=0 tx_mode=Write + 16 String8 0 3 0 42 0 r[3]='42' + 17 Integer 1 6 0 0 r[6]=1 + 18 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/graph_traversal_text_pk__cte-neighbor-traversal-text-pk.snap b/conformance/sqlite-sqltests/snapshots/graph_traversal_text_pk__cte-neighbor-traversal-text-pk.snap index e097285e..53f9844a 100644 --- a/conformance/sqlite-sqltests/snapshots/graph_traversal_text_pk__cte-neighbor-traversal-text-pk.snap +++ b/conformance/sqlite-sqltests/snapshots/graph_traversal_text_pk__cte-neighbor-traversal-text-pk.snap @@ -27,6 +27,6 @@ info: --- QUERY PLAN |--SCAN seed_articles AS s -| `--SEARCH a USING INDEX sqlite_autoindex_article_1 (id=?) +| `--SEARCH a USING COVERING INDEX sqlite_autoindex_article_1 (id=?) |--MULTI-INDEX OR l (idx_link_source, idx_link_target) `--MULTI-INDEX OR a (sqlite_autoindex_article_1, sqlite_autoindex_article_1) diff --git a/conformance/sqlite-sqltests/snapshots/multi_index_intersection__composite-index-preferred-over-intersection-eqp.snap b/conformance/sqlite-sqltests/snapshots/multi_index_intersection__composite-index-preferred-over-intersection-eqp.snap index d854ddd1..f0a68a44 100644 --- a/conformance/sqlite-sqltests/snapshots/multi_index_intersection__composite-index-preferred-over-intersection-eqp.snap +++ b/conformance/sqlite-sqltests/snapshots/multi_index_intersection__composite-index-preferred-over-intersection-eqp.snap @@ -10,4 +10,4 @@ info: database: ':memory:' --- QUERY PLAN -`--SEARCH t USING INDEX idx_ab (a=? AND b=?) +`--SEARCH t USING COVERING INDEX idx_ab (a=? AND b=?) diff --git a/conformance/sqlite-sqltests/snapshots/multi_index_intersection__multi_index_and.snap b/conformance/sqlite-sqltests/snapshots/multi_index_intersection__multi_index_and.snap new file mode 100644 index 00000000..aafc084b --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/multi_index_intersection__multi_index_and.snap @@ -0,0 +1,44 @@ +--- +source: multi_index_intersection.sqltest +expression: SELECT * FROM t WHERE a = 1 AND b = 2; +info: + statement_type: SELECT + tables: + - t + setup_blocks: + - multi_index_and_setup + database: ':memory:' +--- +QUERY PLAN +`--MULTI-INDEX AND t (ia, ib) + +BYTECODE +addr opcode p1 p2 p3 p4 p5 comment + 0 Init 0 26 0 0 Start at 26 + 1 OpenRead 0 2 0 k(12,B,B,B,B,B,B,B,B,B,B,B) 0 table=t, root=2, iDb=0 + 2 OpenRead 1 3 0 k(2,B) 0 index=ia, root=3, iDb=0 + 3 OpenRead 2 4 0 k(2,B) 0 index=ib, root=4, iDb=0 + 4 Null 0 13 0 0 r[13]=NULL + 5 Null 0 14 0 0 r[14]=NULL + 6 Integer 1 15 0 0 r[15]=1 + 7 SeekGE 1 12 15 0 key=[15..15] + 8 IdxGT 1 12 15 0 key=[15..15] + 9 IdxRowId 1 12 0 0 r[12]=cursor 1 for index ia.rowid + 10 RowSetAdd 13 12 0 0 + 11 Next 1 8 0 0 + 12 Integer 2 16 0 0 r[16]=2 + 13 SeekGE 2 20 16 0 key=[16..16] + 14 IdxGT 2 20 16 0 key=[16..16] + 15 IdxRowId 2 12 0 0 r[12]=cursor 2 for index ib.rowid + 16 RowSetTest 13 18 12 -1 0 + 17 Goto 0 19 0 0 + 18 RowSetAdd 14 12 0 0 + 19 Next 2 14 0 0 + 20 RowSetRead 14 25 12 0 + 21 SeekRowid 0 12 22 0 if (r[12]!=cursor 0 for table t.rowid) goto 22 + 22 ColumnRange 0 0 1 11 0 r[1..11]=t.a..c8 + 23 ResultRow 1 11 0 0 output=r[1..11] + 24 Goto 0 20 0 0 + 25 Halt 0 0 0 0 + 26 Transaction 0 1 4 0 iDb=0 tx_mode=Read + 27 Goto 0 1 0 0 diff --git a/conformance/sqlite-sqltests/snapshots/multi_index_intersection__multi_index_and_eqp.snap b/conformance/sqlite-sqltests/snapshots/multi_index_intersection__multi_index_and_eqp.snap new file mode 100644 index 00000000..2aa72555 --- /dev/null +++ b/conformance/sqlite-sqltests/snapshots/multi_index_intersection__multi_index_and_eqp.snap @@ -0,0 +1,13 @@ +--- +source: multi_index_intersection.sqltest +expression: SELECT * FROM t WHERE a = 1 AND b = 2; +info: + statement_type: SELECT + tables: + - t + setup_blocks: + - multi_index_and_setup + database: ':memory:' +--- +QUERY PLAN +`--MULTI-INDEX AND t (ia, ib) diff --git a/conformance/sqlite-sqltests/snapshots/multi_index_or_compound__compound-or-correlated-subquery-eqp.snap b/conformance/sqlite-sqltests/snapshots/multi_index_or_compound__compound-or-correlated-subquery-eqp.snap index 5ae060f0..711d0dfa 100644 --- a/conformance/sqlite-sqltests/snapshots/multi_index_or_compound__compound-or-correlated-subquery-eqp.snap +++ b/conformance/sqlite-sqltests/snapshots/multi_index_or_compound__compound-or-correlated-subquery-eqp.snap @@ -14,6 +14,6 @@ info: database: ':memory:' --- QUERY PLAN -|--SCAN t +|--SEARCH t USING INDEX idx_t_a (a=?) |--SCAN u `--SCAN u diff --git a/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-indexed-by-left-join-null-rejecting-where-can-prove-after-rewrite-eqp.snap b/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-indexed-by-left-join-null-rejecting-where-can-prove-after-rewrite-eqp.snap index 7e3ecec9..434a69dc 100644 --- a/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-indexed-by-left-join-null-rejecting-where-can-prove-after-rewrite-eqp.snap +++ b/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-indexed-by-left-join-null-rejecting-where-can-prove-after-rewrite-eqp.snap @@ -17,4 +17,4 @@ info: --- QUERY PLAN |--SCAN left_items -`--SEARCH right_items USING INDEX right_marker_five_idx (id=?) +`--SEARCH right_items USING COVERING INDEX right_marker_five_idx (id=?) diff --git a/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-left-join-on-predicate-can-prove-index-eqp.snap b/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-left-join-on-predicate-can-prove-index-eqp.snap index 39a53373..80b6c7ed 100644 --- a/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-left-join-on-predicate-can-prove-index-eqp.snap +++ b/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-left-join-on-predicate-can-prove-index-eqp.snap @@ -17,5 +17,5 @@ info: --- QUERY PLAN |--SCAN left_items -|--SCAN right_items USING INDEX right_marker_null_idx +|--SCAN right_items USING INDEX right_marker_null_idx LEFT-JOIN `--USE SORTER FOR ORDER BY diff --git a/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-left-join-where-predicate-cannot-prove-index-eqp.snap b/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-left-join-where-predicate-cannot-prove-index-eqp.snap index e2219736..a9031846 100644 --- a/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-left-join-where-predicate-cannot-prove-index-eqp.snap +++ b/conformance/sqlite-sqltests/snapshots/partial_idx__partial-index-left-join-where-predicate-cannot-prove-index-eqp.snap @@ -16,4 +16,4 @@ info: --- QUERY PLAN |--SCAN left_items -`--SCAN right_items +`--SCAN right_items LEFT-JOIN diff --git a/conformance/sqlite-sqltests/sql-error-message-prefix.sqltest b/conformance/sqlite-sqltests/sql-error-message-prefix.sqltest index 06d6d961..1d21e0ef 100644 --- a/conformance/sqlite-sqltests/sql-error-message-prefix.sqltest +++ b/conformance/sqlite-sqltests/sql-error-message-prefix.sqltest @@ -13,3 +13,18 @@ test sql-error-has-no-internal-error-prefix { expect error { ^second argument to nth_value must be a positive integer$ } + +# SQLITE_FULL is reported bare too. Turso used to stamp a "Database is +# full: " prefix on it, so PRAGMA max_page_count overflow read +# "Database is full: database or disk is full". + +@skip-if mvcc "MVCC keeps new rows in memory until checkpoint, so the page limit is not hit at INSERT time" +test database-full-error-has-no-prefix { + PRAGMA page_size = 1024; + PRAGMA max_page_count = 2; + CREATE TABLE f (x); + INSERT INTO f VALUES (zeroblob(4000)); +} +expect error { + ^database or disk is full$ +} diff --git a/conformance/sqlite-sqltests/subquery-from-name-vs-outer-alias.sqltest b/conformance/sqlite-sqltests/subquery-from-name-vs-outer-alias.sqltest new file mode 100644 index 00000000..6289f4c1 --- /dev/null +++ b/conformance/sqlite-sqltests/subquery-from-name-vs-outer-alias.sqltest @@ -0,0 +1,116 @@ +@database :memory: + +setup schema { + CREATE TABLE t(a INT); + CREATE TABLE u(a INT); + INSERT INTO t VALUES(1),(2),(3); + INSERT INTO u VALUES(2); +} + +@setup schema +@cross-check-integrity +test delete-target-alias-does-not-shadow-table-in-subquery { + DELETE FROM t AS u WHERE a IN (SELECT a FROM u); + SELECT a FROM t ORDER BY a; +} +expect { + 1 + 3 +} + +@setup schema +test select-outer-alias-does-not-shadow-table-in-subquery { + SELECT (SELECT sum(a) FROM u) FROM t AS u ORDER BY 1 LIMIT 1; +} +expect { + 2 +} + +@setup schema +test unshadowed-subquery-from-name-still-reads-the-table { + SELECT (SELECT sum(a) FROM u) FROM u AS x; +} +expect { + 2 +} + +setup update_schema { + CREATE TABLE w(a INT); + INSERT INTO w VALUES(5),(6); +} + +@setup update_schema +@cross-check-integrity +test update-target-does-not-shadow-cte-in-subquery { + WITH w(a) AS (SELECT 9) UPDATE w SET a = (SELECT a FROM w); + SELECT a FROM w ORDER BY a; +} +expect { + 9 + 9 +} + +@setup schema +test cte-alias-does-not-shadow-table-in-subquery { + WITH q AS (SELECT 1 AS a) SELECT (SELECT sum(a) FROM u) FROM q AS u; +} +expect { + 2 +} + +@setup schema +test from-subquery-alias-does-not-shadow-table-in-subquery { + SELECT (SELECT sum(a) FROM u) FROM (SELECT 1 AS a) AS u; +} +expect { + 2 +} + +@setup schema +test outer-alias-naming-no-table-is-not-resolvable-in-subquery { + SELECT (SELECT count(*) FROM y) FROM t AS y; +} +expect error { + no such table: y +} + +@setup schema +test from-subquery-alias-naming-no-table-is-not-resolvable-in-subquery { + SELECT (SELECT count(*) FROM x) FROM (SELECT 1 AS a) AS x; +} +expect error { + no such table: x +} + +@setup schema +test cte-name-is-still-resolvable-in-a-subquery-from-clause { + WITH q AS (SELECT 42 AS a) SELECT (SELECT sum(a) FROM q) FROM q; +} +expect { + 42 +} + +@setup schema +test cte-name-is-resolvable-in-a-subquery-when-its-use-is-aliased { + WITH q AS (SELECT 42 AS a) SELECT (SELECT sum(a) FROM q) FROM q AS z; +} +expect { + 42 +} + +@setup schema +test recursive-cte-alias-does-not-shadow-table-in-subquery { + WITH RECURSIVE seq(x) AS ( + VALUES(1) + UNION ALL + SELECT u.x + (SELECT sum(a) FROM u) + FROM seq AS u + WHERE u.x < 5 + ) + SELECT x FROM seq ORDER BY x; +} +expect { + 1 + 3 + 5 +} diff --git a/conformance/sqlite-sqltests/subquery/memory.sqltest b/conformance/sqlite-sqltests/subquery/memory.sqltest index f9c9c2f2..7b0b61d1 100644 --- a/conformance/sqlite-sqltests/subquery/memory.sqltest +++ b/conformance/sqlite-sqltests/subquery/memory.sqltest @@ -215,7 +215,7 @@ test subquery-not-exists-all-match { select * from blocked_emails b where b.email = u.email ); } -expect { +expect unordered { 1|alice@test.com 2|bob@test.com } diff --git a/conformance/sqlite-sqltests/turso/alter_column.sqltest b/conformance/sqlite-sqltests/turso/alter_column.sqltest index 9423e427..32c1aa55 100644 --- a/conformance/sqlite-sqltests/turso/alter_column.sqltest +++ b/conformance/sqlite-sqltests/turso/alter_column.sqltest @@ -1,5 +1,6 @@ @database :memory: +@skip-if mvcc "rebuilding affected indexes is not supported in MVCC mode" test alter-column-rename-and-type { CREATE TABLE t (a INTEGER); CREATE INDEX i ON t (a); @@ -611,6 +612,161 @@ expect { text|abc } +# Regression test for https://github.com/tursodatabase/turso/issues/6876. +test alter-column-rewrites-indexed-affinity-change { + PRAGMA journal_mode=WAL; + CREATE TABLE t(x NUMERIC); + CREATE INDEX idx_x ON t(x); + INSERT INTO t VALUES(10),(2),(30); + ALTER TABLE t ALTER COLUMN x TO y TEXT; + SELECT y, typeof(y) FROM t INDEXED BY idx_x WHERE y >= '0' ORDER BY y; + PRAGMA integrity_check; +} +expect { + wal + 10|text + 2|text + 30|text + ok +} + +test alter-column-rewrites-expression-index-affinity-change { + PRAGMA journal_mode=WAL; + CREATE TABLE t(x NUMERIC, z INTEGER); + CREATE INDEX idx_expr ON t(typeof(x), x); + INSERT INTO t VALUES(10,1),(2,2),(30,3); + ALTER TABLE t ALTER COLUMN x TO y TEXT; + SELECT y FROM t INDEXED BY idx_expr WHERE typeof(y) = 'text' ORDER BY y; + PRAGMA integrity_check; +} +expect { + wal + 10 + 2 + 30 + ok +} + +test alter-column-rewrites-partial-index-affinity-change { + PRAGMA journal_mode=WAL; + CREATE TABLE t(x NUMERIC, z INTEGER); + CREATE INDEX idx_partial ON t(z) WHERE typeof(x) = 'text'; + INSERT INTO t VALUES(10,1),(2,2),(30,3); + ALTER TABLE t ALTER COLUMN x TO y TEXT; + SELECT z FROM t INDEXED BY idx_partial WHERE typeof(y) = 'text' ORDER BY z; + PRAGMA integrity_check; +} +expect { + wal + 1 + 2 + 3 + ok +} + +test alter-column-rewrites-generated-index-affinity-change { + PRAGMA journal_mode=WAL; + CREATE TABLE t(x NUMERIC, g TEXT GENERATED ALWAYS AS (typeof(x) || ':' || x) VIRTUAL); + CREATE INDEX idx_g ON t(g); + INSERT INTO t(x) VALUES(10),(2),(30); + ALTER TABLE t ALTER COLUMN x TO y TEXT; + SELECT g FROM t INDEXED BY idx_g WHERE g >= 'text:' ORDER BY g; + PRAGMA integrity_check; +} +expect { + wal + text:10 + text:2 + text:30 + ok +} + +test alter-column-rewrites-indexed-virtual-to-regular { + PRAGMA journal_mode=WAL; + CREATE TABLE t(id TEXT, g TEXT GENERATED ALWAYS AS ('old:' || id) VIRTUAL); + CREATE INDEX idx_g ON t(g); + INSERT INTO t(id) VALUES('a'),('b'); + ALTER TABLE t ALTER COLUMN g TO h TEXT; + SELECT id, h FROM t INDEXED BY idx_g WHERE h IS NULL ORDER BY id; + PRAGMA integrity_check; +} +expect { + wal + a| + b| + ok +} + +test alter-column-rewrites-indexed-regular-to-virtual { + PRAGMA journal_mode=WAL; + CREATE TABLE t(a TEXT, b TEXT, c TEXT); + CREATE INDEX idx_b ON t(b); + INSERT INTO t VALUES('a','stale:2','2'),('a','stale:1','1'); + ALTER TABLE t ALTER COLUMN b TO d TEXT GENERATED ALWAYS AS (a || ':' || c) VIRTUAL; + SELECT d FROM t INDEXED BY idx_b WHERE d >= 'a:' ORDER BY d; + PRAGMA integrity_check; +} +expect { + wal + a:1 + a:2 + ok +} + +test alter-column-rewrites-indexed-virtual-expression-change { + PRAGMA journal_mode=WAL; + CREATE TABLE t(a TEXT, g TEXT GENERATED ALWAYS AS ('old:' || a) VIRTUAL); + CREATE INDEX idx_g ON t(g); + INSERT INTO t(a) VALUES('a'),('b'); + ALTER TABLE t ALTER COLUMN g TO h TEXT GENERATED ALWAYS AS ('new:' || a) VIRTUAL; + SELECT h FROM t INDEXED BY idx_g WHERE h >= 'new:' ORDER BY h; + PRAGMA integrity_check; +} +expect { + wal + new:a + new:b + ok +} + +test alter-column-rewrites-partial-index-virtual-expression-change { + PRAGMA journal_mode=WAL; + CREATE TABLE t(a TEXT, g TEXT GENERATED ALWAYS AS ('old:' || a) VIRTUAL); + CREATE INDEX idx_partial ON t(a) WHERE g LIKE 'new:%'; + INSERT INTO t(a) VALUES('b'),('a'); + ALTER TABLE t ALTER COLUMN g TO h TEXT GENERATED ALWAYS AS ('new:' || a) VIRTUAL; + SELECT a FROM t INDEXED BY idx_partial WHERE h LIKE 'new:%' ORDER BY a; + PRAGMA integrity_check; +} +expect { + wal + a + b + ok +} + +test alter-column-rejects-index-rewrite-mvcc { + PRAGMA journal_mode = 'mvcc'; + CREATE TABLE t(x NUMERIC); + CREATE INDEX idx_x ON t(x); + INSERT INTO t VALUES(10),(2),(30); + ALTER TABLE t ALTER COLUMN x TO y TEXT; +} +expect error { + Parse error: cannot ALTER COLUMN "x": rebuilding affected indexes is not supported in MVCC mode +} + +test alter-column-rejects-index-only-rewrite-mvcc { + PRAGMA journal_mode = 'mvcc'; + CREATE TABLE t(a TEXT, g TEXT GENERATED ALWAYS AS ('old:' || a) VIRTUAL); + CREATE INDEX idx_g ON t(g); + INSERT INTO t(a) VALUES('a'),('b'); + ALTER TABLE t ALTER COLUMN g TO h TEXT GENERATED ALWAYS AS ('new:' || a) VIRTUAL; +} +expect error { + Parse error: cannot ALTER COLUMN "g": rebuilding affected indexes is not supported in MVCC mode +} + test alter-column-text-to-integer-coerces-existing-rows { CREATE TABLE t (x TEXT); INSERT INTO t VALUES ('42'), ('abc'); @@ -717,3 +873,17 @@ expect { a2|a2c2|c2 ok } + +@cross-check-integrity +test alter-column-strict-any-gencol-affinity { + CREATE TABLE t(id INTEGER PRIMARY KEY, a TEXT, c ANY) STRICT; + ALTER TABLE t ALTER COLUMN c TO g ANY AS (a); + INSERT INTO t(id,a) VALUES(1,'42'); + SELECT typeof(g) FROM t; + CREATE INDEX t_g ON t(g); + PRAGMA integrity_check; +} +expect { + text + ok +} diff --git a/conformance/sqlite-sqltests/turso/explain-query-plan-json.sqltest b/conformance/sqlite-sqltests/turso/explain-query-plan-json.sqltest index 243c8368..f2e31621 100644 --- a/conformance/sqlite-sqltests/turso/explain-query-plan-json.sqltest +++ b/conformance/sqlite-sqltests/turso/explain-query-plan-json.sqltest @@ -17,7 +17,7 @@ test search-nodes-name-the-index-the-constraints-and-the-join { ORDER BY o.amount; } expect { - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, o.amount\n FROM users u\n LEFT JOIN orders o ON o.user_id = u.id\n WHERE u.age > 21\n ORDER BY o.amount;","result_columns":["name","amount"],"nodes":[{"id":1,"parent":null,"detail":"SEARCH u USING INDEX idx_users_age (age>?)","op":{"type":"search","table":"users","alias":"u","search_kind":"seek","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"constraints":["age>?"]}},{"id":2,"parent":null,"detail":"SEARCH o USING INDEX idx_orders_user (user_id=?) LEFT-JOIN","op":{"type":"search","table":"orders","alias":"o","join":"left","search_kind":"seek","index":{"name":"idx_orders_user","covering":false,"ephemeral":false},"constraints":["user_id=?"]}},{"id":28,"parent":null,"detail":"USE SORTER FOR ORDER BY","op":{"type":"order_by","method":"sorter"}}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, o.amount\n FROM users u\n LEFT JOIN orders o ON o.user_id = u.id\n WHERE u.age > 21\n ORDER BY o.amount;","result_columns":["name","amount"],"nodes":[{"id":1,"parent":null,"detail":"SEARCH u USING INDEX idx_users_age (age>?)","op":{"type":"search","table":"users","alias":"u","search_kind":"seek","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"constraints":["age>?"],"estimate":{"input_rows":1,"rows_per_input":400000,"output_rows":400000,"access_cost":13203.51,"total_cost":13203.51}}},{"id":2,"parent":null,"detail":"SEARCH o USING INDEX idx_orders_user (user_id=?) LEFT-JOIN","op":{"type":"search","table":"orders","alias":"o","join":"left","search_kind":"seek","index":{"name":"idx_orders_user","covering":false,"ephemeral":false},"constraints":["user_id=?"],"estimate":{"input_rows":400000,"rows_per_input":1000,"output_rows":400000000,"access_cost":11150678.633333333,"total_cost":11163882.143333333}}},{"id":28,"parent":null,"detail":"USE SORTER FOR ORDER BY","op":{"type":"order_by","method":"sorter"}}]} } @setup schema @@ -28,7 +28,7 @@ test exists-is-reported-as-a-semi-join { WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id); } expect { - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users u\n WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id);","result_columns":["id","name","age","city_id"],"nodes":[{"id":1,"parent":null,"detail":"SCAN users AS u","op":{"type":"scan","table":"users","alias":"u","source":"table"}},{"id":2,"parent":null,"detail":"SEARCH o USING COVERING INDEX idx_orders_user (user_id=?)","op":{"type":"search","table":"orders","alias":"o","join":"semi","search_kind":"seek","index":{"name":"idx_orders_user","covering":true,"ephemeral":false},"constraints":["user_id=?"]}}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users u\n WHERE EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id);","result_columns":["id","name","age","city_id"],"nodes":[{"id":1,"parent":null,"detail":"SCAN users AS u","op":{"type":"scan","table":"users","alias":"u","source":"table","estimate":{"input_rows":1,"rows_per_input":1000000,"output_rows":1000000,"access_cost":23000,"total_cost":23000}}},{"id":2,"parent":null,"detail":"SEARCH o USING COVERING INDEX idx_orders_user (user_id=?)","op":{"type":"search","table":"orders","alias":"o","join":"semi","search_kind":"seek","index":{"name":"idx_orders_user","covering":true,"ephemeral":false},"constraints":["user_id=?"],"estimate":{"input_rows":1000000,"rows_per_input":1,"output_rows":1000000,"access_cost":7876678.633333334,"total_cost":7899678.633333334}}}]} } @setup schema @@ -39,7 +39,7 @@ test not-exists-is-reported-as-an-anti-join { WHERE NOT EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id); } expect { - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users u\n WHERE NOT EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id);","result_columns":["id","name","age","city_id"],"nodes":[{"id":1,"parent":null,"detail":"SCAN users AS u","op":{"type":"scan","table":"users","alias":"u","source":"table"}},{"id":2,"parent":null,"detail":"SEARCH o USING COVERING INDEX idx_orders_user (user_id=?)","op":{"type":"search","table":"orders","alias":"o","join":"anti","search_kind":"seek","index":{"name":"idx_orders_user","covering":true,"ephemeral":false},"constraints":["user_id=?"]}}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users u\n WHERE NOT EXISTS (SELECT 1 FROM orders o WHERE o.user_id = u.id);","result_columns":["id","name","age","city_id"],"nodes":[{"id":1,"parent":null,"detail":"SCAN users AS u","op":{"type":"scan","table":"users","alias":"u","source":"table","estimate":{"input_rows":1,"rows_per_input":1000000,"output_rows":1000000,"access_cost":23000,"total_cost":23000}}},{"id":2,"parent":null,"detail":"SEARCH o USING COVERING INDEX idx_orders_user (user_id=?)","op":{"type":"search","table":"orders","alias":"o","join":"anti","search_kind":"seek","index":{"name":"idx_orders_user","covering":true,"ephemeral":false},"constraints":["user_id=?"],"estimate":{"input_rows":1000000,"rows_per_input":1,"output_rows":1000000,"access_cost":7876678.633333334,"total_cost":7899678.633333334}}}]} } @setup schema @@ -54,7 +54,7 @@ test both-readers-of-a-shared-cte-point-at-one-materialization { JOIN spenders s2 ON s2.user_id = u.city_id; } expect { - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH spenders AS (\n SELECT user_id, sum(amount) AS total FROM orders GROUP BY user_id\n )\n SELECT u.name, s1.total\n FROM users u\n JOIN spenders s1 ON s1.user_id = u.id\n JOIN spenders s2 ON s2.user_id = u.city_id;","result_columns":["name","total"],"nodes":[{"id":2,"parent":null,"detail":"SCAN orders USING INDEX idx_orders_user","op":{"type":"scan","table":"orders","source":"table","index":{"name":"idx_orders_user","covering":false,"ephemeral":false}}},{"id":41,"parent":null,"detail":"SCAN users AS u","op":{"type":"scan","table":"users","alias":"u","source":"table"}},{"id":42,"parent":null,"detail":"SEARCH s1 USING INDEX ephemeral_subquery_t3 (user_id=?)","op":{"type":"search","table":"spenders","alias":"s1","join":"inner","subquery":{"execution":"materialized_reuse","cte_id":0},"search_kind":"seek","index":{"name":"ephemeral_subquery_t3","covering":false,"ephemeral":true},"constraints":["user_id=?"]}},{"id":43,"parent":null,"detail":"SEARCH s2 USING INDEX ephemeral_subquery_t5 (user_id=?)","op":{"type":"search","table":"spenders","alias":"s2","join":"inner","subquery":{"execution":"materialized_reuse","cte_id":0},"search_kind":"seek","index":{"name":"ephemeral_subquery_t5","covering":false,"ephemeral":true},"constraints":["user_id=?"]}}],"cte_materializations":[{"cte_id":0,"name":"spenders","nodes":[2]}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH spenders AS (\n SELECT user_id, sum(amount) AS total FROM orders GROUP BY user_id\n )\n SELECT u.name, s1.total\n FROM users u\n JOIN spenders s1 ON s1.user_id = u.id\n JOIN spenders s2 ON s2.user_id = u.city_id;","result_columns":["name","total"],"nodes":[{"id":3,"parent":null,"detail":"SCAN orders USING INDEX idx_orders_user","op":{"type":"scan","table":"orders","source":"table","index":{"name":"idx_orders_user","covering":false,"ephemeral":false},"estimate":{"input_rows":1,"rows_per_input":1000000,"output_rows":1000000,"access_cost":36336.84333333334,"total_cost":36336.84333333334}}},{"id":42,"parent":null,"detail":"SCAN users AS u","op":{"type":"scan","table":"users","alias":"u","source":"table","estimate":{"input_rows":1,"rows_per_input":1000000,"output_rows":1000000,"access_cost":23000,"total_cost":23000}}},{"id":43,"parent":null,"detail":"SEARCH s1 USING INDEX ephemeral_subquery_t3 (user_id=?)","op":{"type":"search","table":"spenders","alias":"s1","join":"inner","subquery":{"execution":"materialized_reuse","cte_id":0},"search_kind":"seek","index":{"name":"ephemeral_subquery_t3","covering":false,"ephemeral":true},"constraints":["user_id=?"],"estimate":{"input_rows":1000000,"rows_per_input":1,"output_rows":1000000,"access_cost":271652.5290265751,"total_cost":294652.5290265751}}},{"id":44,"parent":null,"detail":"SEARCH s2 USING INDEX ephemeral_subquery_t5 (user_id=?)","op":{"type":"search","table":"spenders","alias":"s2","join":"inner","subquery":{"execution":"materialized_reuse","cte_id":0},"search_kind":"seek","index":{"name":"ephemeral_subquery_t5","covering":false,"ephemeral":true},"constraints":["user_id=?"],"estimate":{"input_rows":1000000,"rows_per_input":1,"output_rows":1000000,"access_cost":271652.5290265751,"total_cost":566305.0580531502}}}],"cte_materializations":[{"cte_id":0,"name":"spenders","nodes":[3]}]} } @setup schema @@ -66,7 +66,7 @@ test a-from-clause-subquery-nests-its-body-under-its-own-scan { ORDER BY avg(age); } expect { - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT city_id, avg(age)\n FROM (SELECT * FROM users WHERE age > 18)\n GROUP BY city_id\n ORDER BY avg(age);","result_columns":["city_id","avg(age)"],"nodes":[{"id":1,"parent":null,"detail":"SCAN (subquery-0)","op":{"type":"scan","table":"(subquery-0)","subquery":{"execution":"coroutine"},"source":"subquery"}},{"id":3,"parent":1,"detail":"SEARCH users USING INDEX idx_users_age (age>?)","op":{"type":"search","table":"users","search_kind":"seek","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"constraints":["age>?"]}},{"id":19,"parent":null,"detail":"USE SORTER FOR GROUP BY","op":{"type":"group_by","method":"sorter"}},{"id":61,"parent":null,"detail":"USE SORTER FOR ORDER BY","op":{"type":"order_by","method":"sorter"}}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT city_id, avg(age)\n FROM (SELECT * FROM users WHERE age > 18)\n GROUP BY city_id\n ORDER BY avg(age);","result_columns":["city_id","avg(age)"],"nodes":[{"id":1,"parent":null,"detail":"SCAN (subquery-0)","op":{"type":"scan","table":"(subquery-0)","subquery":{"execution":"coroutine"},"source":"subquery","estimate":{"input_rows":1,"rows_per_input":400000,"output_rows":400000,"access_cost":22403.510000000002,"total_cost":22403.510000000002}}},{"id":3,"parent":1,"detail":"SEARCH users USING INDEX idx_users_age (age>?)","op":{"type":"search","table":"users","search_kind":"seek","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"constraints":["age>?"],"estimate":{"input_rows":1,"rows_per_input":400000,"output_rows":400000,"access_cost":13203.51,"total_cost":13203.51}}},{"id":19,"parent":null,"detail":"USE SORTER FOR GROUP BY","op":{"type":"group_by","method":"sorter"}},{"id":61,"parent":null,"detail":"USE SORTER FOR ORDER BY","op":{"type":"order_by","method":"sorter"}}]} } @setup schema @@ -78,7 +78,7 @@ test a-compound-select-lists-its-arms { ORDER BY name; } expect { - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT name FROM users WHERE age > 30\n UNION\n SELECT name FROM users\n ORDER BY name;","result_columns":["name"],"nodes":[{"id":2,"parent":null,"detail":"COMPOUND QUERY","op":{"type":"compound"}},{"id":4,"parent":2,"detail":"LEFT-MOST SUBQUERY","op":{"type":"compound_arm","op":"left_most","temp_btree":false}},{"id":5,"parent":4,"detail":"SEARCH users USING INDEX idx_users_age (age>?)","op":{"type":"search","table":"users","search_kind":"seek","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"constraints":["age>?"]}},{"id":15,"parent":2,"detail":"UNION USING TEMP B-TREE","op":{"type":"compound_arm","op":"union","temp_btree":true}},{"id":16,"parent":15,"detail":"SCAN users","op":{"type":"scan","table":"users","source":"table"}},{"id":40,"parent":null,"detail":"USE SORTER FOR ORDER BY","op":{"type":"order_by","method":"sorter"}}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT name FROM users WHERE age > 30\n UNION\n SELECT name FROM users\n ORDER BY name;","result_columns":["name"],"nodes":[{"id":2,"parent":null,"detail":"COMPOUND QUERY","op":{"type":"compound"}},{"id":4,"parent":2,"detail":"LEFT-MOST SUBQUERY","op":{"type":"compound_arm","op":"left_most","temp_btree":false}},{"id":5,"parent":4,"detail":"SEARCH users USING INDEX idx_users_age (age>?)","op":{"type":"search","table":"users","search_kind":"seek","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"constraints":["age>?"],"estimate":{"input_rows":1,"rows_per_input":400000,"output_rows":400000,"access_cost":13203.51,"total_cost":13203.51}}},{"id":15,"parent":2,"detail":"UNION USING TEMP B-TREE","op":{"type":"compound_arm","op":"union","temp_btree":true}},{"id":16,"parent":15,"detail":"SCAN users","op":{"type":"scan","table":"users","source":"table","estimate":{"input_rows":1,"rows_per_input":1000000,"output_rows":1000000,"access_cost":23000,"total_cost":23000}}},{"id":40,"parent":null,"detail":"USE SORTER FOR ORDER BY","op":{"type":"order_by","method":"sorter"}}]} } @setup schema @@ -89,7 +89,7 @@ test list-and-scalar-subqueries-report-whether-they-are-correlated { WHERE u.age IN (SELECT age FROM users WHERE city_id = 3); } expect { - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, (SELECT count(*) FROM orders o WHERE o.user_id = u.id)\n FROM users u\n WHERE u.age IN (SELECT age FROM users WHERE city_id = 3);","result_columns":["name","(SELECT count(*) FROM orders o WHERE o.user_id = u.id)"],"nodes":[{"id":1,"parent":null,"detail":"LIST SUBQUERY 1","op":{"type":"list_subquery","subquery_id":1,"correlated":false}},{"id":4,"parent":1,"detail":"SCAN users","op":{"type":"scan","table":"users","source":"table"}},{"id":13,"parent":null,"detail":"SEARCH u USING INDEX idx_users_age (age=?)","op":{"type":"search","table":"users","alias":"u","search_kind":"in_seek","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"constraints":["age=?"]}},{"id":23,"parent":null,"detail":"CORRELATED SCALAR SUBQUERY 2","op":{"type":"scalar_subquery","subquery_id":2,"correlated":true}},{"id":26,"parent":23,"detail":"SEARCH o USING COVERING INDEX idx_orders_user (user_id=?)","op":{"type":"search","table":"orders","alias":"o","search_kind":"seek","index":{"name":"idx_orders_user","covering":true,"ephemeral":false},"constraints":["user_id=?"]}}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, (SELECT count(*) FROM orders o WHERE o.user_id = u.id)\n FROM users u\n WHERE u.age IN (SELECT age FROM users WHERE city_id = 3);","result_columns":["name","(SELECT count(*) FROM orders o WHERE o.user_id = u.id)"],"nodes":[{"id":1,"parent":null,"detail":"LIST SUBQUERY 1","op":{"type":"list_subquery","subquery_id":1,"correlated":false}},{"id":4,"parent":1,"detail":"SCAN users","op":{"type":"scan","table":"users","source":"table","estimate":{"input_rows":1,"rows_per_input":100000,"output_rows":100000,"access_cost":23000,"total_cost":23000}}},{"id":13,"parent":null,"detail":"SEARCH u USING INDEX idx_users_age (age=?)","op":{"type":"search","table":"users","alias":"u","search_kind":"in_seek","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"constraints":["age=?"],"estimate":{"input_rows":1,"rows_per_input":1000,"output_rows":1000,"access_cost":3139.4238618387276,"total_cost":3139.4238618387276}}},{"id":23,"parent":null,"detail":"CORRELATED SCALAR SUBQUERY 2","op":{"type":"scalar_subquery","subquery_id":2,"correlated":true}},{"id":26,"parent":23,"detail":"SEARCH o USING COVERING INDEX idx_orders_user (user_id=?)","op":{"type":"search","table":"orders","alias":"o","search_kind":"seek","index":{"name":"idx_orders_user","covering":true,"ephemeral":false},"constraints":["user_id=?"],"estimate":{"input_rows":1000,"rows_per_input":1000,"output_rows":1000000,"access_cost":7888.633333333333,"total_cost":7888.633333333333}}}]} } @setup schema @@ -100,7 +100,7 @@ test an-or-over-two-indexes-lists-both-branches { WHERE id = 5 OR age = 30; } expect { - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users\n WHERE id = 5 OR age = 30;","result_columns":["id","name","age","city_id"],"nodes":[{"id":1,"parent":null,"detail":"MULTI-INDEX OR users (PRIMARY KEY, idx_users_age)","op":{"type":"multi_index","table":"users","set_op":"or","indexes":["PRIMARY KEY","idx_users_age"]}}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users\n WHERE id = 5 OR age = 30;","result_columns":["id","name","age","city_id"],"nodes":[{"id":1,"parent":null,"detail":"MULTI-INDEX OR users (PRIMARY KEY, idx_users_age)","op":{"type":"multi_index","table":"users","set_op":"or","indexes":["PRIMARY KEY","idx_users_age"],"estimate":{"input_rows":1,"rows_per_input":1000.9990000000579,"output_rows":1000.9990000000579,"access_cost":52.0549740000015,"total_cost":52.0549740000015}}}]} } @setup schema @@ -114,7 +114,7 @@ test a-recursive-cte-reports-its-setup-and-step-phases { SELECT x FROM cnt; } expect { - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH RECURSIVE cnt(x) AS (\n SELECT 1\n UNION ALL\n SELECT x+1 FROM cnt WHERE x < 10\n )\n SELECT x FROM cnt;","result_columns":["x"],"nodes":[{"id":1,"parent":null,"detail":"SCAN cnt","op":{"type":"scan","table":"cnt","subquery":{"execution":"coroutine","cte_id":0,"recursive":true},"source":"subquery"}},{"id":5,"parent":1,"detail":"SETUP","op":{"type":"recursive_setup"}},{"id":6,"parent":5,"detail":"SCAN CONSTANT ROW","op":{"type":"constant_row"}},{"id":17,"parent":1,"detail":"RECURSIVE STEP","op":{"type":"recursive_step"}},{"id":18,"parent":17,"detail":"SCAN cnt","op":{"type":"scan","table":"cnt","source":"recursive_cte_input"}}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH RECURSIVE cnt(x) AS (\n SELECT 1\n UNION ALL\n SELECT x+1 FROM cnt WHERE x < 10\n )\n SELECT x FROM cnt;","result_columns":["x"],"nodes":[{"id":1,"parent":null,"detail":"SCAN cnt","op":{"type":"scan","table":"cnt","subquery":{"execution":"coroutine","cte_id":0,"recursive":true},"source":"subquery","estimate":{"input_rows":1,"rows_per_input":1000000,"output_rows":1000000,"access_cost":23000,"total_cost":23000}}},{"id":5,"parent":1,"detail":"SETUP","op":{"type":"recursive_setup"}},{"id":6,"parent":5,"detail":"SCAN CONSTANT ROW","op":{"type":"constant_row"}},{"id":17,"parent":1,"detail":"RECURSIVE STEP","op":{"type":"recursive_step"}},{"id":18,"parent":17,"detail":"SCAN cnt","op":{"type":"scan","table":"cnt","source":"recursive_cte_input","estimate":{"input_rows":1,"rows_per_input":0.4,"output_rows":0.4,"access_cost":1.003,"total_cost":1.003}}}]} } @setup schema @@ -136,7 +136,7 @@ expect { 4|1|0|SCAN users 13|0|0|SEARCH u USING INDEX idx_users_age (age=?) 29|0|0|USE SORTER FOR ORDER BY - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.age IN (SELECT age FROM users WHERE city_id = 3)\n ORDER BY u.name;","result_columns":["name"],"nodes":[{"id":1,"parent":null,"detail":"LIST SUBQUERY 1","op":{"type":"list_subquery","subquery_id":1,"correlated":false}},{"id":4,"parent":1,"detail":"SCAN users","op":{"type":"scan","table":"users","source":"table"}},{"id":13,"parent":null,"detail":"SEARCH u USING INDEX idx_users_age (age=?)","op":{"type":"search","table":"users","alias":"u","search_kind":"in_seek","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"constraints":["age=?"]}},{"id":29,"parent":null,"detail":"USE SORTER FOR ORDER BY","op":{"type":"order_by","method":"sorter"}}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.age IN (SELECT age FROM users WHERE city_id = 3)\n ORDER BY u.name;","result_columns":["name"],"nodes":[{"id":1,"parent":null,"detail":"LIST SUBQUERY 1","op":{"type":"list_subquery","subquery_id":1,"correlated":false}},{"id":4,"parent":1,"detail":"SCAN users","op":{"type":"scan","table":"users","source":"table","estimate":{"input_rows":1,"rows_per_input":100000,"output_rows":100000,"access_cost":23000,"total_cost":23000}}},{"id":13,"parent":null,"detail":"SEARCH u USING INDEX idx_users_age (age=?)","op":{"type":"search","table":"users","alias":"u","search_kind":"in_seek","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"constraints":["age=?"],"estimate":{"input_rows":1,"rows_per_input":1000,"output_rows":1000,"access_cost":3139.4238618387276,"total_cost":3139.4238618387276}}},{"id":29,"parent":null,"detail":"USE SORTER FOR ORDER BY","op":{"type":"order_by","method":"sorter"}}]} } @setup schema @@ -169,7 +169,7 @@ test quotes-in-the-statement-are-escaped-in-the-echo { ORDER BY u.age; } expect { - {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.name = 'a \"quoted\" name'\n ORDER BY u.age;","result_columns":["name"],"nodes":[{"id":1,"parent":null,"detail":"SCAN users AS u USING INDEX idx_users_age","op":{"type":"scan","table":"users","alias":"u","source":"table","index":{"name":"idx_users_age","covering":false,"ephemeral":false}}}]} + {"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.name = 'a \"quoted\" name'\n ORDER BY u.age;","result_columns":["name"],"nodes":[{"id":1,"parent":null,"detail":"SCAN users AS u USING INDEX idx_users_age","op":{"type":"scan","table":"users","alias":"u","source":"table","index":{"name":"idx_users_age","covering":false,"ephemeral":false},"estimate":{"input_rows":1,"rows_per_input":100000,"output_rows":100000,"access_cost":33003.51,"total_cost":33003.51}}}]} } test the-format-clause-is-case-insensitive { diff --git a/conformance/sqlite-sqltests/turso/fts.sqltest b/conformance/sqlite-sqltests/turso/fts.sqltest index e852eb07..f011b05f 100644 --- a/conformance/sqlite-sqltests/turso/fts.sqltest +++ b/conformance/sqlite-sqltests/turso/fts.sqltest @@ -2,6 +2,29 @@ @skip-file-if sqlite "sqlite does not support index method syntax or fts_match()" @skip-file-if mvcc "index methods not supported in mvcc mode" +@backend cli +test fts-ranked-limit-offset { + CREATE TABLE d(id INTEGER PRIMARY KEY, body TEXT); + CREATE INDEX fx ON d USING fts(body); + INSERT INTO d VALUES (1, 'x'), (2, 'x x'), (3, 'x x x'), (4, 'x x x x'); + SELECT id FROM d WHERE fts_match(body, 'x') ORDER BY fts_score(body, 'x') DESC; + SELECT id FROM d WHERE fts_match(body, 'x') ORDER BY fts_score(body, 'x') DESC LIMIT 2 OFFSET 1; + SELECT id FROM d WHERE fts_match(body, 'x') ORDER BY fts_score(body, 'x') DESC LIMIT 2 OFFSET 3; + SELECT id FROM d WHERE fts_match(body, 'x') ORDER BY fts_score(body, 'x') DESC LIMIT 2 OFFSET 4; + SELECT id FROM d WHERE fts_match(body, 'x') ORDER BY fts_score(body, 'x') DESC LIMIT 2 OFFSET 0; +} +expect { + 4 + 3 + 2 + 1 + 3 + 2 + 1 + 4 + 3 +} + @backend cli test fts-delete-where-match { CREATE TABLE t (id INTEGER PRIMARY KEY, x TEXT); @@ -981,3 +1004,68 @@ test fts-create-index-batch-flush-returns-each-rowid-once { expect { 20000|20000|20000 } + +@backend cli +test fts-row-dependent-index-query { + CREATE TABLE d(id INTEGER PRIMARY KEY, body); + CREATE INDEX fx ON d USING fts(body); + INSERT INTO d VALUES + (1, 'database'), (2, 'sql'), (3, 'database sql'), (4, 'database NOT sql'); + SELECT id FROM d WHERE fts_match(body, body) ORDER BY id; + SELECT id FROM d WHERE fts_match(body, coalesce(body, 'database')) ORDER BY id; + CREATE INDEX source ON d(body); + SELECT id FROM d INDEXED BY source WHERE fts_match(body, body) ORDER BY id; + SELECT count(*) FROM d WHERE id = 4 AND fts_match(body, body); +} +expect { + 1 + 2 + 3 + 1 + 2 + 3 + 1 + 2 + 3 + 0 +} + +@backend cli +test fts-row-dependent-index-query-joins { + CREATE TABLE d(id INTEGER PRIMARY KEY, body); + CREATE INDEX fx ON d USING fts(body); + INSERT INTO d VALUES (1, 'database'), (2, 'sql'), (3, 'database sql'); + CREATE TABLE q(id INTEGER PRIMARY KEY, term); + INSERT INTO q VALUES + (1, 'database'), (2, 'missing'), (3, 'sql'), + (4, 'database'), (5, 'database NOT sql'); + SELECT q.id, d.id FROM d CROSS JOIN q + WHERE fts_match(d.body, q.term) ORDER BY q.id, d.id; + SELECT q.id, d.id FROM q CROSS JOIN d + WHERE fts_match(d.body, q.term) ORDER BY q.id, d.id; + SELECT q.id, d.id FROM d JOIN q + WHERE fts_match(d.body, coalesce(q.term, 'missing')) ORDER BY q.id, d.id; +} +expect { + 1|1 + 1|3 + 3|2 + 3|3 + 4|1 + 4|3 + 5|1 + 1|1 + 1|3 + 3|2 + 3|3 + 4|1 + 4|3 + 5|1 + 1|1 + 1|3 + 3|2 + 3|3 + 4|1 + 4|3 + 5|1 +} diff --git a/conformance/sqlite-sqltests/turso/mvcc_checkpoint_delete_order.sqltest b/conformance/sqlite-sqltests/turso/mvcc_checkpoint_delete_order.sqltest new file mode 100644 index 00000000..4069c2cd --- /dev/null +++ b/conformance/sqlite-sqltests/turso/mvcc_checkpoint_delete_order.sqltest @@ -0,0 +1,51 @@ +@database :memory: + +test mvcc-checkpoint-before-delete { + PRAGMA journal_mode = 'mvcc'; + CREATE TABLE t (x); + INSERT INTO t SELECT value FROM generate_series(1, 3); + CREATE INDEX lol ON t(x); + PRAGMA wal_checkpoint(TRUNCATE); + DELETE FROM t WHERE x = 2; + SELECT x FROM t ORDER BY x; +} +expect { + mvcc + 0|0|0 + 1 + 3 +} + +test mvcc-delete-then-checkpoint { + PRAGMA journal_mode = 'mvcc'; + CREATE TABLE t (x); + INSERT INTO t SELECT value FROM generate_series(1, 3); + CREATE INDEX lol ON t(x); + DELETE FROM t WHERE x = 2; + PRAGMA wal_checkpoint(TRUNCATE); + SELECT x FROM t ORDER BY x; +} +expect { + mvcc + 0|0|0 + 1 + 3 +} + +test mvcc-checkpoint-delete-checkpoint { + PRAGMA journal_mode = 'mvcc'; + CREATE TABLE t (x); + INSERT INTO t SELECT value FROM generate_series(1, 3); + CREATE INDEX lol ON t(x); + PRAGMA wal_checkpoint(TRUNCATE); + DELETE FROM t WHERE x = 2; + PRAGMA wal_checkpoint(TRUNCATE); + SELECT x FROM t ORDER BY x; +} +expect { + mvcc + 0|0|0 + 0|0|0 + 1 + 3 +} diff --git a/conformance/sqlite-sqltests/turso/mvcc_sequence.sqltest b/conformance/sqlite-sqltests/turso/mvcc_sequence.sqltest index 0187cfaf..7820ff95 100644 --- a/conformance/sqlite-sqltests/turso/mvcc_sequence.sqltest +++ b/conformance/sqlite-sqltests/turso/mvcc_sequence.sqltest @@ -228,3 +228,185 @@ expect { 3|c 4|d } + +test mvcc-autoincrement-multiple-tables-sqlite-sequence { + PRAGMA journal_mode = 'mvcc'; + CREATE TABLE a (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE b (id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + INSERT INTO a(v) VALUES ('a1'),('a2'),('a3'); + INSERT INTO b(v) VALUES ('b1'),('b2'),('b3'),('b4'),('b5'),('b6'),('b7'); + PRAGMA wal_checkpoint(TRUNCATE); + SELECT name, seq FROM sqlite_sequence ORDER BY name; +} +expect { + mvcc + 0|0|0 + a|3 + b|7 +} + +test mvcc-autoincrement-checkpoint-compaction-many-tables { + PRAGMA journal_mode = 'mvcc'; + CREATE TABLE stress_00(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_01(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_02(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_03(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_04(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_05(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_06(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_07(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_08(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_09(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_10(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_11(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_12(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_13(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_14(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_15(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_16(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_17(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_18(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + CREATE TABLE stress_19(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + INSERT INTO stress_00(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_01(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_02(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_03(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_04(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_05(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_06(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_07(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_08(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_09(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_10(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_11(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_12(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_13(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_14(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_15(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_16(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_17(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_18(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + INSERT INTO stress_19(v) VALUES ('r0'),('r1'),('r2'),('r3'),('r4'),('r5'),('r6'),('r7'),('r8'),('r9'),('r10'),('r11'),('r12'),('r13'),('r14'),('r15'),('r16'),('r17'),('r18'),('r19'),('r20'),('r21'),('r22'),('r23'),('r24'),('r25'),('r26'),('r27'),('r28'),('r29'),('r30'),('r31'),('r32'),('r33'),('r34'),('r35'),('r36'),('r37'),('r38'),('r39'),('r40'),('r41'),('r42'),('r43'),('r44'),('r45'),('r46'),('r47'),('r48'),('r49'); + PRAGMA wal_checkpoint(TRUNCATE); + SELECT name, seq FROM sqlite_sequence ORDER BY name; +} +expect { + mvcc + 0|0|0 + stress_00|50 + stress_01|50 + stress_02|50 + stress_03|50 + stress_04|50 + stress_05|50 + stress_06|50 + stress_07|50 + stress_08|50 + stress_09|50 + stress_10|50 + stress_11|50 + stress_12|50 + stress_13|50 + stress_14|50 + stress_15|50 + stress_16|50 + stress_17|50 + stress_18|50 + stress_19|50 +} + +test mvcc-user-sequence-backing-table-compacted-after-commit { + PRAGMA journal_mode = 'mvcc'; + CREATE SEQUENCE compact_seq START 1 INCREMENT 1; + SELECT nextval('compact_seq'); + SELECT nextval('compact_seq'); + SELECT nextval('compact_seq'); + SELECT nextval('compact_seq'); + SELECT nextval('compact_seq'); + PRAGMA wal_checkpoint(TRUNCATE); + SELECT COUNT(*) FROM "__turso_internal_seq_compact_seq"; + SELECT value FROM "__turso_internal_seq_compact_seq"; +} +expect { + mvcc + 1 + 2 + 3 + 4 + 5 + 0|0|0 + 1 + 5 +} + +test mvcc-autoincrement-backing-table-compacted-after-commit { + PRAGMA journal_mode = 'mvcc'; + CREATE TABLE compacted(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + INSERT INTO compacted(v) VALUES ('row1'); + INSERT INTO compacted(v) VALUES ('row2'); + INSERT INTO compacted(v) VALUES ('row3'); + INSERT INTO compacted(v) VALUES ('row4'); + INSERT INTO compacted(v) VALUES ('row5'); + PRAGMA wal_checkpoint(TRUNCATE); + SELECT COUNT(*) FROM "__turso_internal_seq___turso_internal_autoincrement_compacted"; +} +expect { + mvcc + 0|0|0 + 1 +} + +test mvcc-autoincrement-rollback-does-not-pollute-sqlite-sequence { + PRAGMA journal_mode = 'mvcc'; + CREATE TABLE rollback_t(id INTEGER PRIMARY KEY AUTOINCREMENT, v TEXT); + INSERT INTO rollback_t(v) VALUES ('committed1'); + INSERT INTO rollback_t(v) VALUES ('committed2'); + INSERT INTO rollback_t(v) VALUES ('committed3'); + BEGIN; + INSERT INTO rollback_t(v) VALUES ('rolledback1'); + INSERT INTO rollback_t(v) VALUES ('rolledback2'); + INSERT INTO rollback_t(v) VALUES ('rolledback3'); + INSERT INTO rollback_t(v) VALUES ('rolledback4'); + INSERT INTO rollback_t(v) VALUES ('rolledback5'); + ROLLBACK; + PRAGMA wal_checkpoint(TRUNCATE); + SELECT name, seq FROM sqlite_sequence WHERE name = 'rollback_t'; + INSERT INTO rollback_t(v) VALUES ('after_rollback'); + SELECT id FROM rollback_t WHERE v = 'after_rollback'; +} +expect { + mvcc + 0|0|0 + rollback_t|3 + 4 +} + +test mvcc-user-sequence-rollback-does-not-pollute-watermark { + PRAGMA journal_mode = 'mvcc'; + CREATE SEQUENCE rb_seq START 1 INCREMENT 1; + SELECT nextval('rb_seq'); + SELECT nextval('rb_seq'); + SELECT nextval('rb_seq'); + BEGIN; + SELECT nextval('rb_seq'); + SELECT nextval('rb_seq'); + SELECT nextval('rb_seq'); + SELECT nextval('rb_seq'); + SELECT nextval('rb_seq'); + ROLLBACK; + PRAGMA wal_checkpoint(TRUNCATE); + SELECT value FROM "__turso_internal_seq_rb_seq"; +} +expect { + mvcc + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 0|0|0 + 3 +} diff --git a/conformance/sqlite-sqltests/turso/sequence.sqltest b/conformance/sqlite-sqltests/turso/sequence.sqltest index 01589d57..bf8ac26c 100644 --- a/conformance/sqlite-sqltests/turso/sequence.sqltest +++ b/conformance/sqlite-sqltests/turso/sequence.sqltest @@ -272,6 +272,38 @@ expect { # --- Reserved namespace --- +test sequence-backing-table-rejects-uppercase-insert { + CREATE SEQUENCE s1; + INSERT INTO __TURSO_INTERNAL_SEQ_S1 DEFAULT VALUES; +} +expect error { + table __TURSO_INTERNAL_SEQ_S1 may not be modified +} + +test sequence-backing-table-rejects-mixed-case-insert { + CREATE SEQUENCE s2; + INSERT INTO __TuRsO_iNtErNaL_sEq_S2 DEFAULT VALUES; +} +expect error { + table __TuRsO_iNtErNaL_sEq_S2 may not be modified +} + +test sequence-backing-table-rejects-uppercase-update { + CREATE SEQUENCE s3; + UPDATE __TURSO_INTERNAL_SEQ_S3 SET value = 2; +} +expect error { + table __TURSO_INTERNAL_SEQ_S3 may not be modified +} + +test sequence-backing-table-rejects-mixed-case-update { + CREATE SEQUENCE s4; + UPDATE __TuRsO_iNtErNaL_sEq_S4 SET value = 2; +} +expect error { + table __TuRsO_iNtErNaL_sEq_S4 may not be modified +} + # `__turso_internal_autoincrement_` is the implicit sequence # name CREATE TABLE … AUTOINCREMENT installs internally. The nextval/ # advance_past/setval translators detect this prefix to mirror the diff --git a/conformance/sqlite-sqltests/turso/upsert-target-alias.sqltest b/conformance/sqlite-sqltests/turso/upsert-target-alias.sqltest new file mode 100644 index 00000000..7976bbca --- /dev/null +++ b/conformance/sqlite-sqltests/turso/upsert-target-alias.sqltest @@ -0,0 +1,43 @@ +@database :memory: + +test upsert-target-alias-shadows-excluded { + CREATE TABLE t(k INT PRIMARY KEY, n INT); + INSERT INTO t VALUES(1,10); + INSERT INTO t AS excluded VALUES(1,99) + ON CONFLICT(k) DO UPDATE SET n = excluded.n; + SELECT n FROM t; +} +expect { + 10 +} + +test upsert-target-alias-is-only-target-name { + CREATE TABLE t(k INT PRIMARY KEY, n INT); + INSERT INTO t VALUES(1,10); + INSERT INTO t AS x VALUES(1,7) + ON CONFLICT(k) DO UPDATE SET n = x.n + 1; + SELECT n FROM t; +} +expect { + 11 +} + +test upsert-base-name-is-out-of-scope-when-aliased { + CREATE TABLE t(k INT PRIMARY KEY, n INT); + INSERT INTO t VALUES(1,10); + INSERT INTO t AS x VALUES(1,7) + ON CONFLICT(k) DO UPDATE SET n = t.n + 1; +} +expect error { + no such column +} + +test plain-update-target-alias-still-works { + CREATE TABLE t(k INT PRIMARY KEY, n INT); + INSERT INTO t VALUES(1,10); + UPDATE t AS x SET n = x.n + 1; + SELECT n FROM t; +} +expect { + 11 +} diff --git a/conformance/sqlite-sqltests/turso/vacuum.sqltest b/conformance/sqlite-sqltests/turso/vacuum.sqltest new file mode 100644 index 00000000..d01b63aa --- /dev/null +++ b/conformance/sqlite-sqltests/turso/vacuum.sqltest @@ -0,0 +1,33 @@ +@database :temp: + +test vacuum-nonexistent-schema { + VACUUM nonexistent INTO 'out.db'; +} +expect error { + no such database: nonexistent +} + +@skip-if mvcc "plain VACUUM requires PRAGMA wal_checkpoint(TRUNCATE) first on an MVCC database" +test vacuum-preserves-user-sequence { + CREATE SEQUENCE s START WITH 1 INCREMENT BY 1; + SELECT nextval('s'); + SELECT nextval('s'); + VACUUM; + PRAGMA integrity_check; + SELECT nextval('s'); +} +expect { + 1 + 2 + ok + 3 +} + +test vacuum-rejects-query-only { + CREATE TABLE t(a INTEGER); + PRAGMA query_only = 1; + VACUUM; +} +expect error { + Cannot execute VACUUM in query_only mode +} diff --git a/conformance/sqlite-sqltests/turso/without_rowid.sqltest b/conformance/sqlite-sqltests/turso/without_rowid.sqltest index ab15737b..11379423 100644 --- a/conformance/sqlite-sqltests/turso/without_rowid.sqltest +++ b/conformance/sqlite-sqltests/turso/without_rowid.sqltest @@ -19,6 +19,17 @@ expect error { UNIQUE constraint failed } +test without-rowid-join-does-not-use-a-hash-join { + CREATE TABLE left_table(k1 INTEGER, k2 INTEGER, PRIMARY KEY(k2, k1)) WITHOUT ROWID; + CREATE TABLE right_table(k1 INTEGER, k2 INTEGER, PRIMARY KEY(k2, k1)) WITHOUT ROWID; + INSERT INTO left_table SELECT value, value % 23 FROM generate_series(1, 200); + INSERT INTO right_table SELECT value, value % 23 FROM generate_series(1, 220); + SELECT count(*) FROM left_table JOIN right_table ON left_table.k1 = right_table.k1; +} +expect { + 200 +} + test without-rowid-missing-primary-key-rejected { CREATE TABLE t(a INTEGER, b TEXT) WITHOUT ROWID; } diff --git a/conformance/sqlite-sqltests/type_affinity.sqltest b/conformance/sqlite-sqltests/type_affinity.sqltest new file mode 100644 index 00000000..cdd5e985 --- /dev/null +++ b/conformance/sqlite-sqltests/type_affinity.sqltest @@ -0,0 +1,54 @@ +@database :memory: + +@cross-check-integrity +test type-affinity-text-column-numeric-literal-float { + CREATE TABLE t (c TEXT); + INSERT INTO t VALUES ('0.0'); + SELECT c FROM t WHERE c = 0.0; +} +expect { + 0.0 +} + +@cross-check-integrity +test type-affinity-text-column-numeric-literal-int { + CREATE TABLE t (c TEXT); + INSERT INTO t VALUES ('42'); + SELECT c FROM t WHERE c = 42; +} +expect { + 42 +} + +@cross-check-integrity +test type-affinity-text-column-negative-numeric-literal { + CREATE TABLE t (c TEXT); + INSERT INTO t VALUES ('-5.5'); + SELECT c FROM t WHERE c = -5.5; +} +expect { + -5.5 +} + +@cross-check-integrity +test type-affinity-numeric-column-text-literal { + CREATE TABLE t (c NUMERIC); + INSERT INTO t VALUES ('123.45'); + SELECT c FROM t WHERE c = 123.45; +} +expect { + 123.45 +} + +@cross-check-integrity +test type-affinity-text-column-order-by-numeric-filter { + CREATE TABLE t (id INTEGER, c TEXT); + INSERT INTO t VALUES (1, '0.0'); + INSERT INTO t VALUES (2, '1.0'); + INSERT INTO t VALUES (3, '0.0'); + SELECT id, c FROM t WHERE c = 0.0 ORDER BY c ASC, rowid ASC; +} +expect { + 1|0.0 + 3|0.0 +} diff --git a/conformance/sqlite-sqltests/unnest-correlated.sqltest b/conformance/sqlite-sqltests/unnest-correlated.sqltest index 7f420ba7..1cee6b20 100644 --- a/conformance/sqlite-sqltests/unnest-correlated.sqltest +++ b/conformance/sqlite-sqltests/unnest-correlated.sqltest @@ -28,6 +28,73 @@ setup correlated_rows { INSERT INTO outer_key2 VALUES (1, 10), (3, 20), (5, 40); } +setup correlated_in_indexes { + CREATE INDEX inner_amount ON inner_rows(amount); + CREATE INDEX inner_key_amount ON inner_rows(key1, amount); + CREATE INDEX inner_text_nocase ON inner_rows(text_key COLLATE NOCASE); +} + +setup correlated_comparison_many_rows { + CREATE TABLE many_outer(id INTEGER PRIMARY KEY, k INTEGER, v, text_key TEXT COLLATE NOCASE); + CREATE TABLE many_inner(k INTEGER, v, text_key TEXT COLLATE BINARY); + WITH RECURSIVE numbers(n) AS ( + VALUES(1) UNION ALL SELECT n + 1 FROM numbers WHERE n < 45 + ) + INSERT INTO many_outer + SELECT n, + CASE WHEN n % 7 = 0 THEN NULL ELSE n % 5 END, + CASE WHEN n % 9 = 0 THEN NULL + WHEN n % 15 = 0 THEN x'0102' + WHEN n % 13 = 0 THEN CAST(n % 17 AS REAL) + WHEN n % 3 = 0 THEN CAST(n % 17 AS TEXT) + ELSE n % 17 END, + CASE WHEN n % 11 = 0 THEN NULL ELSE char(65 + n % 5) END + FROM numbers; + WITH RECURSIVE numbers(n) AS ( + VALUES(1) UNION ALL SELECT n + 1 FROM numbers WHERE n < 50 + ) + INSERT INTO many_inner + SELECT CASE WHEN n % 8 = 0 THEN NULL ELSE n % 5 END, + CASE WHEN n % 10 = 0 THEN NULL + WHEN n % 15 = 0 THEN x'0102' + WHEN n % 13 = 0 THEN CAST(n % 17 AS REAL) + WHEN n % 4 = 0 THEN CAST(n % 17 AS TEXT) + ELSE n % 17 END, + CASE WHEN n % 12 = 0 THEN NULL ELSE char(65 + n % 5) END + FROM numbers; + CREATE INDEX many_inner_k_v ON many_inner(k, v); + CREATE INDEX many_inner_v ON many_inner(v); + ANALYZE; +} + +setup selective_join_plan_rows { + CREATE TABLE plan_outer( + id INTEGER PRIMARY KEY, + k INTEGER, + lim INTEGER, + max_sum INTEGER, + text_key TEXT + ); + CREATE TABLE plan_inner(k INTEGER, x INTEGER, v TEXT, text_key TEXT); + CREATE TABLE plan_list(k INTEGER); + WITH RECURSIVE rows(value) AS ( + VALUES(1) + UNION ALL + SELECT value + 1 FROM rows WHERE value < 100 + ) + INSERT INTO plan_outer + SELECT value, value, value, value, value FROM rows; + WITH RECURSIVE rows(value) AS ( + VALUES(1) + UNION ALL + SELECT value + 1 FROM rows WHERE value < 110 + ) + INSERT INTO plan_inner + SELECT value, value, json_object('n', value), value FROM rows; + INSERT INTO plan_list SELECT k FROM plan_outer; + ANALYZE; +} + # A table function uses hidden columns to hold its arguments. Those checks # must stay inside the correlated subquery. test one-value-table-function-keeps-arguments { @@ -262,20 +329,20 @@ expect pattern { # A direct comparison can join the requested rows before building each string. # This avoids both the correlated scan and strings for unused inner keys. -@setup correlated_rows +@setup selective_join_plan_rows @skip-if sqlite "SQLite keeps this group_concat as a correlated subquery" test one-value-group-concat-joins-before-grouping { EXPLAIN QUERY PLAN SELECT o.id - FROM outer_rows o + FROM plan_outer o WHERE o.text_key = ( SELECT group_concat(i.text_key, ':') - FROM inner_rows i - WHERE i.key1 = o.key1 + FROM plan_inner i + WHERE i.k = o.k AND i.x = 1 ); } expect pattern { - (?s)JOIN inner_rows AS i.*SCAN outer_rows AS o.*USE SORTER FOR GROUP BY + (?s)JOIN plan_inner AS i.*USE SORTER FOR GROUP BY } # The joined form keeps a separate string for each outer row. A key with no @@ -399,6 +466,32 @@ expect { 5 } +# An outer query with no input rows does not need to build or probe a grouped +# table for its scalar subquery. +@setup correlated_rows +test one-value-under-false-where-stays-unused { + SELECT (SELECT avg(i.amount) FROM inner_rows i WHERE i.key1 = o.key1) / 2 + FROM outer_rows o + WHERE 0; +} +expect { +} + +# Unnesting the IN subquery moves its WHERE clause to the outer query, and the +# text term there is always false. The grouped table built for the scalar +# subquery then has nothing to read, so the outer query keeps its correlated +# form. +@setup correlated_rows +test one-value-beside-in-subquery-with-false-text-term { + SELECT (SELECT count(i.amount) FROM inner_rows i WHERE i.key1 = o.key1) + FROM outer_rows o + WHERE o.amount IN ( + SELECT k.key2 FROM outer_key2 k WHERE 'not a number' AND o.key1 = k.key2 + ); +} +expect { +} + @setup correlated_rows test one-value-math-with-null-result { SELECT id @@ -557,6 +650,170 @@ expect { 2 } +@setup correlated_rows +@setup correlated_in_indexes +@var left_value { o.amount | o.key1 | 10 | NULL } +@var comparison { + i.amount = o.amount | o.amount = i.amount + | i.amount <> o.amount | o.amount <> i.amount + | i.amount < o.amount | o.amount < i.amount + | i.amount <= o.amount | o.amount <= i.amount + | i.amount > o.amount | o.amount > i.amount + | i.amount >= o.amount | o.amount >= i.amount + | i.amount IS o.amount | o.amount IS i.amount + | i.amount IS NOT o.amount | o.amount IS NOT i.amount +} +@var link { i.key1 = o.key1 | i.key1 IS o.key1 | i.key1 < o.key1 } +matrix correlated-in-comparison-results { + SELECT o.id + FROM outer_rows AS o + WHERE $left_value IN ( + SELECT i.amount FROM inner_rows AS i + WHERE $comparison AND $link + ) + ORDER BY o.id; +} + +@setup correlated_comparison_many_rows +@var left_value { o.v | 5 | NULL } +@var comparison { + i.v < o.v | o.v < i.v + | i.v <= o.v | o.v <= i.v + | i.v <> o.v | o.v <> i.v + | i.v IS o.v | i.v IS NOT o.v +} +@var link { i.k = o.k | i.k IS o.k | i.k < o.k } +matrix correlated-in-comparison-many-rows { + SELECT o.id + FROM many_outer AS o + WHERE $left_value IN ( + SELECT i.v FROM many_inner AS i WHERE $comparison AND $link + ) + ORDER BY o.id; +} + +@setup correlated_comparison_many_rows +@var polarity { EXISTS | NOT EXISTS } +@var comparison { + i.v < o.v | o.v < i.v + | i.v <= o.v | o.v <= i.v + | i.v <> o.v | o.v <> i.v + | i.v IS o.v | i.v IS NOT o.v +} +@var link { i.k = o.k | i.k IS o.k | i.k < o.k } +matrix correlated-exists-comparison-many-rows { + SELECT o.id + FROM many_outer AS o + WHERE $polarity ( + SELECT 1 FROM many_inner AS i WHERE $comparison AND $link + ) + ORDER BY o.id; +} + +@setup correlated_rows +@setup correlated_in_indexes +@var left_value { o.amount | 10 | NULL } +@var comparison { + i.key1 < o.key1 | o.key1 < i.key1 + | i.key1 <= o.key1 | o.key1 <= i.key1 + | i.key1 <> o.key1 | o.key1 <> i.key1 + | i.key1 IS o.key1 | i.key1 IS NOT o.key1 +} +matrix correlated-in-with-only-one-comparison { + SELECT o.id + FROM outer_rows AS o + WHERE $left_value IN ( + SELECT i.amount FROM inner_rows AS i WHERE $comparison + ) + ORDER BY o.id; +} + +@setup correlated_rows +@setup correlated_in_indexes +@var comparison { + i.text_key < o.text_key COLLATE NOCASE + | o.text_key COLLATE NOCASE < i.text_key + | i.text_key IS NOT o.text_key COLLATE NOCASE +} +matrix correlated-in-keeps-text-collation { + SELECT o.id + FROM outer_rows AS o + WHERE o.text_key COLLATE NOCASE IN ( + SELECT i.text_key FROM inner_rows AS i WHERE $comparison + ) + ORDER BY o.id; +} + +@setup correlated_rows +@var join_type { LEFT | RIGHT } +@var comparison { i.amount < k.key2 | k.key2 < i.amount | i.amount IS NOT k.key2 } +matrix correlated-in-after-outer-join { + SELECT o.id, k.id + FROM outer_rows AS o $join_type JOIN outer_key2 AS k ON o.id = k.id + WHERE k.key2 IN ( + SELECT i.amount FROM inner_rows AS i WHERE $comparison + ) + ORDER BY o.id, k.id; +} + +@setup correlated_rows +@var comparison { s.amount < i.amount | i.amount < s.amount | s.amount IS NOT i.amount } +matrix correlated-in-after-full-join { + SELECT o.id, k.id, i.key1, i.amount + FROM outer_rows AS o + JOIN outer_key2 AS k ON o.id = k.id + FULL JOIN inner_rows AS i ON k.key2 = i.key2 + WHERE i.amount IN ( + SELECT s.amount FROM outer_rows AS s WHERE $comparison + ) + ORDER BY o.id, k.id, i.key1, i.amount; +} + +@setup correlated_rows +@var operator { IN | NOT IN } +@var comparison { i.amount < o.amount | o.amount < i.amount | i.amount IS NOT o.amount } +@var or_term { | OR o.id = 4 } +matrix correlated-in-with-not-in-and-or { + SELECT o.id + FROM outer_rows AS o + WHERE o.amount $operator ( + SELECT i.amount FROM inner_rows AS i WHERE $comparison + ) $or_term + ORDER BY o.id; +} + +@setup correlated_rows +@setup correlated_in_indexes +@skip-if sqlite "query plan explanation output is different in sqlite" +test correlated-in-inequality-uses-indexed-semi-join { + EXPLAIN QUERY PLAN + SELECT o.id + FROM outer_rows AS o + WHERE o.amount IN ( + SELECT i.amount FROM inner_rows AS i + WHERE i.key1 = o.key1 AND i.amount < o.amount + ); +} +expect pattern { + (?s)SCAN outer_rows AS o.*SEARCH i USING COVERING INDEX inner_key_amount +} + +@setup correlated_rows +@setup correlated_in_indexes +@skip-if sqlite "query plan explanation output is different in sqlite" +test correlated-in-with-only-inequality-uses-indexed-semi-join { + EXPLAIN QUERY PLAN + SELECT o.id + FROM outer_rows AS o + WHERE o.amount IN ( + SELECT i.amount FROM inner_rows AS i INDEXED BY inner_key_amount + WHERE i.key1 < o.key1 + ); +} +expect pattern { + (?s)SCAN outer_rows AS o.*SEARCH i USING COVERING INDEX inner_key_amount \(key1<\?\) +} + @setup correlated_rows test correlated-in-through-correlated-derived-table { SELECT o.id @@ -830,21 +1087,19 @@ expect { # A sum can join the requested inner rows before grouping. This avoids one # subquery call per outer row without adding values for unused inner keys. @skip-if sqlite "SQLite keeps this sum as a correlated subquery" +@setup selective_join_plan_rows test one-value-sum-joins-before-grouping { - CREATE TABLE sum_join_outer(id INTEGER PRIMARY KEY, k INTEGER, max_sum INTEGER); - CREATE TABLE sum_join_inner(k INTEGER, x INTEGER); - EXPLAIN QUERY PLAN SELECT o.id - FROM sum_join_outer o + FROM plan_outer o WHERE o.max_sum > ( SELECT sum(i.x) - FROM sum_join_inner i - WHERE i.k = o.k + FROM plan_inner i + WHERE i.k = o.k AND i.x = 1 ); } expect pattern { - (?s)HASH JOIN sum_join_inner AS i.*SCAN sum_join_outer AS o.*USE SORTER FOR GROUP BY + (?s)JOIN plan_inner AS i.*USE SORTER FOR GROUP BY } # Each outer row keeps its own sum. A missing match gives NULL. The values for @@ -1000,42 +1255,37 @@ expect pattern { # result is the same for every joined copy, so it does not block the joined # form. TPC-H q20 has this shape. @skip-if sqlite "SQLite keeps this sum as a correlated subquery" +@setup selective_join_plan_rows test one-value-sum-join-allows-uncorrelated-subquery-term { - CREATE TABLE in_term_outer(id INTEGER PRIMARY KEY, k INTEGER, lim INTEGER); - CREATE TABLE in_term_inner(k INTEGER, x INTEGER); - CREATE TABLE in_term_list(k INTEGER); - EXPLAIN QUERY PLAN SELECT o.id - FROM in_term_outer o - WHERE o.k IN (SELECT k FROM in_term_list) + FROM plan_outer o + WHERE o.k IN (SELECT k FROM plan_list) AND o.lim > ( SELECT sum(i.x) - FROM in_term_inner i - WHERE i.k = o.k + FROM plan_inner i + WHERE i.k = o.k AND i.x = 1 ); } expect pattern { - (?s)JOIN in_term_inner AS i.*USE SORTER FOR GROUP BY + (?s)JOIN plan_inner AS i.*USE SORTER FOR GROUP BY } # The subquery can be either side of the comparison. @skip-if sqlite "SQLite keeps this sum as a correlated subquery" +@setup selective_join_plan_rows test one-value-sum-join-subquery-on-left-side { - CREATE TABLE left_side_outer(id INTEGER PRIMARY KEY, k INTEGER, lim INTEGER); - CREATE TABLE left_side_inner(k INTEGER, x INTEGER); - EXPLAIN QUERY PLAN SELECT o.id - FROM left_side_outer o + FROM plan_outer o WHERE ( SELECT sum(i.x) - FROM left_side_inner i - WHERE i.k = o.k + FROM plan_inner i + WHERE i.k = o.k AND i.x = 1 ) > o.lim; } expect pattern { - (?s)JOIN left_side_inner AS i.*USE SORTER FOR GROUP BY + (?s)JOIN plan_inner AS i.*USE SORTER FOR GROUP BY } # key 1: 20 > 15. key 2: 10 > 5. key 3: no rows, sum is NULL, not returned. @@ -1239,21 +1489,19 @@ expect { } @skip-if sqlite "Turso uses a different plan for this query" +@setup selective_join_plan_rows test one-value-json-sum-uses-joined-form { - CREATE TABLE json_sum_plan_outer(id INTEGER PRIMARY KEY, k INTEGER, max_sum INTEGER); - CREATE TABLE json_sum_plan_inner(k INTEGER, v TEXT); - EXPLAIN QUERY PLAN SELECT o.id - FROM json_sum_plan_outer o + FROM plan_outer o WHERE o.max_sum > ( SELECT sum(json_extract(i.v, '$.n')) - FROM json_sum_plan_inner i - WHERE i.k = o.k + FROM plan_inner i + WHERE i.k = o.k AND i.x = 1 ); } expect pattern { - (?s)JOIN json_sum_plan_inner AS i.*SCAN json_sum_plan_outer AS o.*USE SORTER FOR GROUP BY + (?s)JOIN plan_inner AS i.*USE SORTER FOR GROUP BY } # json_extract runs before max. A grouped table would also run it for the @@ -1311,6 +1559,148 @@ expect { 2|0|0|SEARCH i USING COVERING INDEX ephemeral_limit_inner_t3 (k=? AND x=?) } +# Semi-joins and anti-joins must apply the full comparison before they record a match. +@setup correlated_rows +test exists-keeps-correlated-comparison { + SELECT 'exists', o.id + FROM outer_rows AS o + WHERE EXISTS ( + SELECT 1 + FROM inner_rows AS i + WHERE i.key1 = o.key1 + AND i.amount <> o.amount + ) + ORDER BY o.id; + + SELECT 'not exists', o.id + FROM outer_rows AS o + WHERE NOT EXISTS ( + SELECT 1 + FROM inner_rows AS i + WHERE i.key1 = o.key1 + AND i.amount <> o.amount + ) + ORDER BY o.id; +} +expect { + exists|1 + exists|2 + exists|3 + not exists|4 + not exists|5 + not exists|6 +} + +# NULL-aware comparisons must keep their original EXISTS rules after the rewrite. +@setup correlated_rows +test exists-keeps-null-aware-correlated-comparisons { + SELECT 'exists', o.id + FROM outer_rows AS o + WHERE EXISTS ( + SELECT 1 + FROM inner_rows AS i + WHERE i.key1 IS o.key1 + AND i.amount IS o.amount + ) + ORDER BY o.id; + + SELECT 'not exists', o.id + FROM outer_rows AS o + WHERE NOT EXISTS ( + SELECT 1 + FROM inner_rows AS i + WHERE i.key1 IS o.key1 + AND i.amount IS NOT o.amount + ) + ORDER BY o.id; +} +expect { + exists|3 + exists|6 + not exists|4 +} + +@setup correlated_rows +@var polarity { EXISTS | NOT EXISTS } +@var link { i.key1 = o.key1 | i.key1 IS o.key1 | i.key1 < o.key1 } +@var comparison { + i.amount = o.amount | o.amount = i.amount + | i.amount <> o.amount | o.amount <> i.amount + | i.amount < o.amount | o.amount < i.amount + | i.amount <= o.amount | o.amount <= i.amount + | i.amount > o.amount | o.amount > i.amount + | i.amount >= o.amount | o.amount >= i.amount + | i.amount IS o.amount | o.amount IS i.amount + | i.amount IS NOT o.amount | o.amount IS NOT i.amount +} +matrix correlated-comparison-results { + SELECT o.id + FROM outer_rows AS o + WHERE $polarity ( + SELECT 1 FROM inner_rows AS i + WHERE $link AND $comparison + ) + ORDER BY o.id; +} + +@setup correlated_rows +@var join_type { LEFT | RIGHT } +@var polarity { EXISTS | NOT EXISTS } +@var comparison { i.amount < k.key2 | i.amount IS NOT k.key2 } +matrix correlated-comparison-after-outer-join { + SELECT o.id, k.id + FROM outer_rows AS o $join_type JOIN outer_key2 AS k ON o.id = k.id + WHERE $polarity ( + SELECT 1 FROM inner_rows AS i WHERE $comparison + ) + ORDER BY o.id, k.id; +} + +@setup correlated_rows +@var polarity { EXISTS | NOT EXISTS } +@var comparison { s.amount < i.amount | s.amount IS NOT i.amount } +matrix correlated-comparison-after-full-join { + SELECT o.id, k.id, i.key1, i.amount + FROM outer_rows AS o + JOIN outer_key2 AS k ON o.id = k.id + FULL JOIN inner_rows AS i ON k.key2 = i.key2 + WHERE $polarity ( + SELECT 1 FROM outer_rows AS s WHERE $comparison + ) + ORDER BY o.id, k.id, i.key1, i.amount; +} + +# Both forms must use a join instead of a subquery call for each row. +@setup correlated_rows +@skip-if sqlite "query plan explanation output is different in sqlite" +test correlated-comparison-uses-joins { + EXPLAIN QUERY PLAN + SELECT o.id + FROM outer_rows AS o + WHERE EXISTS ( + SELECT 1 + FROM inner_rows AS i + WHERE i.key1 = o.key1 + AND i.amount <> o.amount + ); + + EXPLAIN QUERY PLAN + SELECT o.id + FROM outer_rows AS o + WHERE NOT EXISTS ( + SELECT 1 + FROM inner_rows AS i + WHERE i.key1 = o.key1 + AND i.amount <> o.amount + ); +} +expect { + 1|0|0|SCAN outer_rows AS o + 2|0|0|SEARCH i USING COVERING INDEX ephemeral_inner_rows_t3 (key1=?) + 1|0|0|HASH JOIN inner_rows AS i + 2|0|0|SCAN outer_rows AS o +} + # Each EXISTS keeps its own outer row. A second one must not make both use # nested scans. @skip-if sqlite "query plan explanation output is different in sqlite" @@ -1355,10 +1745,223 @@ expect { 2|0|0|SEARCH i USING COVERING INDEX ephemeral_filtered_exists_inner_t3 (key1=?) } +setup hash_anti_rows { + CREATE TABLE hash_anti_outer( + id INTEGER PRIMARY KEY, + key_value INTEGER, + active INTEGER + ); + CREATE TABLE hash_anti_inner(key_value INTEGER, value INTEGER); + INSERT INTO hash_anti_outer VALUES + (1,1,1), + (2,2,1), + (3,3,0), + (4,NULL,1), + (5,4,1), + (6,1,1), + (7,5,1), + (8,6,1); + INSERT INTO hash_anti_inner VALUES + (1,1), + (2,0), + (NULL,1), + (5,1), + (5,1), + (6,0), + (6,1); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('hash_anti_outer', NULL, '100000'), + ('hash_anti_inner', NULL, '1000000'); + ANALYZE sqlite_schema; +} + +# A large anti-join can scan its inner table once without a temporary index. +@setup hash_anti_rows +@skip-if sqlite "SQLite uses a different NOT EXISTS plan" +test not-exists-can-use-hash-anti-join { + EXPLAIN QUERY PLAN + SELECT o.id + FROM hash_anti_outer o + WHERE o.active = 1 + AND NOT EXISTS ( + SELECT 1 + FROM hash_anti_inner i + WHERE i.key_value = o.key_value + AND i.value > 0 + ); +} +expect pattern { + (?s)HASH JOIN hash_anti_inner AS i.*SCAN hash_anti_outer AS o +} + +# A hash anti-join keeps only build rows that have no complete inner match. +@setup hash_anti_rows +test hash-anti-join-keeps-unmatched-build-rows { + SELECT o.id + FROM hash_anti_outer o + WHERE o.active = 1 + AND NOT EXISTS ( + SELECT 1 + FROM hash_anti_inner i + WHERE i.key_value = o.key_value + AND i.value > 0 + ) + ORDER BY o.id; +} +expect { + 2 + 4 + 5 +} + +@setup hash_anti_rows +test hash-anti-join-keeps-all-build-rows-when-probe-is-empty { + SELECT o.id + FROM hash_anti_outer o + WHERE o.active = 1 + AND NOT EXISTS ( + SELECT 1 + FROM hash_anti_inner i + WHERE i.key_value = o.key_value + AND i.value > 100 + ) + ORDER BY o.id; +} +expect { + 1 + 2 + 4 + 5 + 6 + 7 + 8 +} + +@setup hash_anti_rows +test hash-anti-join-keeps-empty-build-input-empty { + SELECT o.id + FROM hash_anti_outer o + WHERE o.active = 9 + AND NOT EXISTS ( + SELECT 1 + FROM hash_anti_inner i + WHERE i.key_value = o.key_value + ); +} +expect { +} + +setup hash_anti_spill_rows { + CREATE TABLE hash_anti_spill_outer( + id INTEGER PRIMARY KEY, + key_value INTEGER, + active INTEGER, + payload TEXT + ); + CREATE TABLE hash_anti_spill_inner(key_value INTEGER, value INTEGER); + INSERT INTO hash_anti_spill_outer + SELECT value, value, 1, printf('%.*c', 10000, 'x') + FROM generate_series(1, 150); + INSERT INTO hash_anti_spill_inner + SELECT value, 1 + FROM generate_series(76, 225); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('hash_anti_spill_outer', NULL, '100000'), + ('hash_anti_spill_inner', NULL, '1000000'); + ANALYZE sqlite_schema; +} + +# The large payload makes the debug hash table spill to disk. +@setup hash_anti_spill_rows +test hash-anti-join-keeps-unmatched-rows-after-spill { + SELECT count(*), sum(length(o.payload)) + FROM hash_anti_spill_outer o + WHERE o.active = 1 + AND NOT EXISTS ( + SELECT 1 + FROM hash_anti_spill_inner i + WHERE i.key_value = o.key_value + AND i.value > 0 + ); +} +expect { + 75|750000 +} + +setup exists_filter_before_anti_join { + CREATE TABLE exists_filter_outer(id INTEGER PRIMARY KEY, k INTEGER, flag INTEGER); + CREATE TABLE exists_filter_probe(k INTEGER, v INTEGER); + CREATE TABLE exists_filter_anti(k INTEGER); + INSERT INTO exists_filter_outer VALUES (1,10,1),(2,20,1),(3,30,1),(4,40,1); + INSERT INTO exists_filter_probe VALUES (10, 99); + INSERT INTO exists_filter_anti VALUES (20),(20),(40); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('exists_filter_outer', NULL, '100000'), + ('exists_filter_anti', NULL, '1000000'), + ('exists_filter_probe', NULL, '50'); + ANALYZE sqlite_schema; +} + +# The correlated EXISTS filter must still apply when the optimizer moves its +# evaluation to a later loop and that loop is a NOT EXISTS anti join. Inside an +# anti join loop, a failed filter looks like "no match", which emits the outer +# row instead of rejecting it. +@setup exists_filter_before_anti_join +test exists-filter-applies-when-moved-to-an-anti-join-loop { + SELECT o.id FROM exists_filter_outer o + WHERE EXISTS ( + SELECT 1 FROM exists_filter_probe e + WHERE e.k = o.k AND e.v <> o.id + ) + AND o.flag = 1 + AND NOT EXISTS ( + SELECT 1 FROM exists_filter_anti n + WHERE n.k = o.k + ) + ORDER BY o.id; +} +expect { + 1 +} + +# Same shape with an index on the anti-join key, so the anti join seeks a real +# index instead of building an ephemeral one. +@setup exists_filter_before_anti_join +test exists-filter-applies-when-moved-to-an-indexed-anti-join-loop { + CREATE INDEX exists_filter_anti_k ON exists_filter_anti(k); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl, idx, stat) VALUES + ('exists_filter_outer', NULL, '100000'), + ('exists_filter_anti', 'exists_filter_anti_k', '100 1'), + ('exists_filter_probe', NULL, '50'); + ANALYZE sqlite_schema; + + SELECT o.id FROM exists_filter_outer o + WHERE EXISTS ( + SELECT 1 FROM exists_filter_probe e + WHERE e.k = o.k AND e.v <> o.id + ) + AND o.flag = 1 + AND NOT EXISTS ( + SELECT 1 FROM exists_filter_anti n + WHERE n.k = o.k + ) + ORDER BY o.id; +} +expect { + 1 +} + # With table counts from ANALYZE, one outer row should use the saved inner # index. All outer rows should group the inner table once. @skip-if sqlite "query plan explanation output is different in sqlite" -@skip-if mvcc "MVCC does not reload sqlite_stat1 after this test changes it" test indexed-average-uses-outer-row-count { CREATE TABLE one_indexed_avg_outer(id INTEGER PRIMARY KEY, k INTEGER, x REAL); CREATE TABLE many_indexed_avg_outer(id INTEGER PRIMARY KEY, k INTEGER, x REAL); @@ -1710,6 +2313,37 @@ setup comparison_keys { INSERT INTO nocase_inner VALUES ('a', 10), ('A', 20); } +setup comparison_key_indexes { + CREATE INDEX text_inner_key_amount ON text_inner(key1, amount); + CREATE INDEX binary_inner_key_amount ON binary_inner(key1, amount); + CREATE INDEX nocase_inner_key_amount ON nocase_inner(key1, amount); +} + +@setup comparison_keys +@setup comparison_key_indexes +@var comparison { i.amount > o.id | o.id < i.amount | i.amount IS NOT o.id } +matrix correlated-in-inequality-keeps-number-to-text-comparison { + SELECT o.id + FROM number_outer AS o + WHERE o.key1 IN ( + SELECT i.key1 FROM text_inner AS i WHERE $comparison + ) + ORDER BY o.id; +} + +@setup comparison_keys +@setup comparison_key_indexes +@var inner_table { binary_inner | nocase_inner } +@var comparison { i.amount > o.id | o.id < i.amount | i.amount IS NOT o.id } +matrix correlated-in-inequality-keeps-text-order { + SELECT o.id + FROM nocase_outer AS o + WHERE o.key1 IN ( + SELECT i.key1 FROM $inner_table AS i WHERE $comparison + ) + ORDER BY o.id; +} + @setup comparison_keys test one-value-number-and-text-stay-correlated { SELECT o.id, (SELECT sum(i.amount) FROM text_inner i WHERE i.key1 = o.key1) @@ -1789,6 +2423,123 @@ expect error { .*malformed JSON.* } +setup correlated_comparison_errors { + CREATE TABLE error_outer(id INTEGER, wanted INTEGER, limit_value INTEGER, text_value TEXT); + CREATE TABLE error_inner(value INTEGER, json_value TEXT); + INSERT INTO error_outer VALUES (1, 1, 10, '1'), (2, 1, 0, 'bad JSON'); + INSERT INTO error_inner VALUES (1, '1'), (2, 'bad JSON'); +} + +@setup correlated_comparison_errors +@var polarity { EXISTS | NOT EXISTS } +@var comparison { + json_extract(i.json_value, '$') < o.limit_value + | o.limit_value > json_extract(i.json_value, '$') + | i.value < json_extract(o.text_value, '$') +} +@var other_term { | AND i.value = o.wanted } +matrix correlated-exists-inequality-expression-errors { + SELECT o.id + FROM error_outer AS o + WHERE $polarity ( + SELECT 1 FROM error_inner AS i WHERE $comparison $other_term + ) + ORDER BY o.id; +} + +@setup correlated_comparison_errors +@var left_value { (SELECT json_extract(o.text_value, '$')) | (SELECT o.wanted) } +@var comparison { i.value < o.limit_value | i.value >= o.limit_value } +matrix correlated-in-with-left-subquery { + SELECT o.id + FROM error_outer AS o + WHERE $left_value IN ( + SELECT i.value FROM error_inner AS i WHERE $comparison + ) + ORDER BY o.id; +} + +@setup correlated_comparison_errors +@var comparison { i.value < o.limit_value | o.limit_value > i.value } +matrix correlated-in-nested-result-error { + SELECT o.id + FROM error_outer AS o + WHERE o.wanted IN ( + SELECT CASE WHEN i.value = 1 THEN 1 + ELSE (SELECT json_extract('bad JSON', '$')) END + FROM error_inner AS i + WHERE $comparison + ) + ORDER BY o.id; +} + +@setup correlated_comparison_errors +test correlated-in-inequality-keeps-inner-filter-error { + SELECT o.id + FROM error_outer AS o + WHERE o.wanted IN ( + SELECT i.value FROM error_inner AS i + WHERE json_extract(i.json_value, '$') < o.limit_value + ); +} +expect error { + .*malformed JSON.* +} + +@setup correlated_comparison_errors +test correlated-in-inequality-keeps-result-error { + SELECT o.id + FROM error_outer AS o + WHERE o.wanted IN ( + SELECT json_extract(i.json_value, '$') FROM error_inner AS i + WHERE i.value < o.limit_value + ); +} +expect error { + .*malformed JSON.* +} + +@setup correlated_comparison_errors +test correlated-in-inequality-keeps-left-error { + SELECT o.id + FROM error_outer AS o + WHERE json_extract(o.text_value, '$') IN ( + SELECT i.value FROM error_inner AS i + WHERE i.value < o.limit_value + ); +} +expect error { + .*malformed JSON.* +} + +@setup correlated_comparison_errors +@skip-if sqlite "query plan explanation output is different in sqlite" +test correlated-in-error-expressions-stay-subqueries { + EXPLAIN QUERY PLAN + SELECT o.id FROM error_outer AS o + WHERE o.wanted IN ( + SELECT i.value FROM error_inner AS i + WHERE json_extract(i.json_value, '$') < o.limit_value + ); + + EXPLAIN QUERY PLAN + SELECT o.id FROM error_outer AS o + WHERE o.wanted IN ( + SELECT json_extract(i.json_value, '$') FROM error_inner AS i + WHERE i.value < o.limit_value + ); + + EXPLAIN QUERY PLAN + SELECT o.id FROM error_outer AS o + WHERE json_extract(o.text_value, '$') IN ( + SELECT i.value FROM error_inner AS i + WHERE i.value < o.limit_value + ); +} +expect pattern { + (?s)CORRELATED LIST SUBQUERY.*CORRELATED LIST SUBQUERY.*CORRELATED LIST SUBQUERY +} + # These cases run every generated query against Turso and SQLite. The data has # repeated outer keys, repeated inner keys, NULL values, missing groups, and a # NULL outer key. @@ -1812,6 +2563,27 @@ matrix one-value-aggregate-comparisons { ORDER BY o.id; } +# Check that automatic-index prefiltering preserves results for equality and +# range conditions, reversed operands, NULLs, and one- or two-column keys. +@setup correlated_rows +@var comparison { = | < | <= | > | >= } +@var link { i.key1 = o.key1 | i.key1 = o.key1 AND i.key2 = o.key2 } +@var inner_filter { AND i.amount = 7 | AND i.amount > 6 | AND i.amount <= 20 | AND i.amount >= 6 AND i.amount < 20 | AND 20 >= i.amount AND 6 <= i.amount | AND i.amount IS NULL } +matrix join-first-autoindex-prefilter { + SELECT candidate.id + FROM outer_rows candidate + WHERE candidate.id IN ( + SELECT o.id + FROM outer_rows o + WHERE o.amount $comparison ( + SELECT sum(i.amount) + FROM inner_rows i + WHERE $link $inner_filter + ) + ) + ORDER BY candidate.id; +} + @setup correlated_rows @var aggregate { avg(i.amount) | min(i.amount) | max(i.amount) | count(*) | count(i.amount) | total(i.amount) } matrix two-copies-of-one-aggregate { diff --git a/conformance/sqlite-sqltests/upsert.sqltest b/conformance/sqlite-sqltests/upsert.sqltest index dda2df82..c6837227 100644 --- a/conformance/sqlite-sqltests/upsert.sqltest +++ b/conformance/sqlite-sqltests/upsert.sqltest @@ -886,3 +886,46 @@ test upsert-conflict-target-nulls-first-rejected { expect error { NULLS } + +@cross-check-integrity +test upsert-before-update-trigger-index-delete { + CREATE TABLE t(a INTEGER PRIMARY KEY, c INTEGER); + CREATE INDEX i ON t(c); + INSERT INTO t VALUES(1,1); + CREATE TRIGGER tg BEFORE UPDATE ON t BEGIN + UPDATE t SET c=c+1 WHERE a=NEW.a; + END; + INSERT INTO t VALUES(1,9) ON CONFLICT(a) DO UPDATE SET c=c+5; + SELECT count(*), group_concat(c) FROM (SELECT c FROM t WHERE c>0 ORDER BY c); + PRAGMA integrity_check; + DELETE FROM t WHERE a=1; + SELECT count(*) FROM t WHERE c>0; + PRAGMA integrity_check; +} +expect { + 1|6 + ok + 0 + ok +} + +@cross-check-integrity +test upsert-before-update-trigger-changes-non-set-column { + -- The trigger changes c, which is not in the SET list. The final row + -- must keep the trigger's c value; only b gets the SET value. + CREATE TABLE t(a INTEGER PRIMARY KEY, b INTEGER, c INTEGER); + CREATE INDEX i ON t(c); + INSERT INTO t VALUES(1,1,1); + CREATE TRIGGER tg BEFORE UPDATE ON t BEGIN + UPDATE t SET c=c+10 WHERE a=NEW.a; + END; + INSERT INTO t VALUES(1,9,9) ON CONFLICT(a) DO UPDATE SET b=b+5; + SELECT a, b, c FROM t; + SELECT c FROM t WHERE c>0 ORDER BY c; + PRAGMA integrity_check; +} +expect { + 1|6|11 + 11 + ok +} diff --git a/conformance/sqlite-sqltests/vacuum.sqltest b/conformance/sqlite-sqltests/vacuum.sqltest new file mode 100644 index 00000000..fe206d1e --- /dev/null +++ b/conformance/sqlite-sqltests/vacuum.sqltest @@ -0,0 +1,69 @@ +@database :temp: + +@skip-if mvcc "plain VACUUM requires PRAGMA wal_checkpoint(TRUNCATE) first on an MVCC database" +test vacuum-basic { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c REAL); + INSERT INTO t1 VALUES(1, 'hello', 3.125); + INSERT INTO t1 VALUES(2, 'world', 2.725); + INSERT INTO t1 VALUES(3, 'test', 1.625); + DELETE FROM t1 WHERE a = 2; + VACUUM; + PRAGMA integrity_check; + SELECT a, b, c FROM t1 ORDER BY a; +} +expect { + ok + 1|hello|3.125 + 3|test|1.625 +} + +@skip-if mvcc "plain VACUUM requires PRAGMA wal_checkpoint(TRUNCATE) first on an MVCC database" +test vacuum-preserves-metadata { + CREATE TABLE t(a INTEGER); + INSERT INTO t VALUES(1); + PRAGMA user_version = 42; + PRAGMA application_id = 99; + PRAGMA schema_version; + VACUUM; + PRAGMA integrity_check; + PRAGMA user_version; + PRAGMA application_id; + PRAGMA schema_version; +} +expect { + 1 + ok + 42 + 99 + 2 +} + +@skip-if mvcc "plain VACUUM requires PRAGMA wal_checkpoint(TRUNCATE) first on an MVCC database" +test vacuum-preserves-autoincrement { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b TEXT); + INSERT INTO t1(b) VALUES('one'); + INSERT INTO t1(b) VALUES('two'); + INSERT INTO t1(b) VALUES('three'); + DELETE FROM t1 WHERE b = 'two'; + VACUUM; + PRAGMA integrity_check; + SELECT name, seq FROM sqlite_sequence; + INSERT INTO t1(b) VALUES('four'); + SELECT a, b FROM t1 ORDER BY a; +} +expect { + ok + t1|3 + 1|one + 3|three + 4|four +} + +test vacuum-rejects-active-transaction { + CREATE TABLE t(a INTEGER); + BEGIN; + VACUUM; +} +expect error { + cannot VACUUM from within a transaction +} diff --git a/docs/turso-gap-inventory.json b/docs/turso-gap-inventory.json index dd60dceb..5c5dcc6e 100644 --- a/docs/turso-gap-inventory.json +++ b/docs/turso-gap-inventory.json @@ -14,8 +14,8 @@ "commit_at_analysis": "7b62a3f" }, "conformance_ground_truth": "src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt (606 failure lines at analysis time)", - "current_conformance_expected_failures": 78, - "current_conformance_expected_failures_note": "Integrated work closes the original SQL/window/CTE correctness groups, 24 new SQL-extension, 14 generated-column/reserved-name, three legacy-view, safe ATTACH routing/CDC/sequence behavior, and the cycling-sequence commit failure. Non-cycling watermarks are repaired; its private row-count representation remains different. Multiple memory-only databases can commit atomically; any physical database write must be the transaction's sole database mutation. The planner and Plan/EXPLAIN parity work closes 31 additional conformance markers, including JSON EQP, grouped-derived probes, and correlated-IN automatic-index cursor allocation. Other assigned gaps and intentional feature/representation/eager-corruption differences remain. Executing the previously CLI-gated turso/fts.sqltest corpus adds five markers, all the CLI \"Parse error:\" prefix on otherwise identical MATCH-context errors. These are not 78 independent bugs. The remaining-gap plan tracks current work; the 217 closed records below are historical dispositions.", + "current_conformance_expected_failures": 191, + "current_conformance_expected_failures_note": "Integrated work closes the original SQL/window/CTE correctness groups, 24 new SQL-extension, 14 generated-column/reserved-name, three legacy-view, safe ATTACH routing/CDC/sequence behavior, and the cycling-sequence commit failure. Non-cycling watermarks are repaired; its private row-count representation remains different. Multiple memory-only databases can commit atomically; any physical database write must be the transaction's sole database mutation. The planner and Plan/EXPLAIN parity work closes 31 additional conformance markers, including JSON EQP, grouped-derived probes, and correlated-IN automatic-index cursor allocation. Other assigned gaps and intentional feature/representation/eager-corruption differences remain. Executing the previously CLI-gated turso/fts.sqltest corpus adds five markers, all the CLI \"Parse error:\" prefix on otherwise identical MATCH-context errors. Refreshing the corpus to the pinned v0.8.1 content (33 new and 41 refreshed files) adds 113 markers pending closure. These are not 191 independent bugs. The remaining-gap plan tracks current work; the 217 closed records below are historical dispositions.", "current_execution_plan": "docs/turso-remaining-gap-plan.md", "last_reconciled": "2026-09-23", "entry_count": 217, diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index c8fe8fdd..3d47aa06 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -159,3 +159,120 @@ turso/mvcc_sequence_adversarial.sqltest::mvcc-user-sequence-survives-concurrent- turso/mvcc_sequence.sqltest::mvcc-to-wal-autoincrement-continues | intentional storage-mode policy: a :memory: WAL request retains mvcc mode; committed AUTOINCREMENT values still advance correctly turso/mvcc_sequence.sqltest::mvcc-to-wal-user-sequence-continues | intentional storage-mode policy: a :memory: WAL request retains mvcc mode; committed user-sequence values still advance correctly turso/mvcc_sequence.sqltest::mvcc-wal-mvcc-roundtrip-autoincrement | intentional storage-mode policy: a :memory: WAL request retains mvcc mode without losing allocation watermarks + +# v0.8.1 corpus adoption (2026-09-30): the vendored corpus was refreshed from the +# pinned turso-src v0.8.1 (8549c1659): 33 new files and 41 refreshed files. The +# entries below are the differences that refresh exposed, pending closure. +agg-functions/json-group-object-null-label.sqltest::json_group_object_null_label_only | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{}/{}/{"a":null}/[null]' but was '{"":1}/{"":1}/{"a":null}/[null]' +agg-functions/json-group-object-null-label.sqltest::json_group_object_skips_null_label | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"a":1,"b":3}' but was '{"a":1,"":2,"b":3}' +agg-functions/json-group-object-null-label.sqltest::jsonb_group_object_skips_null_label | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"a":1,"b":3}' but was '{"a":1,"":2,"b":3}' +alter_table.sqltest::alter-table-add-column-preserve-schema-collate-locale | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected 'CREATE TABLE tbl (c TEXT COLLATE "fr-FR", d)' but was 'CREATE TABLE tbl (c TEXT COLLATE 'fr-FR', d)' +alter_table.sqltest::fail-alter-table-rename-to-existing-view | database variant 1/1 (:memory:): error 'there is already a view named y' does not match pattern 'there is already another table or index with this name: y' +alter_table.sqltest::fail-alter-table-rename-to-existing-view-different-case | database variant 1/1 (:memory:): error 'there is already a view named Y' does not match pattern 'there is already another table or index with this name: Y' +gencol.sqltest::gencol_issue6161_strict_compute_after_replace_default | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '42/42/integer' but was '42//null' +gencol.sqltest::gencol_issue7058_alter_add_default_on_gencol | database variant 1/1 (:memory:): error 'cannot use DEFAULT on a generated column' does not match pattern 'error in generated column "b"' +gencol.sqltest::gencol_issue7059_unknown_type_token_rejected | database variant 1/1 (:memory:): error 'Unsupported column constraint 'WAT'. At SQL offset 27.' does not match pattern 'error in generated column "b"' +gencol.sqltest::gencol_issue7060_multiple_generated_clauses_rejected | database variant 1/1 (:memory:): expected an error but the statement succeeded +gencol.sqltest::gencol_issue7068_check_sees_recomputed_value | database variant 1/1 (:memory:): expected success but got error: CHECK constraint failed: b IS 6 +gencol.sqltest::gencol_issue7068_multiple_defaults_before_generated_notnull | database variant 1/1 (:memory:): expected success but got error: NOT NULL constraint failed: t.b +gencol.sqltest::gencol_issue7068_returning_recomputed_after_default | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '5/6' but was '5/' +gencol.sqltest::gencol_issue7068_unique_index_uses_recomputed_value | database variant 1/1 (:memory:): expected 2 row(s), got 2 row(s); row 0 expected '1/5/6' but was '1/5/' +generate_series.sqltest::generate-series-null-arguments-return-no-rows | database variant 1/1 (:memory:): expected success but got error: generate_series() requires integer arguments +integrity_check/memory.sqltest::integrity-check-qualified-corrupt-attached-database | database variant 1/1 (:memory:): expected success but got error: Managed ATTACH requires a file-backed managed primary database so attachments share its file system. +join/hash.sqltest::hash-join-result-matrix-full-join-uses-a-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN matrix_right AS r' +join/hash.sqltest::hash-join-result-matrix-left-join-uses-a-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN matrix_right AS r' +join/hash.sqltest::hash-join-result-matrix-uses-a-hash-join | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN matrix_right AS r.*SCAN matrix_left AS l' +join/hash_join_order_by.sqltest::hash-join-keeps-outer-table-order-elision | database variant 1/1 (:memory:): expected 3 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN c' but was '0/0/0/MANAGED EVALUATOR FALLBACK' +join/hash_join_order_by.sqltest::hash-join-materialized-prefix-uses-sorter | database variant 1/1 (:memory:): output does not match pattern '(?s)MATERIALIZE hash build input for a.*HASH JOIN b.*USE SORTER FOR ORDER BY' +join/hash_join_order_by.sqltest::hash-join-order-by-build-table-uses-sorter | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN b.*SCAN a.*USE SORTER FOR ORDER BY' +join/memory.sqltest::equality-prefix-keeps-index-search | database variant 1/1 (:memory:): output does not match pattern 'SEARCH child USING COVERING INDEX sqlite_autoindex_prefix_child_1 \(group_id=\? AND parent_id=\?\)' +join/memory.sqltest::expression-index-after-earlier-join-reads-source-table | database variant 1/1 (:memory:): output does not match pattern '(?s)MATERIALIZE hash build input for expression_build_orders AS orders.*SCAN expression_build_customers AS customers.*SEARCH orders USIN +join/memory.sqltest::expression-index-hash-build-reads-source-table | database variant 1/1 (:memory:): output does not match pattern '(?s)MATERIALIZE hash build input for expression_build_orders AS orders.*HASH JOIN expression_build_inventory AS inventory.*SEARCH orders +join/memory.sqltest::future-table-cannot-make-index-usable | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 121. +join/memory.sqltest::future-table-index-does-not-change-join-results | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 133. +join/memory.sqltest::group-sort-cost-keeps-cheaper-correlated-seeks | database variant 1/1 (:memory:): output does not match pattern '(?s)CORRELATED SCALAR SUBQUERY.*SEARCH child USING INDEX sqlite_autoindex_sort_cost_child_1 \(parent_id=\?\)' +join/memory.sqltest::hash-build-row-limit-counts-prefix-rows | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 155. +join/memory.sqltest::hash-join-cost-includes-rows-visited | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN hash_cost_right".*"access_cost":300110000' +join/memory.sqltest::index-suffix-does-not-prevent-hash-join | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN suffix_hash_child AS child.*SCAN suffix_hash_parent AS parent' +join/memory.sqltest::indexed-build-hashes-an-unfiltered-large-probe | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN indexed_hash_child AS child.*SCAN indexed_hash_parent AS parent' +join/memory.sqltest::materialized-hash-build-keeps-index-search | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"MATERIALIZE hash build input for selected_hash_build AS build_rows".*"detail":"SEARCH build_rows USING INDEX selected_hash +join/memory.sqltest::no-stats-hash-join-keeps-unindexed-probe-fanout | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 79. +join/memory.sqltest::partial-stats-hash-join-uses-known-unique-key-count | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 79. +join/memory.sqltest::range-prefix-still-allows-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN prefix_child AS child' +join/memory.sqltest::unique-composite-hash-key-uses-one-key-estimate | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN composite_hash_child AS child".*"rows_per_input":5' +join/memory.sqltest::unique-hash-key-does-not-apply-to-a-different-collation | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 127. +json/default.sqltest::jsonb_null_is_sql_null | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected 'null/1' but was 'blob/0' +json/json_tree_columns.sqltest::json-tree-rowid-into-table | database variant 1/1 (:memory:): expected success but got error: no such column: rt.rowid +json/json_tree_columns.sqltest::json-tree-rowid-is-selectable | database variant 1/1 (:memory:): expected success but got error: no such column: rt.rowid +json/json_tree_columns.sqltest::json-tree-rowid-unqualified | database variant 1/1 (:memory:): expected success but got error: no such column: rowid +multi_index_or_compound.sqltest::compound-or-keeps-correlated-index-columns-together | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '1/0/0/MULTI-INDEX OR t (txy, txy)' but was '0/0/0/MANAGED EVALUATOR FALLBACK' +multi_index_or_compound.sqltest::derived-or-filter-skips-correlated-columns | database variant 1/1 (:memory:): output does not match pattern '"rows_per_input":1000' +multi_index_or_compound.sqltest::implied-or-filter-accepts-negative-numbers | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' +multi_index_or_compound.sqltest::implied-or-filter-accepts-parenthesized-equalities | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' +multi_index_or_compound.sqltest::implied-or-filter-keeps-compound-index-searches | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' +multi_index_or_compound.sqltest::implied-or-filter-uses-only-indexed-column | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' +multi_index_or_compound.sqltest::nested-and-branches-use-one-index-search | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '1/0/0/SEARCH t USING INDEX tx (x=?)' but was '0/0/0/MANAGED EVALUATOR FALLBACK' +multi_index_or_compound.sqltest::nested-or-branches-use-one-index-search | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '1/0/0/SEARCH t USING COVERING INDEX tx (x=?)' but was '0/0/0/MANAGED EVALUATOR FALLBACK' +mvcc-analyze-refreshes-planner-stats.sqltest::mvcc-analyze-next-plan-uses-new-row-estimate | database variant 1/1 (:temp:): output does not match pattern '"rows_per_input":100,"output_rows":100,' +mvcc-analyze-refreshes-planner-stats.sqltest::wal-analyze-next-plan-uses-new-row-estimate | database variant 1/1 (:temp:): output does not match pattern '"rows_per_input":100,"output_rows":100,' +partial_idx.sqltest::partial-index-or-right-term-with-range-search | database variant 1/1 (:memory:): expected 2 row(s), got 2 row(s); row 0 expected '1/0/0/SEARCH t USING COVERING INDEX idx (a>?)' but was '1/0/0/SCAN t' +partial_idx.sqltest::partial-index-or-term-with-covering-scan | database variant 1/1 (:memory:): expected 3 row(s), got 3 row(s); row 0 expected '1/0/0/SCAN t USING COVERING INDEX idx' but was '1/0/0/SCAN t' +partial_idx.sqltest::partial-index-or-term-with-equality-search | database variant 1/1 (:memory:): expected 2 row(s), got 2 row(s); row 0 expected '1/0/0/SEARCH t USING INDEX idx (a=?)' but was '1/0/0/SCAN t' +partial_idx.sqltest::partial-index-or-term-with-order | database variant 1/1 (:memory:): expected 3 row(s), got 4 row(s); row 0 expected '1/0/0/SCAN t USING INDEX idx' but was '1/0/0/SCAN t' +pragma/memory.sqltest::pragma-temp-synchronous-stays-off | database variant 1/1 (:memory:): expected 3 row(s), got 3 row(s); row 0 expected '0' but was '2' +scalar-functions-datetime.sqltest::date-blob-modifier | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '2024-06-16' but was '' +scalar-functions-datetime.sqltest::date-blob-time-value | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '2024-01-01' but was '' +scalar-functions-datetime.sqltest::date-unixepoch-at-1-millisecond-before-epochtime | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '1969-12-31' but was '1970-01-01' +scalar-functions-datetime.sqltest::strftime-at-1-millisecond-before-epochtime | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '-1' but was '0' +scalar-functions-datetime.sqltest::strftime-blob-time-value | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '2024-01-01' but was '' +scalar-functions-datetime.sqltest::timediff-blob-time-values | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '+0000-00-01 00:00:00.000' but was '' +scalar-functions-datetime.sqltest::unixepoch-at-1-millisecond-before-epochtime | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '-1' but was '0' +scalar-functions-datetime.sqltest::unixepoch-blob-time-value | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '1704067200' but was '' +scalar-functions.sqltest::min-float-equals-int | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '1' but was '1.0' +scalar-functions.sqltest::min-int-equals-float | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '1.0' but was '1' +simple-count-optimization.sqltest::simple-count-unusable-partial-index-column-correctness | database variant 1/1 (:memory:): expected success but got error: no query solution +sql-error-message-prefix.sqltest::database-full-error-has-no-prefix | database variant 1/1 (:memory:): expected an error but the statement succeeded +turso/alter_column.sqltest::alter-column-rejects-index-only-rewrite-mvcc | database variant 1/1 (:memory:): expected an error but the statement succeeded +turso/alter_column.sqltest::alter-column-rejects-index-rewrite-mvcc | database variant 1/1 (:memory:): expected an error but the statement succeeded +turso/alter_column.sqltest::alter-column-rewrites-expression-index-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' +turso/alter_column.sqltest::alter-column-rewrites-generated-index-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' +turso/alter_column.sqltest::alter-column-rewrites-indexed-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' +turso/alter_column.sqltest::alter-column-rewrites-indexed-regular-to-virtual | database variant 1/1 (:memory:): expected 4 row(s), got 4 row(s); row 0 expected 'wal' but was 'memory' +turso/alter_column.sqltest::alter-column-rewrites-indexed-virtual-expression-change | database variant 1/1 (:memory:): expected 4 row(s), got 4 row(s); row 0 expected 'wal' but was 'memory' +turso/alter_column.sqltest::alter-column-rewrites-indexed-virtual-to-regular | database variant 1/1 (:memory:): expected 4 row(s), got 4 row(s); row 0 expected 'wal' but was 'memory' +turso/alter_column.sqltest::alter-column-rewrites-partial-index-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' +turso/alter_column.sqltest::alter-column-rewrites-partial-index-virtual-expression-change | database variant 1/1 (:memory:): expected 4 row(s), got 4 row(s); row 0 expected 'wal' but was 'memory' +turso/explain-query-plan-json.sqltest::a-compound-select-lists-its-arms | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT name FROM users WHERE age > 30\n UNION\n SELECT n +turso/explain-query-plan-json.sqltest::a-from-clause-subquery-nests-its-body-under-its-own-scan | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT city_id, avg(age)\n FROM (SELECT * FROM users WHERE +turso/explain-query-plan-json.sqltest::a-recursive-cte-reports-its-setup-and-step-phases | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH RECURSIVE cnt(x) AS (\n SELECT 1\n UNION AL +turso/explain-query-plan-json.sqltest::an-or-over-two-indexes-lists-both-branches | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users\n WHERE id = 5 OR age = 30;","resu +turso/explain-query-plan-json.sqltest::both-readers-of-a-shared-cte-point-at-one-materialization | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH spenders AS (\n SELECT user_id, sum(amount) AS tot +turso/explain-query-plan-json.sqltest::exists-is-reported-as-a-semi-join | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users u\n WHERE EXISTS (SELECT 1 FROM or +turso/explain-query-plan-json.sqltest::list-and-scalar-subqueries-report-whether-they-are-correlated | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, (SELECT count(*) FROM orders o WHERE o.user_id +turso/explain-query-plan-json.sqltest::not-exists-is-reported-as-an-anti-join | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users u\n WHERE NOT EXISTS (SELECT 1 FRO +turso/explain-query-plan-json.sqltest::quotes-in-the-statement-are-escaped-in-the-echo | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.name = 'a \"quote +turso/explain-query-plan-json.sqltest::search-nodes-name-the-index-the-constraints-and-the-join | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, o.amount\n FROM users u\n LEFT JOIN order +turso/explain-query-plan-json.sqltest::the-json-tree-matches-the-text-rows | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 4 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.age IN (SELECT ag +turso/sequence.sqltest::sequence-backing-table-rejects-mixed-case-insert | database variant 1/1 (:memory:): expected an error but the statement succeeded +turso/sequence.sqltest::sequence-backing-table-rejects-mixed-case-update | database variant 1/1 (:memory:): expected an error but the statement succeeded +turso/sequence.sqltest::sequence-backing-table-rejects-uppercase-insert | database variant 1/1 (:memory:): expected an error but the statement succeeded +turso/sequence.sqltest::sequence-backing-table-rejects-uppercase-update | database variant 1/1 (:memory:): expected an error but the statement succeeded +turso/upsert-target-alias.sqltest::upsert-base-name-is-out-of-scope-when-aliased | database variant 1/1 (:memory:): expected an error but the statement succeeded +turso/upsert-target-alias.sqltest::upsert-target-alias-is-only-target-name | database variant 1/1 (:memory:): expected success but got error: no such column: x.n +turso/upsert-target-alias.sqltest::upsert-target-alias-shadows-excluded | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '10' but was '99' +turso/vacuum.sqltest::vacuum-rejects-query-only | database variant 1/1 (:temp:): error 'attempt to write a readonly database' does not match pattern 'Cannot execute VACUUM in query_only mode' +unnest-correlated.sqltest::correlated-comparison-after-full-join[exists][s-amount-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 6 row(s) +unnest-correlated.sqltest::correlated-comparison-after-full-join[exists][s-amount-is-not-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 7 row(s) +unnest-correlated.sqltest::correlated-comparison-after-full-join[not-exists][s-amount-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 1 row(s) +unnest-correlated.sqltest::correlated-comparison-after-full-join[not-exists][s-amount-is-not-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 0 row(s) +unnest-correlated.sqltest::correlated-comparison-uses-joins | database variant 1/1 (:memory:): expected 4 row(s), got 2 row(s); row 0 expected '1/0/0/SCAN outer_rows AS o' but was '0/0/0/MANAGED EVALUATOR FALLBACK' +unnest-correlated.sqltest::correlated-exists-inequality-expression-errors[not-exists][json-extract-i-json-value-o-limit-value][and-i-value-o-wanted] | database variant 1/1 (:memory:): SQLite errored ('SQLite Error 1: 'malformed JSON'.') but managed returned 1 row(s) +unnest-correlated.sqltest::correlated-exists-inequality-expression-errors[not-exists][o-limit-value-json-extract-i-json-value][and-i-value-o-wanted] | database variant 1/1 (:memory:): SQLite errored ('SQLite Error 1: 'malformed JSON'.') but managed returned 1 row(s) +unnest-correlated.sqltest::correlated-in-after-full-join[i-amount-s-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 0 row(s) +unnest-correlated.sqltest::correlated-in-after-full-join[s-amount-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 0 row(s) +unnest-correlated.sqltest::correlated-in-after-full-join[s-amount-is-not-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 0 row(s) +unnest-correlated.sqltest::correlated-in-error-expressions-stay-subqueries | database variant 1/1 (:memory:): output does not match pattern '(?s)CORRELATED LIST SUBQUERY.*CORRELATED LIST SUBQUERY.*CORRELATED LIST SUBQUERY' +unnest-correlated.sqltest::correlated-in-inequality-uses-indexed-semi-join | database variant 1/1 (:memory:): output does not match pattern '(?s)SCAN outer_rows AS o.*SEARCH i USING COVERING INDEX inner_key_amount' +unnest-correlated.sqltest::correlated-in-with-only-inequality-uses-indexed-semi-join | database variant 1/1 (:memory:): output does not match pattern '(?s)SCAN outer_rows AS o.*SEARCH i USING COVERING INDEX inner_key_amount \(key1<\?\)' +unnest-correlated.sqltest::not-exists-can-use-hash-anti-join | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN hash_anti_inner AS i.*SCAN hash_anti_outer AS o' +upsert.sqltest::upsert-before-update-trigger-changes-non-set-column | database variant 1/1 (:memory:): expected 3 row(s), got 3 row(s); row 0 expected '1/6/11' but was '1/6/1' From 95049126d72a60e36092c7da31bd51d7a37ff4e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 14:51:31 -0400 Subject: [PATCH 02/20] Reject stale MVCC writes over committed peers and keep explicit rowids A BEGIN CONCURRENT transaction could overwrite a peer's update committed after its snapshot, or resurrect a row the peer deleted: base-row deletion markers are separate tombstone versions, invisible to the older snapshot, and the no-visible-version path only checked in-flight peers. Treat any begin/end stamp committed after the writer's snapshot as a write-write conflict (Turso 0f7f30eac). Concurrent inserts also promoted every rowid through the store-global allocator, silently rewriting explicit rowids and INTEGER PRIMARY KEY values after a peer delete. Only rowids the statement allocated itself are promoted now; an explicit key is kept and a clash with a concurrent writer's live version is a write-write conflict. Co-Authored-By: Claude Opus 5.5 --- src/Ahtola.Core/EmbeddedDatabase.Triggers.cs | 1 + src/Ahtola.Core/EmbeddedDatabase.cs | 37 ++++- src/Ahtola.Core/Mvcc/MvStore.cs | 72 +++++++-- .../MvccSelectDualCursorRoutingTests.cs | 150 ++++++++++++++++++ 4 files changed, 241 insertions(+), 19 deletions(-) diff --git a/src/Ahtola.Core/EmbeddedDatabase.Triggers.cs b/src/Ahtola.Core/EmbeddedDatabase.Triggers.cs index 622e7016..fa04059d 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.Triggers.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.Triggers.cs @@ -2024,6 +2024,7 @@ private long FinalizeAutomaticRowId( ? plan.AnyRow ? NextAutoRowId(plan.LargestRowId, plan.Used) : 1 : plan.AutoIncrement.NextRowId(plan.AnyRow, plan.LargestRowId); plan.Used.Add(rowId); + context.ConcurrentMvccIdentityTracker?.MarkAutomaticRowId(tableName, rowId); plan.AnyRow = true; if (rowId > plan.LargestRowId) plan.LargestRowId = rowId; diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index b1e16831..cf168bc2 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -1000,10 +1000,16 @@ internal void ReportRowChange( break; var tableId = store.GetOrCreateTableId(txId, tableName); - // Concurrent catalogs may both pick the same local rowid; promote - // to a store-global id so first-committer-wins does not collapse - // two inserts (Turso process-wide allocator). - var allocated = store.AllocateRowId(tableId, minimumExclusive: rowId - 1); + // Concurrent catalogs may both pick the same local automatic rowid; + // promote it to a store-global id so first-committer-wins does not + // collapse two inserts (Turso process-wide allocator). An explicit + // rowid or INTEGER PRIMARY KEY value is kept as written: a clash with + // a concurrent writer is a write-write conflict, not a new key. + var automatic = ConcurrentMvccIdentityTracker?.TryConsumeAutomaticRowId(tableName, rowId) + ?? true; + var allocated = automatic + ? store.AllocateRowId(tableId, minimumExclusive: rowId - 1) + : rowId; if (allocated != rowId) { var promoted = table.Rows[index].ToArray(); @@ -1185,6 +1191,28 @@ internal void Remove(string tableName, long rowId) if (rows.Count == 0) _keys.Remove(tableName); } + + private readonly Dictionary> _automaticRowIds = + new(StringComparer.OrdinalIgnoreCase); + + /// + /// Records a rowid the statement allocated itself (no explicit rowid or INTEGER PRIMARY + /// KEY value). Only such rowids may be promoted to the store-global allocator; an + /// explicit key is the caller's data and must be kept as written. + /// + internal void MarkAutomaticRowId(string tableName, long rowId) + { + if (!_automaticRowIds.TryGetValue(tableName, out var rows)) + { + rows = []; + _automaticRowIds.Add(tableName, rows); + } + + rows.Add(rowId); + } + + internal bool TryConsumeAutomaticRowId(string tableName, long rowId) + => _automaticRowIds.TryGetValue(tableName, out var rows) && rows.Remove(rowId); } /// @@ -14417,6 +14445,7 @@ private InsertPlan PrepareInsert( ? plan.AnyRow ? NextAutoRowId(plan.LargestRowId, plan.Used) : 1 : plan.AutoIncrement.NextRowId(plan.AnyRow, plan.LargestRowId); plan.Used.Add(rowid); + context.ConcurrentMvccIdentityTracker?.MarkAutomaticRowId(statement.TableName, rowid); } else if (EmbeddedTable.TryCoerceRowid(rowidSource, out var explicitRowid)) { diff --git a/src/Ahtola.Core/Mvcc/MvStore.cs b/src/Ahtola.Core/Mvcc/MvStore.cs index aaeab3ae..c0a03097 100644 --- a/src/Ahtola.Core/Mvcc/MvStore.cs +++ b/src/Ahtola.Core/Mvcc/MvStore.cs @@ -863,6 +863,10 @@ internal void Insert(MvccTxId txId, MvccRowId rowId, SqlValue[] cells) { ThrowIfTypedKeyInsertConflict(tx, chain); } + else + { + ThrowIfConcurrentLiveInsert(tx, chain); + } chain.Add(version); tx.RecordLogOp(CreateLogUpsert(rowId, version.Cells)); @@ -1520,28 +1524,70 @@ or MvccTransactionState.Preparing /// /// Concurrent pure base tombstones/inserts share End=null, so the end-stamp WW - /// path never fires. Detect peer Active/Preparing begins (or ends) on the chain. + /// path never fires. Detect peer Active/Preparing begins (or ends) on the chain, + /// and peer writes committed after this snapshot: a base row's deletion marker is + /// a separate tombstone version here, so a post-snapshot commit is invisible to + /// yet still owns the row (Turso 0f7f30eac: an older snapshot + /// may read the row, but must not write it again without a conflict). /// private void ThrowIfConcurrentWriterOnRow(MvccTransaction tx, List chain) { foreach (var version in chain) { - if (version.Begin is { IsTimestamp: false, Value: var beginTx } - && beginTx != tx.Id.Value - && IsActiveOrPreparingTx(beginTx)) - { + if (version.Begin is { } begin && IsConcurrentWriteStamp(tx, begin)) throw new EmbeddedWriteWriteConflictException(); - } - if (version.End is { IsTimestamp: false, Value: var endTx } - && endTx != tx.Id.Value - && IsActiveOrPreparingTx(endTx)) - { + if (version.End is { } end && IsConcurrentWriteStamp(tx, end)) + throw new EmbeddedWriteWriteConflictException(); + } + } + + /// + /// Automatic rowids come from the store-global allocator, so two live versions of one + /// integer rowid can only arise from explicit keys. A peer's in-flight or post-snapshot + /// live version already owns the rowid: inserting it again must conflict instead of + /// letting both commits keep the same key. + /// + private void ThrowIfConcurrentLiveInsert(MvccTransaction tx, List chain) + { + foreach (var version in chain) + { + if (version.IsTombstone || version.End is not null) + continue; + if (version.Begin is { } begin && IsConcurrentWriteStamp(tx, begin)) throw new EmbeddedWriteWriteConflictException(); - } } } + /// + /// True when belongs to a peer that is still in flight or + /// that committed after 's snapshot. + /// + private bool IsConcurrentWriteStamp(MvccTransaction tx, MvccStamp stamp) + { + if (stamp.IsTimestamp) + return stamp.Value > tx.BeginTimestamp; + + if (stamp.Value == tx.Id.Value) + return false; + + if (_transactions.TryGetValue(stamp.Value, out var other)) + { + return other.State switch + { + MvccTransactionState.Active or MvccTransactionState.Preparing => true, + MvccTransactionState.Committed => + other.CommitTimestamp is not { } cts || cts > tx.BeginTimestamp, + _ => false, + }; + } + + return _finalizedStates.TryGetValue(stamp.Value, out var finalized) + && finalized == MvccTransactionState.Committed + && _finalizedCommitTimestamps.TryGetValue(stamp.Value, out var finalizedCts) + && finalizedCts > tx.BeginTimestamp; + } + /// /// A rowid allocator makes ordinary concurrent inserts distinct. A typed /// primary key has no such allocator: two live versions for the same key @@ -1566,10 +1612,6 @@ private static void ThrowIfTypedKeyInsertConflict( } } - private bool IsActiveOrPreparingTx(ulong otherTxId) - => _transactions.TryGetValue(otherTxId, out var other) - && other.State is MvccTransactionState.Active or MvccTransactionState.Preparing; - private void ClearExclusive(MvccTxId id) { if (_exclusiveTxId == id.Value) diff --git a/src/Ahtola.Tests/MvccSelectDualCursorRoutingTests.cs b/src/Ahtola.Tests/MvccSelectDualCursorRoutingTests.cs index 90503469..e7076443 100644 --- a/src/Ahtola.Tests/MvccSelectDualCursorRoutingTests.cs +++ b/src/Ahtola.Tests/MvccSelectDualCursorRoutingTests.cs @@ -118,6 +118,156 @@ public void ConcurrentDeleteOfSameBaseRowRaisesWriteWriteConflict() a.ExecuteNonQuery("COMMIT;"); } + // Turso 0f7f30eac: an older snapshot may still read a row a peer changed and + // committed, but writing that row again must raise a write-write conflict instead + // of overwriting the peer (lost update) or resurrecting a deleted row. + [TestCase("UPDATE t SET v = 10 WHERE rowid = 1;", "UPDATE t SET v = 20 WHERE rowid = 1;", true, "10")] + [TestCase("UPDATE t SET v = 10 WHERE rowid = 1;", "DELETE FROM t WHERE rowid = 1;", true, "10")] + [TestCase("DELETE FROM t WHERE rowid = 1;", "UPDATE t SET v = 20 WHERE rowid = 1;", true, "")] + [TestCase("DELETE FROM t WHERE rowid = 1;", "DELETE FROM t WHERE rowid = 1;", true, "")] + [TestCase("UPDATE t SET v = 10 WHERE rowid = 1;", "UPDATE t SET v = 20 WHERE rowid = 1;", false, "10")] + [TestCase("UPDATE t SET v = 10 WHERE rowid = 1;", "DELETE FROM t WHERE rowid = 1;", false, "10")] + [TestCase("DELETE FROM t WHERE rowid = 1;", "UPDATE t SET v = 20 WHERE rowid = 1;", false, "")] + [TestCase("DELETE FROM t WHERE rowid = 1;", "DELETE FROM t WHERE rowid = 1;", false, "")] + public void StaleWriteAfterPeerCommitRaisesWriteWriteConflict( + string peerWrite, + string staleWrite, + bool checkpointBaseRow, + string expectedAfter) + { + using var db = new RoutingFileDatabase(); + using var seeder = db.Connect(); + seeder.ExecuteNonQuery("PRAGMA journal_mode=mvcc;"); + seeder.ExecuteNonQuery("BEGIN CONCURRENT;"); + seeder.ExecuteNonQuery("INSERT INTO t(rowid, v) VALUES (1, 0);"); + seeder.ExecuteNonQuery("COMMIT;"); + if (checkpointBaseRow) + seeder.ExecuteNonQuery("PRAGMA wal_checkpoint(TRUNCATE);"); + + using var a = db.Connect(); + using var b = db.Connect(); + b.ExecuteNonQuery("BEGIN CONCURRENT;"); + Convert.ToInt64(Scalar(b, "SELECT v FROM t WHERE rowid = 1;")).Should().Be(0L); + + a.ExecuteNonQuery("BEGIN CONCURRENT;"); + a.ExecuteNonQuery(peerWrite); + a.ExecuteNonQuery("COMMIT;"); + + // B's snapshot still reads the pre-peer row. + Convert.ToInt64(Scalar(b, "SELECT v FROM t WHERE rowid = 1;")).Should().Be(0L); + + var error = Capture(() => b.ExecuteNonQuery(staleWrite)); + error.Should().NotBeNull("a write over a peer's post-snapshot commit must conflict"); + error!.Message.Should().ContainEquivalentOf("write-write conflict"); + b.ExecuteNonQuery("ROLLBACK;"); + + ReadValue(a, "SELECT group_concat(v) FROM t;").Should().Be(expectedAfter); + } + + [Test] + public void StaleWriteAfterPeerCommitAndCheckpointRaisesWriteWriteConflict() + { + using var db = new RoutingFileDatabase(); + using var seeder = db.Connect(); + seeder.ExecuteNonQuery("PRAGMA journal_mode=mvcc;"); + seeder.ExecuteNonQuery("INSERT INTO t(rowid, v) VALUES (1, 0);"); + seeder.ExecuteNonQuery("PRAGMA wal_checkpoint(TRUNCATE);"); + + using var a = db.Connect(); + using var b = db.Connect(); + b.ExecuteNonQuery("BEGIN CONCURRENT;"); + Convert.ToInt64(Scalar(b, "SELECT v FROM t WHERE rowid = 1;")).Should().Be(0L); + + a.ExecuteNonQuery("BEGIN CONCURRENT;"); + a.ExecuteNonQuery("UPDATE t SET v = 10 WHERE rowid = 1;"); + a.ExecuteNonQuery("COMMIT;"); + // A checkpoint may or may not be admitted while B's snapshot is open; either way + // B must not be able to overwrite A's committed update. + Capture(() => seeder.ExecuteNonQuery("PRAGMA wal_checkpoint(TRUNCATE);")); + + var error = Capture(() => b.ExecuteNonQuery("UPDATE t SET v = 20 WHERE rowid = 1;")); + error.Should().NotBeNull("a write over a peer's post-snapshot commit must conflict"); + error!.Message.Should().ContainEquivalentOf("write-write conflict"); + b.ExecuteNonQuery("ROLLBACK;"); + + Convert.ToInt64(Scalar(a, "SELECT v FROM t WHERE rowid = 1;")).Should().Be(10L); + } + + [TestCase(true)] + [TestCase(false)] + public void WriteAfterPeerCommitThatPrecedesTheSnapshotDoesNotConflict(bool checkpointBaseRow) + { + using var db = new RoutingFileDatabase(); + using var seeder = db.Connect(); + seeder.ExecuteNonQuery("PRAGMA journal_mode=mvcc;"); + seeder.ExecuteNonQuery("BEGIN CONCURRENT;"); + seeder.ExecuteNonQuery("INSERT INTO t(rowid, v) VALUES (1, 0), (2, 0);"); + seeder.ExecuteNonQuery("COMMIT;"); + if (checkpointBaseRow) + seeder.ExecuteNonQuery("PRAGMA wal_checkpoint(TRUNCATE);"); + + using var a = db.Connect(); + using var b = db.Connect(); + a.ExecuteNonQuery("BEGIN CONCURRENT;"); + a.ExecuteNonQuery("UPDATE t SET v = 10 WHERE rowid = 1;"); + a.ExecuteNonQuery("DELETE FROM t WHERE rowid = 2;"); + a.ExecuteNonQuery("COMMIT;"); + + b.ExecuteNonQuery("BEGIN CONCURRENT;"); + b.ExecuteNonQuery("UPDATE t SET v = v + 1 WHERE rowid = 1;"); + b.ExecuteNonQuery("INSERT INTO t(rowid, v) VALUES (2, 5);"); + b.ExecuteNonQuery("COMMIT;"); + + ReadValue(a, "SELECT group_concat(rowid || ':' || v, ',') FROM t;").Should().Be("1:11,2:5"); + } + + [TestCase("CREATE TABLE t(v INTEGER);", "INSERT INTO t(rowid, v) VALUES (2, 5);")] + [TestCase("CREATE TABLE t(id INTEGER PRIMARY KEY, v INTEGER);", "INSERT INTO t(id, v) VALUES (2, 5);")] + public void ConcurrentInsertKeepsAnExplicitRowidAfterAPeerDelete(string schema, string insert) + { + using var db = new RoutingFileDatabase(schema); + using var seeder = db.Connect(); + seeder.ExecuteNonQuery("PRAGMA journal_mode=mvcc;"); + seeder.ExecuteNonQuery("BEGIN CONCURRENT;"); + seeder.ExecuteNonQuery("INSERT INTO t(rowid, v) VALUES (1, 0), (2, 0);"); + seeder.ExecuteNonQuery("COMMIT;"); + + using var a = db.Connect(); + using var b = db.Connect(); + a.ExecuteNonQuery("BEGIN CONCURRENT;"); + a.ExecuteNonQuery("DELETE FROM t WHERE rowid = 2;"); + a.ExecuteNonQuery("COMMIT;"); + + b.ExecuteNonQuery("BEGIN CONCURRENT;"); + b.ExecuteNonQuery(insert); + ReadValue(b, "SELECT group_concat(rowid || ':' || v, ',') FROM t;").Should().Be("1:0,2:5"); + b.ExecuteNonQuery("COMMIT;"); + + ReadValue(a, "SELECT group_concat(rowid || ':' || v, ',') FROM t;").Should().Be("1:0,2:5"); + } + + [Test] + public void ConcurrentExplicitInsertsOfOneRowidRaiseWriteWriteConflict() + { + using var db = new RoutingFileDatabase(); + using var seeder = db.Connect(); + seeder.ExecuteNonQuery("PRAGMA journal_mode=mvcc;"); + + using var a = db.Connect(); + using var b = db.Connect(); + a.ExecuteNonQuery("BEGIN CONCURRENT;"); + b.ExecuteNonQuery("BEGIN CONCURRENT;"); + a.ExecuteNonQuery("INSERT INTO t(rowid, v) VALUES (7, 1);"); + + var error = Capture(() => b.ExecuteNonQuery("INSERT INTO t(rowid, v) VALUES (7, 2);")); + error.Should().NotBeNull("two writers must not both commit rowid 7"); + error!.Message.Should().ContainEquivalentOf("write-write conflict"); + b.ExecuteNonQuery("ROLLBACK;"); + a.ExecuteNonQuery("COMMIT;"); + + ReadValue(a, "SELECT group_concat(rowid || ':' || v, ',') FROM t;").Should().Be("7:1"); + } + [Test] public void PostCommitSelectSeesPeerInsertAndUpdate() { From 5f107a8611c955f2666257a99d91f21d5148abab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 15:13:47 -0400 Subject: [PATCH 03/20] Close v0.8.1 built-in function, JSON, date/time, and TVF corpus differences Port the Turso v0.8.1 behavior the refreshed sqltest corpus exposed: - json_group_object/jsonb_group_object skip rows whose label is SQL NULL (517ec809f "json: skip NULL labels in object aggregates"). - jsonb(NULL) returns SQL NULL (5775d5337 "json: preserve SQL NULL in jsonb()"). - json_each/json_tree bind a selectable rowid (qualified, unqualified, and in INSERT ... SELECT), numbered from zero like SQLite and core/json/vtab.rs; generate_series reports the generated value as its rowid (core/series.rs) (16a02b139 "core/translate: bind rowid on virtual tables"). - Date/time functions read a BLOB value or modifier as UTF-8 text (b2512eb71 "core/functions: read a BLOB date/time argument as text"). - unixepoch() and strftime('%s') floor to whole seconds, so 1 ms before the epoch is -1 (535f684f1 "core/functions: fix datetime rounding"). - Scalar min() keeps the last of tied arguments while max() keeps the first, matching SQLite's minmaxFunc (78b1ac633, merged as d81560556). - generate_series yields no rows when any supplied bound is NULL. - The simple-count INDEXED BY exemption covers COUNT(col) over a provably non-NULL column, as Turso's detect_simple_aggregate does. The integrity_check/memory.sqltest attached-corrupt-database case stays in the ledger as an intentional managed-ATTACH policy. Co-Authored-By: Claude Opus 5.5 --- .../EmbeddedDatabase.AggregateFunctions.cs | 7 +- src/Ahtola.Core/EmbeddedDatabase.cs | 141 ++++++++++++++--- src/Ahtola.Core/TableValuedFunctionModules.cs | 24 ++- src/Ahtola.Core/TableValuedFunctions.cs | 9 +- .../managed-sqltest-expected-failures.txt | 21 +-- .../FromSourcePlannerParityTests.cs | 45 ++++++ ...agedAggregateAndJsonFunctionParityTests.cs | 26 ++- .../ManagedBuiltinFunctionV081ParityTests.cs | 149 ++++++++++++++++++ src/Ahtola.Tests/TableValuedFunctionTests.cs | 7 + 9 files changed, 373 insertions(+), 56 deletions(-) create mode 100644 src/Ahtola.Tests/ManagedBuiltinFunctionV081ParityTests.cs diff --git a/src/Ahtola.Core/EmbeddedDatabase.AggregateFunctions.cs b/src/Ahtola.Core/EmbeddedDatabase.AggregateFunctions.cs index eba263d6..a85ac8ca 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.AggregateFunctions.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.AggregateFunctions.cs @@ -75,8 +75,13 @@ private SqlValue EvaluateJsonGroupObject( var name = Evaluate(function.Arguments[0], parameters, row, context); var value = Evaluate(function.Arguments[1], parameters, row, context); + // SQLite (and Turso's update_agg_payload since 517ec809f) skips rows whose label is + // SQL NULL before encoding the value; a NULL value is still encoded as JSON null. + if (name.Kind == SqlValueKind.Null) + continue; + // Labels are coerced to text so a non-text name still produces a well-formed object. - members.Add(SqlValue.Text(name.Kind == SqlValueKind.Null ? string.Empty : ToSqlText(name))); + members.Add(SqlValue.Text(ToSqlText(name))); members.Add(value); } diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index b1e16831..2c8c8d57 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -9934,6 +9934,13 @@ private static SourceRow CreateQuerySchemaValidationRow( rowId = 0; rowIdQualifier = named.Alias ?? named.Name; } + else if (source is TableValuedFunctionSource function) + { + // Turso 16a02b139 binds rowid on every virtual table, table-valued functions + // included, so json_tree(...) AS rt exposes rt.rowid and a bare rowid. + rowId = 0; + rowIdQualifier = function.Alias ?? function.Name; + } } return new SourceRow( @@ -9967,6 +9974,14 @@ private static void AppendSchemaRowidBindings( qualifiedColumns.TryAdd($"{qualifier}._rowid_", rowIdIndex); qualifiedColumns.TryAdd($"{qualifier}.oid", rowIdIndex); return; + case TableValuedFunctionSource function: + var functionRowIdIndex = values.Count; + values.Add(SqlValue.Integer(0)); + var functionQualifier = function.Alias ?? function.Name; + qualifiedColumns.TryAdd($"{functionQualifier}.rowid", functionRowIdIndex); + qualifiedColumns.TryAdd($"{functionQualifier}._rowid_", functionRowIdIndex); + qualifiedColumns.TryAdd($"{functionQualifier}.oid", functionRowIdIndex); + return; case JoinTableSource join: AppendSchemaRowidBindings(join.Left, context, values, qualifiedColumns); if (!join.Kind.ProducesLeftShapeOnly()) @@ -30367,7 +30382,7 @@ private void ValidateSelectIndexDirectives(SelectStatement statement, QueryConte inheritedJoinConstraints: [], nullSupplying: false, allowUnqualified: CountTableSourceLeaves(statement.Source) <= 1, - simpleCountStar: IsSimpleCountStarSelect(statement), + simpleCountStar: IsSimpleCountStarSelect(statement, context), context); foreach (var projection in statement.Projections) ValidateExpressionIndexDirectives(projection.Expression, context); @@ -30634,28 +30649,60 @@ private void ValidateNamedTableIndexDirective( } } - // Mirrors Turso's `simple_aggregate == SimpleAggregate::Count` check: no WHERE/GROUP BY/ - // HAVING, a single COUNT(*) projection over a single table source. - private static bool IsSimpleCountStarSelect(SelectStatement statement) + // Mirrors Turso's `simple_aggregate == SimpleAggregate::Count` check (detect_simple_aggregate): + // no WHERE/GROUP BY/HAVING, a single COUNT(*) projection - or a non-DISTINCT + // COUNT(col) whose argument is provably non-NULL (`Expr::is_nonnull`) - over a single table + // source. Both count every row, so the whole-table fast path is exact for either. + private static bool IsSimpleCountStarSelect(SelectStatement statement, QueryContext context) { - return statement.Where is null - && statement.Having is null - && statement.GroupBy.Count == 0 - && statement.Projections.Count == 1 - && statement.Source is NamedTableSource - && statement.Projections[0].Expression is FunctionExpression + if (statement.Where is not null + || statement.Having is not null + || statement.GroupBy.Count != 0 + || statement.Projections.Count != 1 + || statement.Source is not NamedTableSource source + || statement.Projections[0].Expression is not FunctionExpression { - CountStar: true, Distinct: false, Filter: null, Window: null, } function - && string.Equals(function.Name, "count", StringComparison.OrdinalIgnoreCase); + || !string.Equals(function.Name, "count", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (function.CountStar) + return true; + + return function.Arguments.Count == 1 + && context.Tables.TryGetValue(source.Name, out var table) + && IsSimpleCountArgumentNonNull(function.Arguments[0], source, table); + } + + // The conservative column subset of Turso's `Expr::is_nonnull`: a NOT NULL column, the + // INTEGER PRIMARY KEY rowid alias, or the rowid itself of the counted table. + private static bool IsSimpleCountArgumentNonNull(Expression argument, NamedTableSource source, EmbeddedTable table) + { + while (argument is CollationExpression collate) + argument = collate.Expression; + if (argument is not ColumnExpression column + || column.Schema is not null + || column.Qualifier is { } qualifier + && !string.Equals(qualifier, source.Alias ?? source.Name, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var name = column.UnqualifiedName ?? column.Name; + if (table.TryGetColumnIndex(name, out var index)) + return table.ColumnDefinitions[index].NotNull || index == table.RowidAliasColumnIndex; + return table.HasRowid && EmbeddedTable.IsRowidAliasName(name); } // Mirrors Turso's enforce_indexed_by_hints simple-COUNT exemption: SQLite's whole-table // OP_Count fast path skips the WHERE machinery, so an unusable forced partial index is - // ignored for `SELECT COUNT(*) FROM t INDEXED BY idx` and the count must come from the + // ignored for `SELECT COUNT(*) FROM t INDEXED BY idx` (and Turso's non-NULL `COUNT(col)` + // extension of that fast path) and the count must come from the // table rather than the (smaller) index entries. A simple count-star select has no WHERE, // so every partial index is unusable there; drop the directive for the rest of execution // so every planning/evaluation route table-scans. Existence of the named index was already @@ -30664,7 +30711,7 @@ private SelectStatement StripUnusableForcedIndexForCountStar( SelectStatement statement, QueryContext context) { - if (!IsSimpleCountStarSelect(statement) + if (!IsSimpleCountStarSelect(statement, context) || statement.Source is not NamedTableSource { IndexDirective: IndexedByDirective indexedBy } source || !context.Tables.TryGetValue(source.Name, out var table)) { @@ -48632,9 +48679,13 @@ private SqlValue EvaluateScalarMinMax( { if (result.Kind == SqlValueKind.Null || value.Kind == SqlValueKind.Null) return SqlValue.Null; + + // SQLite's minmaxFunc lets min() move to a later argument on a tie and keeps + // max() on the earlier one, so min(1, 1.0) is 1.0 while max(1, 1.0) is 1. Turso + // matches it since 78b1ac633 ("make scalar min keep the last of two tied args"). if (maximum ? Compare(value, result, collation) > 0 - : Compare(value, result, collation) < 0) + : Compare(value, result, collation) <= 0) result = value; } @@ -53143,10 +53194,9 @@ internal static SqlValue Execute(IReadOnlyList args, Func func) for (int i = 1; i < args.Count; i++) { hasModifier = true; - var v = args[i]; - if (v.Kind != SqlValueKind.Text) + if (!TryReadDateTimeText(args[i], out var modifier)) return SqlValue.Null; - if (!ParseModifier(p, v.AsText(), i - 1)) + if (!ParseModifier(p, modifier, i - 1)) return SqlValue.Null; } } @@ -53165,7 +53215,7 @@ internal static SqlValue Execute(IReadOnlyList args, Func func) case Func.UnixEpoch: if (p.UseSubsec) return SqlValue.Real((double)(p.IJd - UnixEpochIJd) / 1000.0); - return SqlValue.Integer((p.IJd - UnixEpochIJd) / 1000); + return SqlValue.Integer(UnixSeconds(p.IJd)); default: p.ComputeYmdHms(); if (p.IsError) @@ -53215,10 +53265,9 @@ internal static SqlValue Strftime(IReadOnlyList args) for (int i = 2; i < args.Count; i++) { - var v = args[i]; - if (v.Kind != SqlValueKind.Text) + if (!TryReadDateTimeText(args[i], out var modifier)) return SqlValue.Null; - if (!ParseModifier(p, v.AsText(), i - 2)) + if (!ParseModifier(p, modifier, i - 2)) return SqlValue.Null; } } @@ -53231,12 +53280,54 @@ internal static SqlValue Strftime(IReadOnlyList args) return FormatStrftime(fmt, p); } + /// + /// Whole Unix seconds for a julian-day millisecond count, rounded toward negative + /// infinity like SQLite's iJD/1000 - 21086676*10000 (iJD is never negative + /// there), so 1969-12-31 23:59:59.999 is -1 rather than 0. Turso matches it since + /// 535f684f1 ("core/functions: fix datetime rounding"). + /// + private static long UnixSeconds(long iJd) + => Math.DivRem(iJd - UnixEpochIJd, 1000L, out var remainder) - (remainder < 0 ? 1 : 0); + + /// + /// Reads a date/time argument or modifier the way SQLite's sqlite3_value_text() does: + /// text as-is and a BLOB's bytes as UTF-8 text. Bytes that are not UTF-8 are left + /// unread and the caller reports NULL (Turso b2512eb71, "read a BLOB date/time + /// argument as text"). + /// + private static bool TryReadDateTimeText(SqlValue value, out string text) + { + switch (value.Kind) + { + case SqlValueKind.Text: + text = value.AsText(); + return true; + case SqlValueKind.Blob: + try + { + text = StrictUtf8.GetString(value.AsBlob().Span); + return true; + } + catch (DecoderFallbackException) + { + break; + } + } + + text = string.Empty; + return false; + } + + private static readonly UTF8Encoding StrictUtf8 = new(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + private static bool InitTimeValue(Dt p, SqlValue value) { switch (value.Kind) { case SqlValueKind.Text: return ParseDateOrTime(value.AsText(), p); + case SqlValueKind.Blob: + return TryReadDateTimeText(value, out var blobText) && ParseDateOrTime(blobText, p); case SqlValueKind.Integer: SetRawNumber(p, value.AsInteger()); return true; @@ -53401,7 +53492,7 @@ private static SqlValue FormatStrftime(string fmt, Dt p) if (p.UseSubsec) res.Append(((double)(p.IJd - UnixEpochIJd) / 1000.0).ToString("F3", CultureInfo.InvariantCulture)); else - res.Append(((p.IJd - UnixEpochIJd) / 1000).ToString(CultureInfo.InvariantCulture)); + res.Append(UnixSeconds(p.IJd).ToString(CultureInfo.InvariantCulture)); break; case 'S': res.Append(((int)p.S).ToString("D2", CultureInfo.InvariantCulture)); @@ -54200,8 +54291,10 @@ internal static SqlValue Json(IReadOnlyList args) internal static SqlValue Jsonb(IReadOnlyList args) { RequireArgumentCount("jsonb", args, 1); + // jsonb(NULL) is SQL NULL like SQLite's jsonb(); Turso matches it since 5775d5337 + // ("json: preserve SQL NULL in jsonb()"). return args[0].Kind == SqlValueKind.Null - ? SqlValue.Blob([0]) + ? SqlValue.Null : ToJsonb(args[0]); } diff --git a/src/Ahtola.Core/TableValuedFunctionModules.cs b/src/Ahtola.Core/TableValuedFunctionModules.cs index 9d24d8d4..e9bbb868 100644 --- a/src/Ahtola.Core/TableValuedFunctionModules.cs +++ b/src/Ahtola.Core/TableValuedFunctionModules.cs @@ -20,10 +20,17 @@ internal sealed class GenerateSeriesModule : TableValuedFunctionModule public override IReadOnlyList Enumerate(TableValuedFunctionCall call) { // SQLite's series module produces no rows when the mandatory first argument is - // absent or NULL, and reports an error for a non-integer bound. - if (!call.HasArgument(0) || call.Arguments[0].Kind == SqlValueKind.Null) + // absent or when any supplied bound is NULL (core/series.rs filter), and reports an + // error for a non-integer bound. + if (!call.HasArgument(0)) return []; + for (var index = 0; index < 3; index++) + { + if (call.HasArgument(index) && call.Arguments[index].Kind == SqlValueKind.Null) + return []; + } + var start = RequireInteger(call.Arguments[0]); var stop = call.HasArgument(1) ? RequireInteger(call.Arguments[1]) : 0xffffffffL; var step = call.HasArgument(2) ? RequireInteger(call.Arguments[2]) : 1; @@ -72,6 +79,12 @@ IEnumerable EnumerateRows() } } + /// + /// SQLite's series cursor (and Turso's core/series.rs) reports each generated value as + /// its rowid, so a value keeps its identity across scans with different bounds. + /// + public override long GetRowId(long ordinal, SqlValue[] row) => row[0].AsInteger(); + private static SqlValue[] BuildRow(long value, SqlValue[] bounds) => [SqlValue.Integer(value), bounds[0], bounds[1], bounds[2]]; @@ -141,6 +154,13 @@ public override IReadOnlyList Enumerate(TableValuedFunctionCall call return result; } + + /// + /// SQLite's json_each/json_tree cursor numbers rows from zero in emission order, and + /// Turso's core/json/vtab.rs does the same; rowid became selectable in Turso with + /// 16a02b139 ("core/translate: bind rowid on virtual tables"). + /// + public override long GetRowId(long ordinal, SqlValue[] row) => ordinal - 1; } /// diff --git a/src/Ahtola.Core/TableValuedFunctions.cs b/src/Ahtola.Core/TableValuedFunctions.cs index 29d17cce..7b5f434d 100644 --- a/src/Ahtola.Core/TableValuedFunctions.cs +++ b/src/Ahtola.Core/TableValuedFunctions.cs @@ -85,6 +85,13 @@ internal abstract class TableValuedFunctionModule public virtual int? SchemaNameArgumentIndex => null; public abstract IReadOnlyList Enumerate(TableValuedFunctionCall call); + + /// + /// The rowid the module's cursor reports for an enumerated row. + /// counts rows from one within the current filter, which is what SQLite's pragma virtual + /// tables report; modules whose SQLite/Turso cursor numbers rows differently override it. + /// + public virtual long GetRowId(long ordinal, SqlValue[] row) => ordinal; } /// @@ -251,7 +258,7 @@ public override SqlValue Column(int index) public override long RowId => Eof ? throw new InvalidOperationException("Table-valued function cursor is not positioned.") - : _rowId; + : module.GetRowId(_rowId, _current!); public override void Dispose() => DisposeEnumeration(); diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index 3d47aa06..7cfbc787 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -163,9 +163,6 @@ turso/mvcc_sequence.sqltest::mvcc-wal-mvcc-roundtrip-autoincrement | intentional # v0.8.1 corpus adoption (2026-09-30): the vendored corpus was refreshed from the # pinned turso-src v0.8.1 (8549c1659): 33 new files and 41 refreshed files. The # entries below are the differences that refresh exposed, pending closure. -agg-functions/json-group-object-null-label.sqltest::json_group_object_null_label_only | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{}/{}/{"a":null}/[null]' but was '{"":1}/{"":1}/{"a":null}/[null]' -agg-functions/json-group-object-null-label.sqltest::json_group_object_skips_null_label | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"a":1,"b":3}' but was '{"a":1,"":2,"b":3}' -agg-functions/json-group-object-null-label.sqltest::jsonb_group_object_skips_null_label | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"a":1,"b":3}' but was '{"a":1,"":2,"b":3}' alter_table.sqltest::alter-table-add-column-preserve-schema-collate-locale | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected 'CREATE TABLE tbl (c TEXT COLLATE "fr-FR", d)' but was 'CREATE TABLE tbl (c TEXT COLLATE 'fr-FR', d)' alter_table.sqltest::fail-alter-table-rename-to-existing-view | database variant 1/1 (:memory:): error 'there is already a view named y' does not match pattern 'there is already another table or index with this name: y' alter_table.sqltest::fail-alter-table-rename-to-existing-view-different-case | database variant 1/1 (:memory:): error 'there is already a view named Y' does not match pattern 'there is already another table or index with this name: Y' @@ -177,8 +174,7 @@ gencol.sqltest::gencol_issue7068_check_sees_recomputed_value | database variant gencol.sqltest::gencol_issue7068_multiple_defaults_before_generated_notnull | database variant 1/1 (:memory:): expected success but got error: NOT NULL constraint failed: t.b gencol.sqltest::gencol_issue7068_returning_recomputed_after_default | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '5/6' but was '5/' gencol.sqltest::gencol_issue7068_unique_index_uses_recomputed_value | database variant 1/1 (:memory:): expected 2 row(s), got 2 row(s); row 0 expected '1/5/6' but was '1/5/' -generate_series.sqltest::generate-series-null-arguments-return-no-rows | database variant 1/1 (:memory:): expected success but got error: generate_series() requires integer arguments -integrity_check/memory.sqltest::integrity-check-qualified-corrupt-attached-database | database variant 1/1 (:memory:): expected success but got error: Managed ATTACH requires a file-backed managed primary database so attachments share its file system. +integrity_check/memory.sqltest::integrity-check-qualified-corrupt-attached-database | intentional managed-ATTACH policy: a file ATTACH needs a file-backed managed primary whose IFileSystem the attachment shares (a :memory: primary has none), and the corrupt fixture is independently rejected at open (see the parity_corrupt_index entries) join/hash.sqltest::hash-join-result-matrix-full-join-uses-a-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN matrix_right AS r' join/hash.sqltest::hash-join-result-matrix-left-join-uses-a-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN matrix_right AS r' join/hash.sqltest::hash-join-result-matrix-uses-a-hash-join | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN matrix_right AS r.*SCAN matrix_left AS l' @@ -201,10 +197,6 @@ join/memory.sqltest::partial-stats-hash-join-uses-known-unique-key-count | datab join/memory.sqltest::range-prefix-still-allows-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN prefix_child AS child' join/memory.sqltest::unique-composite-hash-key-uses-one-key-estimate | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN composite_hash_child AS child".*"rows_per_input":5' join/memory.sqltest::unique-hash-key-does-not-apply-to-a-different-collation | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 127. -json/default.sqltest::jsonb_null_is_sql_null | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected 'null/1' but was 'blob/0' -json/json_tree_columns.sqltest::json-tree-rowid-into-table | database variant 1/1 (:memory:): expected success but got error: no such column: rt.rowid -json/json_tree_columns.sqltest::json-tree-rowid-is-selectable | database variant 1/1 (:memory:): expected success but got error: no such column: rt.rowid -json/json_tree_columns.sqltest::json-tree-rowid-unqualified | database variant 1/1 (:memory:): expected success but got error: no such column: rowid multi_index_or_compound.sqltest::compound-or-keeps-correlated-index-columns-together | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '1/0/0/MULTI-INDEX OR t (txy, txy)' but was '0/0/0/MANAGED EVALUATOR FALLBACK' multi_index_or_compound.sqltest::derived-or-filter-skips-correlated-columns | database variant 1/1 (:memory:): output does not match pattern '"rows_per_input":1000' multi_index_or_compound.sqltest::implied-or-filter-accepts-negative-numbers | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' @@ -220,17 +212,6 @@ partial_idx.sqltest::partial-index-or-term-with-covering-scan | database variant partial_idx.sqltest::partial-index-or-term-with-equality-search | database variant 1/1 (:memory:): expected 2 row(s), got 2 row(s); row 0 expected '1/0/0/SEARCH t USING INDEX idx (a=?)' but was '1/0/0/SCAN t' partial_idx.sqltest::partial-index-or-term-with-order | database variant 1/1 (:memory:): expected 3 row(s), got 4 row(s); row 0 expected '1/0/0/SCAN t USING INDEX idx' but was '1/0/0/SCAN t' pragma/memory.sqltest::pragma-temp-synchronous-stays-off | database variant 1/1 (:memory:): expected 3 row(s), got 3 row(s); row 0 expected '0' but was '2' -scalar-functions-datetime.sqltest::date-blob-modifier | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '2024-06-16' but was '' -scalar-functions-datetime.sqltest::date-blob-time-value | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '2024-01-01' but was '' -scalar-functions-datetime.sqltest::date-unixepoch-at-1-millisecond-before-epochtime | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '1969-12-31' but was '1970-01-01' -scalar-functions-datetime.sqltest::strftime-at-1-millisecond-before-epochtime | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '-1' but was '0' -scalar-functions-datetime.sqltest::strftime-blob-time-value | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '2024-01-01' but was '' -scalar-functions-datetime.sqltest::timediff-blob-time-values | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '+0000-00-01 00:00:00.000' but was '' -scalar-functions-datetime.sqltest::unixepoch-at-1-millisecond-before-epochtime | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '-1' but was '0' -scalar-functions-datetime.sqltest::unixepoch-blob-time-value | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '1704067200' but was '' -scalar-functions.sqltest::min-float-equals-int | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '1' but was '1.0' -scalar-functions.sqltest::min-int-equals-float | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '1.0' but was '1' -simple-count-optimization.sqltest::simple-count-unusable-partial-index-column-correctness | database variant 1/1 (:memory:): expected success but got error: no query solution sql-error-message-prefix.sqltest::database-full-error-has-no-prefix | database variant 1/1 (:memory:): expected an error but the statement succeeded turso/alter_column.sqltest::alter-column-rejects-index-only-rewrite-mvcc | database variant 1/1 (:memory:): expected an error but the statement succeeded turso/alter_column.sqltest::alter-column-rejects-index-rewrite-mvcc | database variant 1/1 (:memory:): expected an error but the statement succeeded diff --git a/src/Ahtola.Tests/FromSourcePlannerParityTests.cs b/src/Ahtola.Tests/FromSourcePlannerParityTests.cs index 000c8730..5aa79848 100644 --- a/src/Ahtola.Tests/FromSourcePlannerParityTests.cs +++ b/src/Ahtola.Tests/FromSourcePlannerParityTests.cs @@ -197,6 +197,51 @@ LEFT JOIN right_items INDEXED BY right_null AssertMatchesSqlite(setup, "SELECT * FROM item_view NOT INDEXED ORDER BY id;"); } + [Test] + public void SimpleCountIgnoresAnUnusableForcedPartialIndexForNonNullArguments() + { + // Turso's detect_simple_aggregate treats COUNT(col) over a provably non-NULL column as + // the simple count, so enforce_indexed_by_hints ignores an unusable forced partial + // index there instead of failing with "no query solution". SQLite only does this for + // COUNT(*), and the corpus deliberately follows Turso for the column form + // (simple-count-unusable-partial-index-column-correctness, @skip-if sqlite). + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + Execute( + connection, + """ + CREATE TABLE t(id INTEGER PRIMARY KEY, x NOT NULL, y); + CREATE INDEX pi ON t(x) WHERE x > 5; + INSERT INTO t VALUES (1, 9, NULL), (2, 1, 1), (3, 2, NULL); + """); + + foreach (var query in new[] + { + "SELECT count(*) FROM t INDEXED BY pi;", + "SELECT count(x) FROM t INDEXED BY pi;", + "SELECT count(t.x) FROM t INDEXED BY pi;", + "SELECT count(x COLLATE NOCASE) FROM t INDEXED BY pi;", + "SELECT count(id) FROM t INDEXED BY pi;", + "SELECT count(rowid) FROM t INDEXED BY pi;", + }) + { + Query(connection, query).Should().ContainSingle() + .Which.Should().Equal([SqlValue.Integer(3)], because: query); + } + + // A nullable argument, DISTINCT, or a filter is not a simple count and keeps the error. + foreach (var query in new[] + { + "SELECT count(y) FROM t INDEXED BY pi;", + "SELECT count(DISTINCT x) FROM t INDEXED BY pi;", + "SELECT count(x) FROM t INDEXED BY pi WHERE y IS NULL;", + }) + { + var error = Assert.Throws(() => Query(connection, query)); + error!.Message.Should().Be("no query solution", because: query); + } + } + [Test] public void JoinHintsUseOnlyPredicatesThatAreSafeForEachOuterJoinSide() { diff --git a/src/Ahtola.Tests/ManagedAggregateAndJsonFunctionParityTests.cs b/src/Ahtola.Tests/ManagedAggregateAndJsonFunctionParityTests.cs index 5457f0a2..bf4ec27d 100644 --- a/src/Ahtola.Tests/ManagedAggregateAndJsonFunctionParityTests.cs +++ b/src/Ahtola.Tests/ManagedAggregateAndJsonFunctionParityTests.cs @@ -94,19 +94,27 @@ public void JsonGroupObjectBuildsAnObjectFromNameValuePairs() } [Test] - public void JsonGroupObjectCoercesANullNameToAnEmptyQuotedLabel() + public void JsonGroupObjectSkipsRowsWithANullLabel() { using var database = new EmbeddedDatabase(); using var connection = database.Connect(); Seed(connection); - // SQLite 3.46 emits a bare `:value` here, which is not valid JSON. The Rust core - // converts the name with a to-string conversion, so the managed engine matches the - // core and always produces a parseable document. + // SQLite's jsonObjectStep drops a row whose label is SQL NULL, and Turso matches it + // since 517ec809f ("json: skip NULL labels in object aggregates"). A NULL *value* is + // still kept as a JSON null. ReadValue(connection, "SELECT json_group_object(v, n) FROM t;") - .Should().Be(SqlValue.JsonText("{\"x\":1,\"y\":2,\"\":3}")); - ReadValue(connection, "SELECT json_valid(json_group_object(v, n)) FROM t;") - .Should().Be(SqlValue.Integer(1)); + .Should().Be(SqlValue.JsonText("{\"x\":1,\"y\":2}")); + ReadValue(connection, "SELECT json(jsonb_group_object(v, n)) FROM t;") + .Should().Be(SqlValue.JsonText("{\"x\":1,\"y\":2}")); + ReadValue(connection, "SELECT json_group_object(k, v) FROM t;") + .Should().Be(SqlValue.JsonText("{\"a\":\"x\",\"b\":\"y\",\"c\":null}")); + ReadValue(connection, "SELECT json_group_object(NULL, n) FROM t;") + .Should().Be(SqlValue.JsonText("{}")); + ReadValue(connection, "SELECT json(jsonb_group_object(NULL, n)) FROM t;") + .Should().Be(SqlValue.JsonText("{}")); + ReadColumn(connection, "SELECT json_group_object(v, n) OVER (ORDER BY n) FROM t;") + .Should().Equal("{\"x\":1}", "{\"x\":1,\"y\":2}", "{\"x\":1,\"y\":2}"); } [Test] @@ -151,7 +159,9 @@ public void JsonbFunctionsUseSQLiteBinaryEncodingAndRoundTripThroughJson() .Should().Be(SqlValue.Text("4C17311332")); ReadValue(connection, "SELECT hex(jsonb_group_object(1.5, 2));") .Should().Be(SqlValue.Text("6C37312E351332")); - ReadValue(connection, "SELECT hex(jsonb(NULL));") + ReadValue(connection, "SELECT jsonb(NULL);") + .Should().Be(SqlValue.Null); + ReadValue(connection, "SELECT hex(jsonb('null'));") .Should().Be(SqlValue.Text("00")); ReadValue(connection, "SELECT json(jsonb_group_object(1, jsonb_array(2)));") .Should().Be(SqlValue.JsonText("{\"1\":[2]}")); diff --git a/src/Ahtola.Tests/ManagedBuiltinFunctionV081ParityTests.cs b/src/Ahtola.Tests/ManagedBuiltinFunctionV081ParityTests.cs new file mode 100644 index 00000000..d38a8440 --- /dev/null +++ b/src/Ahtola.Tests/ManagedBuiltinFunctionV081ParityTests.cs @@ -0,0 +1,149 @@ +using AwesomeAssertions; +using MsData = Microsoft.Data.Sqlite; +using Ahtola.Core; + +namespace Ahtola.Tests; + +/// +/// Differential regressions for the built-in function behavior Turso v0.8.1 aligned with +/// SQLite: scalar min/max ties, BLOB date/time arguments, pre-epoch second rounding, and the +/// selectable rowid of json_each/json_tree. Each case is compared with the bundled SQLite. +/// +public sealed class ManagedBuiltinFunctionV081ParityTests +{ + // Upstream 78b1ac633: SQLite's minmaxFunc lets min() take the later of tied arguments + // while max() keeps the earlier one, including under an argument collation. + [TestCase("SELECT min(1, 1.0), max(1, 1.0), min(1.0, 1), max(1.0, 1)")] + [TestCase("SELECT typeof(min(1, 1.0)), typeof(max(1, 1.0)), typeof(min(1.0, 1)), typeof(max(1.0, 1))")] + [TestCase("SELECT min(2, 2.0, 2), min(2.0, 2, 2.0), max(2, 2.0, 2), max(2.0, 2, 2.0)")] + [TestCase("SELECT min('a' COLLATE NOCASE, 'A'), max('A' COLLATE NOCASE, 'a')")] + [TestCase("SELECT min('A', 'a' COLLATE NOCASE), max('a', 'A' COLLATE NOCASE)")] + [TestCase("SELECT min('A' COLLATE NOCASE, 'a', 'B'), max('b' COLLATE NOCASE, 'B', 'a')")] + // Upstream b2512eb71: a BLOB date/time value or modifier is read as its UTF-8 text. + [TestCase("SELECT date(x'323032342D30312D3031'), unixepoch(x'323032342D30312D3031')")] + [TestCase("SELECT date('2024-06-15', X'2B3120646179'), strftime(x'2559', '2024-01-01')")] + [TestCase("SELECT julianday(x'323032342D30312D3031'), time(x'31323A30303A3030')")] + [TestCase("SELECT datetime(x'323032342D30312D3031', x'2B31206D6F6E7468')")] + [TestCase("SELECT timediff(X'323032342D30312D3032', X'323032342D30312D3031')")] + [TestCase("SELECT date(x'FF'), date('2024-01-01', x'FF'), timediff(x'FF', '2024-01-01')")] + // Upstream 535f684f1: whole Unix seconds round toward negative infinity. + [TestCase("SELECT unixepoch('1969-12-31 23:59:59.999'), strftime('%s', '1969-12-31T23:59:59.999')")] + [TestCase("SELECT unixepoch('1969-12-31 23:59:59.001'), unixepoch('1969-12-31 23:59:59.999', 'subsec')")] + [TestCase("SELECT date(unixepoch('1969-12-31T23:59:59.999Z'), 'unixepoch')")] + [TestCase("SELECT strftime('%s', '1969-12-31 23:59:58.500'), unixepoch('1969-12-31 23:59:58.500')")] + [TestCase("SELECT strftime('%s', '0000-01-01'), unixepoch('-4713-11-24 12:00:00'), unixepoch('1970-01-01 00:00:00.999')")] + // Upstream 16a02b139: json_each/json_tree number rows from zero and expose them as rowid. + [TestCase("SELECT rowid, key FROM json_each('[5,6,7]')")] + [TestCase("SELECT rowid, fullkey FROM json_tree('{\"a\":[1,2]}')")] + [TestCase("SELECT rt.rowid, rt.oid, rt._rowid_ FROM json_tree('{\"a\":[1,2]}') AS rt WHERE rt.type NOT IN ('object','array')")] + [TestCase("SELECT rowid FROM json_each('[1,2]') ORDER BY rowid DESC")] + [TestCase("SELECT max(rowid), count(rowid) FROM json_tree('[1,[2,3]]')")] + [TestCase("SELECT je.rowid, je.value FROM (SELECT '[1,2]' AS j UNION ALL SELECT '[3]') AS t, json_each(t.j) AS je")] + [TestCase("SELECT je.rowid, je.value FROM json_each('[1,2]') AS je JOIN json_each('[9]') AS k ON 1")] + [TestCase("SELECT (SELECT max(rowid) FROM json_each('[1,2,3]'))")] + // Upstream 517ec809f and 5775d5337: NULL object labels are skipped, jsonb(NULL) is NULL. + [TestCase("SELECT json_group_object(k, v) FROM (SELECT 'a' AS k, 1 AS v UNION ALL SELECT NULL, 2 UNION ALL SELECT 'b', 3)")] + [TestCase("SELECT json_group_object(NULL, 1), json(jsonb_group_object(NULL, 1)), json_group_object('a', NULL)")] + [TestCase("SELECT json_group_object(k, v) FROM (SELECT NULL AS k, x'00' AS v)")] + [TestCase("SELECT typeof(jsonb(NULL)), jsonb(NULL) IS NULL, hex(jsonb('null'))")] + public void MatchesSqlite(string sql) + { + RunManaged(sql).Should().Equal(RunSqlite(sql), because: sql); + } + + [Test] + public void JsonTreeRowIdCanBeStoredThroughInsertSelect() + { + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + Execute(connection, "CREATE TABLE kv(n, key, val);"); + Execute( + connection, + "INSERT INTO kv SELECT rt.rowid, rt.fullkey, rt.atom FROM json_tree('{\"a\":[1,2]}') AS rt " + + "WHERE rt.type NOT IN ('object','array');"); + + RunManaged(connection, "SELECT n, key, val FROM kv ORDER BY key;") + .Should().Equal("i:2|t:$.a[0]|i:1", "i:3|t:$.a[1]|i:2"); + } + + [Test] + public void GenerateSeriesReportsEachValueAsItsRowId() + { + // SQLite's series extension (not bundled with the oracle build) and Turso's + // core/series.rs both use the generated value as the rowid. + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + + RunManaged(connection, "SELECT rowid, value FROM generate_series(3, 5);") + .Should().Equal("i:3|i:3", "i:4|i:4", "i:5|i:5"); + } + + private static void Execute(EmbeddedConnection connection, string sql) + { + using var statement = connection.Prepare(sql); + statement.Step().Should().Be(StatementStepResult.Done); + } + + private static List RunManaged(string sql) + { + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + return RunManaged(connection, sql); + } + + private static List RunManaged(EmbeddedConnection connection, string sql) + { + using var statement = connection.Prepare(sql); + var rows = new List(); + while (statement.Step() == StatementStepResult.Row) + { + var cells = new string[statement.ColumnCount]; + for (var index = 0; index < cells.Length; index++) + cells[index] = Format(statement.GetValue(index)); + rows.Add(string.Join("|", cells)); + } + + return rows; + } + + private static List RunSqlite(string sql) + { + using var connection = new MsData.SqliteConnection("Data Source=:memory:"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = sql; + using var reader = command.ExecuteReader(); + var rows = new List(); + while (reader.Read()) + { + var cells = new string[reader.FieldCount]; + for (var index = 0; index < cells.Length; index++) + { + cells[index] = reader.GetValue(index) switch + { + DBNull => Format(SqlValue.Null), + long integer => Format(SqlValue.Integer(integer)), + double real => Format(SqlValue.Real(real)), + string text => Format(SqlValue.Text(text)), + byte[] blob => Format(SqlValue.Blob(blob)), + var other => throw new InvalidOperationException($"Unexpected SQLite value {other}."), + }; + } + + rows.Add(string.Join("|", cells)); + } + + return rows; + } + + private static string Format(SqlValue value) + => value.Kind switch + { + SqlValueKind.Null => "null", + SqlValueKind.Integer => "i:" + value.AsInteger().ToString(System.Globalization.CultureInfo.InvariantCulture), + SqlValueKind.Real => "r:" + value.AsReal().ToString("R", System.Globalization.CultureInfo.InvariantCulture), + SqlValueKind.Text => "t:" + value.AsText(), + SqlValueKind.Blob => "b:" + Convert.ToHexString(value.AsBlob().Span), + _ => throw new InvalidOperationException($"Unknown value kind {value.Kind}."), + }; +} diff --git a/src/Ahtola.Tests/TableValuedFunctionTests.cs b/src/Ahtola.Tests/TableValuedFunctionTests.cs index f52c70da..4a14932f 100644 --- a/src/Ahtola.Tests/TableValuedFunctionTests.cs +++ b/src/Ahtola.Tests/TableValuedFunctionTests.cs @@ -422,6 +422,13 @@ public void BareModuleNamesBindArgumentsFromWhereEqualities() Rows(connection, "SELECT count(*) FROM pragma_table_info;").Should().Equal(["0"]); Rows(connection, "SELECT count(*) FROM generate_series(NULL, 5);").Should().Equal(["0"]); + // Any supplied NULL bound - stop or step included - yields no rows rather than an + // integer-argument error (core/series.rs filter, like SQLite's series module). + Rows(connection, "SELECT count(*) FROM generate_series(1, NULL);").Should().Equal(["0"]); + Rows(connection, "SELECT count(*) FROM generate_series(1, 3, NULL);").Should().Equal(["0"]); + Rows(connection, "SELECT count(*) FROM generate_series WHERE start = 1 AND stop = NULL;") + .Should().Equal(["0"]); + Rows(connection, "SELECT schema, name, type, ncol, wr, strict FROM pragma_table_list WHERE name = 'child';") .Should().Equal(["main|child|table|2|0|0"]); } From e288415b883e8e121a3bc73ee7815a90228b4254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 15:19:29 -0400 Subject: [PATCH 04/20] Carry SQLite result codes through the facade, guard explicit checkpoints, coerce FTS queries Error-code fidelity (Microsoft.Data.Sqlite parity): - EmbeddedSqlException.SqliteErrorCode now yields the (extended) SQLite result code for every classic failure class: explicit where the raise site knows it (primary-key/rowid conflicts 1555/2579, RAISE 1811, datatype mismatch 20, sequence exhaustion 13) and otherwise derived from SQLite's own error text (UNIQUE 2067, NOT NULL 1299, CHECK 275, FOREIGN KEY 787, STRICT 3091, readonly 8, busy 5, full 13, too big 18, corrupt 11, ...). Wrappers that re-raise the same failure keep the wrapped code. - The SQLite facade reports the primary code in SqliteErrorCode and the "SQLite Error N" text, and the extended code in SqliteExtendedErrorCode, on the embedded, remote (Hrana error.code names such as SQLITE_CONSTRAINT_UNIQUE) and SqliteRemoteException paths. - A garbage plain header (bad magic, invalid page size, too few usable bytes, too-short file) opened without a key or codec reports SQLITE_NOTADB (26) "file is not a database" like SQLite and upstream 6480cd77e; keyed/codec opens keep the encrypted-or-not-a-database phrase. - SqliteErrorCodeDifferentialTests pins the codes against e_sqlite3 for in-memory, file-backed and MVCC databases. Explicit checkpoints (upstream a95d01284 / 7ad5d9de4): PRAGMA wal_checkpoint is refused with SQLITE_BUSY "cannot checkpoint while another statement is active - SQL statements in progress" while another statement on the same connection is active or suspended; open blob handles do not block it. FTS (upstream d485aa6d8): integer, real and blob fts_match/fts_score queries are searched by their text form on the indexed and scalar paths instead of raising "requires a text query". Co-Authored-By: Claude Opus 5.5 --- .../EmbeddedDatabase.FtsFunctions.cs | 2 - src/Ahtola.Core/EmbeddedDatabase.cs | 122 ++++- src/Ahtola.Core/EmbeddedFileStore.cs | 6 +- .../Execution/ResumableStatement.cs | 4 +- src/Ahtola.Core/Execution/VdbeProgram.cs | 152 ++++++ src/Ahtola.Core/ManagedSequence.cs | 4 +- src/Ahtola.Core/Search/ManagedFtsFunctions.cs | 5 +- .../Search/ManagedFtsPlannerAdapter.cs | 30 +- .../Storage/SqliteDatabaseHeader.cs | 39 +- src/Ahtola.Core/Storage/SqlitePageStore.cs | 10 +- src/Ahtola.Data.Sqlite/SqliteCommand.cs | 55 ++- src/Ahtola.Data.Sqlite/SqliteConnection.cs | 21 + .../SqliteRemoteException.cs | 10 +- ...ManagedIndexMethodReviewRegressionTests.cs | 87 +++- .../MvccCheckpointStateMachineTests.cs | 61 +++ .../SqliteErrorCodeDifferentialTests.cs | 433 ++++++++++++++++++ .../VdbeScalarControlOpcodeTests.cs | 6 +- 17 files changed, 964 insertions(+), 83 deletions(-) create mode 100644 src/Ahtola.Tests/SqliteErrorCodeDifferentialTests.cs diff --git a/src/Ahtola.Core/EmbeddedDatabase.FtsFunctions.cs b/src/Ahtola.Core/EmbeddedDatabase.FtsFunctions.cs index ad93598f..00544fd7 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.FtsFunctions.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.FtsFunctions.cs @@ -154,8 +154,6 @@ private SqlValue EvaluateFtsScore( { if (arguments[^1].Kind == SqlValueKind.Null) return SqlValue.Real(0.0); - if (arguments[^1].Kind != SqlValueKind.Text) - throw new EmbeddedSqlException("fts_score() requires a text query"); if (bound.RowId is not { } rowId) return SqlValue.Real(0.0); diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index b1e16831..0e81c809 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -28,13 +28,15 @@ internal readonly record struct TransactionSnapshot( public class EmbeddedSqlException : Exception { + private readonly int? _explicitSqliteErrorCode; + public EmbeddedSqlException(string message) : base(message) { } public EmbeddedSqlException(string message, int sqliteErrorCode) : base(message) { - SqliteErrorCode = sqliteErrorCode; + _explicitSqliteErrorCode = sqliteErrorCode; } internal EmbeddedSqlException(string message, InsertConflictAlgorithm conflictAlgorithm) : base(message) @@ -45,9 +47,11 @@ internal EmbeddedSqlException(string message, InsertConflictAlgorithm conflictAl internal EmbeddedSqlException( string message, InsertConflictAlgorithm? conflictAlgorithm, - bool constraintViolation = true) : base(message) + bool constraintViolation = true, + int? sqliteErrorCode = null) : base(message) { ConflictAlgorithm = conflictAlgorithm ?? InsertConflictAlgorithm.Abort; + _explicitSqliteErrorCode = sqliteErrorCode; } internal EmbeddedSqlException( @@ -55,18 +59,30 @@ internal EmbeddedSqlException( int sqliteErrorCode, InsertConflictAlgorithm? conflictAlgorithm) : base(message) { - SqliteErrorCode = sqliteErrorCode; + _explicitSqliteErrorCode = sqliteErrorCode; ConflictAlgorithm = conflictAlgorithm; } public EmbeddedSqlException(string message, Exception innerException) : base(message, innerException) { + // A wrapper that re-raises the same failure (conflict rollback/fail, statement abort) + // keeps the wrapped failure's result code, so a primary-key conflict stays 1555 rather + // than being re-derived from its "UNIQUE constraint failed" text. + if (innerException is EmbeddedSqlException { _explicitSqliteErrorCode: { } innerCode } inner + && string.Equals(inner.Message, message, StringComparison.Ordinal)) + { + _explicitSqliteErrorCode = innerCode; + } } /// - /// Optional SQLite result code (e.g. 19 CONSTRAINT) when raised from Halt / HaltIfNull. + /// The SQLite result code for this failure, possibly an extended code (e.g. 2067 + /// SQLITE_CONSTRAINT_UNIQUE, whose primary code is code & 0xFF = 19). + /// Explicit when the raise site knows it (Halt / HaltIfNull, primary-key conflicts); + /// otherwise derived from SQLite's own error text for the classic failure classes. + /// means a plain SQLITE_ERROR. /// - public int? SqliteErrorCode { get; } + public int? SqliteErrorCode => _explicitSqliteErrorCode ?? SqliteResultCode.InferFromMessage(Message); internal InsertConflictAlgorithm? ConflictAlgorithm { get; } } @@ -201,7 +217,7 @@ public EmbeddedStatementFailureException(Exception failure, long lastInsertRowId internal sealed class EmbeddedTriggerRaiseException : EmbeddedSqlException { public EmbeddedTriggerRaiseException(string message, InsertConflictAlgorithm algorithm) - : base(message, algorithm) + : base(message, algorithm, constraintViolation: true, SqliteResultCode.ConstraintTrigger) { Algorithm = algorithm; } @@ -12333,7 +12349,8 @@ private IReadOnlyList FindInsertUniqueConflicts( if (Add( rowPosition, $"UNIQUE constraint failed: {tableName}.{column}", - aliasIndex >= 0 ? table.RowidAliasConflictAlgorithm : null)) + aliasIndex >= 0 ? table.RowidAliasConflictAlgorithm : null, + RowidConflictCode(aliasIndex))) { return conflicts; } @@ -12372,7 +12389,7 @@ private IReadOnlyList FindInsertUniqueConflicts( ? $"UNIQUE constraint failed: index '{index.Name}'" : "UNIQUE constraint failed: " + string.Join(", ", index.Columns.Select(column => $"{tableName}.{column.Name}")); - if (Add(rowPosition, message, index.ConflictAlgorithm)) + if (Add(rowPosition, message, index.ConflictAlgorithm, UniqueIndexConflictCode(index))) return conflicts; } if (!primaryKeyAdded && table.WithoutRowid && AddWithoutRowidPrimaryKeyConflict()) @@ -12400,17 +12417,19 @@ bool AddWithoutRowidPrimaryKeyConflict() rowPosition, "UNIQUE constraint failed: " + string.Join(", ", primaryKey.Select(column => $"{tableName}.{column.Name}")), - table.PrimaryKeyConflictAlgorithm); + table.PrimaryKeyConflictAlgorithm, + SqliteResultCode.ConstraintPrimaryKey); } bool Add( int rowPosition, string message, - InsertConflictAlgorithm? algorithm) + InsertConflictAlgorithm? algorithm, + int sqliteErrorCode) { conflicts.Add(new InsertUniqueConflict( rowPosition, - new EmbeddedSqlException(message, algorithm, constraintViolation: true))); + new EmbeddedSqlException(message, algorithm, constraintViolation: true, sqliteErrorCode))); return shouldStop?.Invoke(conflicts) ?? false; } } @@ -13885,7 +13904,9 @@ private void ValidatePrimaryKey(string tableName, EmbeddedTable table, IReadOnly var columns = primaryKey.Select(entry => $"{tableName}.{table.Columns[entry.Index]}"); throw new EmbeddedSqlException( $"UNIQUE constraint failed: {string.Join(", ", columns)}", - table.PrimaryKeyConflictAlgorithm); + table.PrimaryKeyConflictAlgorithm, + constraintViolation: true, + SqliteResultCode.ConstraintPrimaryKey); } } @@ -13939,7 +13960,9 @@ private void ValidatePrimaryKey(string tableName, EmbeddedTable table, IReadOnly var columns = primaryKey.Select(entry => $"{tableName}.{table.Columns[entry.Index]}"); throw new EmbeddedSqlException( $"UNIQUE constraint failed: {string.Join(", ", columns)}", - table.PrimaryKeyConflictAlgorithm); + table.PrimaryKeyConflictAlgorithm, + constraintViolation: true, + SqliteResultCode.ConstraintPrimaryKey); } } @@ -14433,7 +14456,9 @@ private InsertPlan PrepareInsert( $"UNIQUE constraint failed: {statement.TableName}.{conflictColumn}", plan.AliasIndex >= 0 ? table.RowidAliasConflictAlgorithm - : null); + : null, + constraintViolation: true, + RowidConflictCode(plan.AliasIndex)); } } } @@ -14578,10 +14603,19 @@ private static void ValidateRowids( $"UNIQUE constraint failed: {tableName}.{conflictColumn}", aliasIndex >= 0 ? table.RowidAliasConflictAlgorithm - : null); + : null, + constraintViolation: true, + RowidConflictCode(aliasIndex)); } } + /// + /// SQLite's sqlite3RowidConstraint code: an INTEGER PRIMARY KEY alias collision is + /// SQLITE_CONSTRAINT_PRIMARYKEY, a hidden-rowid collision SQLITE_CONSTRAINT_ROWID. + /// + private static int RowidConflictCode(int aliasIndex) + => aliasIndex >= 0 ? SqliteResultCode.ConstraintPrimaryKey : SqliteResultCode.ConstraintRowId; + // Mutable per-statement INSERT plan: the resolved column targets plus the rowid // allocation state threaded across value rows. private sealed class InsertPlan @@ -17068,7 +17102,9 @@ private static void ValidateRowIdsUnique(string tableName, EmbeddedTable table, : null; throw new EmbeddedSqlException( $"UNIQUE constraint failed: {tableName}.{column}", - conflictAlgorithm); + conflictAlgorithm, + constraintViolation: true, + RowidConflictCode(aliasIndex)); } } } @@ -17128,7 +17164,9 @@ private void ValidateColumnUniqueConstraints( { throw new EmbeddedSqlException( $"UNIQUE constraint failed: {table.Name}.{column.Name}", - column.PrimaryKeyConflictAlgorithm); + column.PrimaryKeyConflictAlgorithm, + constraintViolation: true, + SqliteResultCode.ConstraintPrimaryKey); } } @@ -17153,7 +17191,9 @@ private void ValidateColumnUniqueConstraints( if (values.Any(existing => Compare(existing, value, column.Collation) == 0)) throw new EmbeddedSqlException( $"UNIQUE constraint failed: {table.Name}.{column.Name}", - column.PrimaryKeyConflictAlgorithm); + column.PrimaryKeyConflictAlgorithm, + constraintViolation: true, + SqliteResultCode.ConstraintPrimaryKey); values.Add(value); } @@ -17269,7 +17309,9 @@ private void ValidateUniqueIndex( + string.Join(", ", index.Columns.Select(column => $"{tableName}.{column.Name}")); throw new EmbeddedSqlException( message, - index.ConflictAlgorithm); + index.ConflictAlgorithm, + constraintViolation: true, + UniqueIndexConflictCode(index)); } } @@ -17331,7 +17373,9 @@ private void ValidateUniqueIndex( + string.Join(", ", index.Columns.Select(column => $"{tableName}.{column.Name}")); throw new EmbeddedSqlException( message, - index.ConflictAlgorithm); + index.ConflictAlgorithm, + constraintViolation: true, + UniqueIndexConflictCode(index)); } } @@ -17339,6 +17383,15 @@ private void ValidateUniqueIndex( } } + /// + /// SQLite's sqlite3UniqueConstraint code: a PRIMARY KEY's automatic index reports + /// SQLITE_CONSTRAINT_PRIMARYKEY, every other unique index SQLITE_CONSTRAINT_UNIQUE. + /// + private static int UniqueIndexConflictCode(EmbeddedIndex index) + => index.Origin == EmbeddedIndexOrigin.PrimaryKey + ? SqliteResultCode.ConstraintPrimaryKey + : SqliteResultCode.ConstraintUnique; + private bool RowsConflictOnIndex( EmbeddedTable table, EmbeddedIndex index, @@ -41765,7 +41818,7 @@ private SqlValue EvaluateRaise( QueryContext context) { var message = ResolveRaiseMessage(expression.Message, parameters, row, context); - var error = new EmbeddedSqlException(message); + var error = new EmbeddedSqlException(message, SqliteResultCode.ConstraintTrigger); throw expression.Action switch { RaiseAction.Ignore => new TriggerIgnoreException(), @@ -65053,6 +65106,26 @@ private ExecutionResult ExecutePragmaTempStore(PragmaTempStoreStatement statemen return ExecutionResult.Empty; } + /// + /// Turso's explicit-checkpoint guard (Connection::begin_explicit_checkpoint): an explicit + /// checkpoint must not overlap another root statement on the same connection, because it + /// resets pager state under that statement's cursors. The pragma runs outside its own reader + /// lease, so any registered reader is another active or suspended statement. Incremental + /// blob handles hold no statement reader and never block it, as upstream. + /// + private void ThrowIfStatementsActiveForCheckpoint() + { + lock (_statementReaderGate) + { + if (_statementReaders.Count == 0) + return; + } + + throw new EmbeddedSqlException( + "cannot checkpoint while another statement is active - SQL statements in progress", + SqliteResultCode.Busy); + } + private ExecutionResult ExecutePragmaWalCheckpoint(PragmaWalCheckpointStatement statement) { if (statement.Mode is { } mode) @@ -65075,6 +65148,7 @@ private ExecutionResult ExecutePragmaWalCheckpoint(PragmaWalCheckpointStatement ValidatePragmaSchema(statement.Schema); var database = ResolvePragmaDatabase(statement.Schema); + ThrowIfStatementsActiveForCheckpoint(); if (database.IsMvccEnabled) { // Managed MVCC checkpoint: materialize WAL pages -> backfill/flush @@ -66223,7 +66297,7 @@ private void ExecuteIfNeeded(CancellationToken cancellationToken = default) var mayMutate = _connection.StatementMayMutate(_statement); IDisposable? executionLease = _statement is - VacuumStatement or PragmaJournalModeStatement or PragmaPageSizeStatement + VacuumStatement or PragmaJournalModeStatement or PragmaPageSizeStatement or PragmaWalCheckpointStatement ? null : _connection.OpenStatementReaderLease(); try @@ -69416,7 +69490,9 @@ public void ValidateRows( { throw new EmbeddedSqlException( $"UNIQUE constraint failed: {tableName}.{column.Name}", - column.PrimaryKeyConflictAlgorithm); + column.PrimaryKeyConflictAlgorithm, + constraintViolation: true, + SqliteResultCode.ConstraintPrimaryKey); } } } diff --git a/src/Ahtola.Core/EmbeddedFileStore.cs b/src/Ahtola.Core/EmbeddedFileStore.cs index 92ce9317..dc6c9b4e 100644 --- a/src/Ahtola.Core/EmbeddedFileStore.cs +++ b/src/Ahtola.Core/EmbeddedFileStore.cs @@ -9957,7 +9957,11 @@ private void ValidateBoundedUniqueIndexRecords( { var columns = definition.Index.Columns.Select( column => $"{definition.TableName}.{column.Name}"); - throw new EmbeddedSqlException($"UNIQUE constraint failed: {string.Join(", ", columns)}"); + throw new EmbeddedSqlException( + $"UNIQUE constraint failed: {string.Join(", ", columns)}", + definition.Index.Origin == EmbeddedIndexOrigin.PrimaryKey + ? Execution.SqliteResultCode.ConstraintPrimaryKey + : Execution.SqliteResultCode.ConstraintUnique); } previousNonNullKey = key; diff --git a/src/Ahtola.Core/Execution/ResumableStatement.cs b/src/Ahtola.Core/Execution/ResumableStatement.cs index 4d938501..79f61fdb 100644 --- a/src/Ahtola.Core/Execution/ResumableStatement.cs +++ b/src/Ahtola.Core/Execution/ResumableStatement.cs @@ -1673,7 +1673,7 @@ public ResumableStatementStepResult StepResumable(CancellationToken cancellation { throw new EmbeddedSqlException( "datatype mismatch", - SqliteResultCode.Constraint, + SqliteResultCode.Mismatch, InsertConflictAlgorithm.Abort); } @@ -1704,7 +1704,7 @@ public ResumableStatementStepResult StepResumable(CancellationToken cancellation { throw new EmbeddedSqlException( "datatype mismatch", - SqliteResultCode.Constraint, + SqliteResultCode.Mismatch, InsertConflictAlgorithm.Abort); } diff --git a/src/Ahtola.Core/Execution/VdbeProgram.cs b/src/Ahtola.Core/Execution/VdbeProgram.cs index acb4d5a3..74d91058 100644 --- a/src/Ahtola.Core/Execution/VdbeProgram.cs +++ b/src/Ahtola.Core/Execution/VdbeProgram.cs @@ -709,13 +709,165 @@ public static class SqliteResultCode { public const int Ok = 0; public const int Error = 1; + public const int Abort = 4; + public const int Busy = 5; + public const int Locked = 6; + public const int ReadOnly = 8; + public const int Interrupt = 9; + public const int Corrupt = 11; + public const int Full = 13; + public const int CantOpen = 14; + public const int TooBig = 18; public const int Constraint = 19; + public const int Mismatch = 20; + public const int NotADatabase = 26; public const int ConstraintCheck = 275; public const int ConstraintNotNull = 1299; public const int ConstraintPrimaryKey = 1555; public const int ConstraintUnique = 2067; public const int ConstraintTrigger = 1811; public const int ConstraintForeignKey = 787; + public const int ConstraintRowId = 2579; + public const int ConstraintDataType = 3091; + + /// The primary result code of a possibly extended one (code & 0xFF). + public static int Primary(int code) => code & 0xFF; + + private static readonly Dictionary s_codesByName = new(StringComparer.OrdinalIgnoreCase) + { + ["SQLITE_ERROR"] = 1, + ["SQLITE_INTERNAL"] = 2, + ["SQLITE_PERM"] = 3, + ["SQLITE_ABORT"] = 4, + ["SQLITE_BUSY"] = 5, + ["SQLITE_LOCKED"] = 6, + ["SQLITE_NOMEM"] = 7, + ["SQLITE_READONLY"] = 8, + ["SQLITE_INTERRUPT"] = 9, + ["SQLITE_IOERR"] = 10, + ["SQLITE_CORRUPT"] = 11, + ["SQLITE_NOTFOUND"] = 12, + ["SQLITE_FULL"] = 13, + ["SQLITE_CANTOPEN"] = 14, + ["SQLITE_PROTOCOL"] = 15, + ["SQLITE_EMPTY"] = 16, + ["SQLITE_SCHEMA"] = 17, + ["SQLITE_TOOBIG"] = 18, + ["SQLITE_CONSTRAINT"] = 19, + ["SQLITE_MISMATCH"] = 20, + ["SQLITE_MISUSE"] = 21, + ["SQLITE_NOLFS"] = 22, + ["SQLITE_AUTH"] = 23, + ["SQLITE_FORMAT"] = 24, + ["SQLITE_RANGE"] = 25, + ["SQLITE_NOTADB"] = 26, + ["SQLITE_ERROR_MISSING_COLLSEQ"] = 257, + ["SQLITE_ERROR_RETRY"] = 513, + ["SQLITE_ERROR_SNAPSHOT"] = 769, + ["SQLITE_BUSY_RECOVERY"] = 261, + ["SQLITE_BUSY_SNAPSHOT"] = 517, + ["SQLITE_BUSY_TIMEOUT"] = 773, + ["SQLITE_LOCKED_SHAREDCACHE"] = 262, + ["SQLITE_LOCKED_VTAB"] = 518, + ["SQLITE_READONLY_RECOVERY"] = 264, + ["SQLITE_READONLY_CANTLOCK"] = 520, + ["SQLITE_READONLY_ROLLBACK"] = 776, + ["SQLITE_READONLY_DBMOVED"] = 1032, + ["SQLITE_READONLY_CANTINIT"] = 1288, + ["SQLITE_READONLY_DIRECTORY"] = 1544, + ["SQLITE_ABORT_ROLLBACK"] = 516, + ["SQLITE_CORRUPT_VTAB"] = 267, + ["SQLITE_CORRUPT_SEQUENCE"] = 523, + ["SQLITE_CORRUPT_INDEX"] = 779, + ["SQLITE_CANTOPEN_NOTEMPDIR"] = 270, + ["SQLITE_CANTOPEN_ISDIR"] = 526, + ["SQLITE_CANTOPEN_FULLPATH"] = 782, + ["SQLITE_CANTOPEN_CONVPATH"] = 1038, + ["SQLITE_CANTOPEN_DIRTYWAL"] = 1294, + ["SQLITE_CANTOPEN_SYMLINK"] = 1550, + ["SQLITE_CONSTRAINT_CHECK"] = ConstraintCheck, + ["SQLITE_CONSTRAINT_COMMITHOOK"] = 531, + ["SQLITE_CONSTRAINT_FOREIGNKEY"] = ConstraintForeignKey, + ["SQLITE_CONSTRAINT_FUNCTION"] = 1043, + ["SQLITE_CONSTRAINT_NOTNULL"] = ConstraintNotNull, + ["SQLITE_CONSTRAINT_PRIMARYKEY"] = ConstraintPrimaryKey, + ["SQLITE_CONSTRAINT_TRIGGER"] = ConstraintTrigger, + ["SQLITE_CONSTRAINT_UNIQUE"] = ConstraintUnique, + ["SQLITE_CONSTRAINT_VTAB"] = 2323, + ["SQLITE_CONSTRAINT_ROWID"] = ConstraintRowId, + ["SQLITE_CONSTRAINT_PINNED"] = 2835, + ["SQLITE_CONSTRAINT_DATATYPE"] = ConstraintDataType, + }; + + /// + /// Resolves a symbolic SQLite result code name as remote protocols report it (Hrana's + /// error.code, e.g. SQLITE_CONSTRAINT_UNIQUE) to its numeric value. An + /// extended name this table does not list resolves to its primary family + /// (SQLITE_IOERR_SHORT_READ → 10). Non-SQLite codes (SQL_PARSE_ERROR, + /// STREAM_EXPIRED) are not SQLite result codes and return . + /// + public static bool TryParseName(string? name, out int code) + { + code = 0; + if (string.IsNullOrWhiteSpace(name)) + return false; + + name = name.Trim(); + if (s_codesByName.TryGetValue(name, out code)) + return true; + + for (var separator = name.LastIndexOf('_'); separator > "SQLITE_".Length; separator = name.LastIndexOf('_', separator - 1)) + { + if (s_codesByName.TryGetValue(name[..separator], out code) && code == Primary(code)) + return true; + } + + code = 0; + return false; + } + + /// + /// Derives SQLite's (extended) result code from the exact error text SQLite reports for + /// that failure class, for raise sites that carry only the message. Returns + /// for text that is an ordinary SQLITE_ERROR. Primary-key + /// conflicts share the "UNIQUE constraint failed" text, so their raise sites pass + /// explicitly. + /// + public static int? InferFromMessage(string? message) + { + if (string.IsNullOrEmpty(message)) + return null; + + if (message.StartsWith("UNIQUE constraint failed:", StringComparison.Ordinal)) + return ConstraintUnique; + if (message.StartsWith("NOT NULL constraint failed:", StringComparison.Ordinal)) + return ConstraintNotNull; + if (message.StartsWith("CHECK constraint failed:", StringComparison.Ordinal)) + return ConstraintCheck; + if (message.Equals("FOREIGN KEY constraint failed", StringComparison.Ordinal)) + return ConstraintForeignKey; + if (message.StartsWith("cannot store ", StringComparison.Ordinal) + && message.Contains(" value in ", StringComparison.Ordinal) + && message.Contains(" column ", StringComparison.Ordinal)) + { + return ConstraintDataType; + } + + return message switch + { + "datatype mismatch" => Mismatch, + "attempt to write a readonly database" => ReadOnly, + "database is locked" => Busy, + "database table is locked" => Locked, + "database or disk is full" => Full, + "string or blob too big" => TooBig, + "database disk image is malformed" => Corrupt, + "file is not a database" => NotADatabase, + "unable to open database file" => CantOpen, + "interrupted" => Interrupt, + _ => null, + }; + } } /// diff --git a/src/Ahtola.Core/ManagedSequence.cs b/src/Ahtola.Core/ManagedSequence.cs index 0c9fd595..3c452211 100644 --- a/src/Ahtola.Core/ManagedSequence.cs +++ b/src/Ahtola.Core/ManagedSequence.cs @@ -90,8 +90,10 @@ public long ComputeNext(long current, bool isCalled, bool wasEmpty) if (Cycle) return Ascending ? MinValue : MaxValue; + // Upstream's SequenceExhausted: its own text, but SQLITE_FULL (13) like DatabaseFull. throw new EmbeddedSqlException( - $"nextval: reached {(Ascending ? "maximum" : "minimum")} value of sequence \"{Name}\""); + $"nextval: reached {(Ascending ? "maximum" : "minimum")} value of sequence \"{Name}\"", + Execution.SqliteResultCode.Full); } return checked(current + IncrementBy); diff --git a/src/Ahtola.Core/Search/ManagedFtsFunctions.cs b/src/Ahtola.Core/Search/ManagedFtsFunctions.cs index f31ec155..0e09cca6 100644 --- a/src/Ahtola.Core/Search/ManagedFtsFunctions.cs +++ b/src/Ahtola.Core/Search/ManagedFtsFunctions.cs @@ -906,9 +906,8 @@ private static (IReadOnlyList Columns, string? Query) Split(string fun var query = arguments[^1]; if (query.Kind == SqlValueKind.Null) return (arguments.Take(arguments.Count - 1).ToArray(), null); - if (query.Kind != SqlValueKind.Text) - throw new EmbeddedSqlException($"{function}() requires a text query"); - return (arguments.Take(arguments.Count - 1).ToArray(), query.AsText()); + // Turso converts a non-text query with Value::to_string rather than rejecting it. + return (arguments.Take(arguments.Count - 1).ToArray(), ManagedFtsPlannerAdapter.RequireQueryText(query)); } } diff --git a/src/Ahtola.Core/Search/ManagedFtsPlannerAdapter.cs b/src/Ahtola.Core/Search/ManagedFtsPlannerAdapter.cs index 67901d5c..9b026000 100644 --- a/src/Ahtola.Core/Search/ManagedFtsPlannerAdapter.cs +++ b/src/Ahtola.Core/Search/ManagedFtsPlannerAdapter.cs @@ -85,7 +85,6 @@ public bool TryMatch( shape, matchExpression, FiltersRows: true, - ValidateArgument: ValidateMatchArgument, RetainsUnrankedRows: false); return true; } @@ -107,37 +106,20 @@ public bool TryMatch( ManagedIndexPatternShape.Score, orderedScoreExpression, FiltersRows: false, - ValidateArgument: ValidateScoreArgument, UnrankedMergePolicy: ManagedIndexUnrankedMergePolicy.MergeByDescendingRank, UnrankedRank: 0.0); return true; } /// - /// Reproduces the exact type error fts_match() raises for the same argument, so choosing - /// the index can never turn a scalar error into an empty result set. + /// Reads a non-NULL query argument as text. Like Turso's FtsCursor::query_start and + /// scalar fts_match (upstream d485aa6d8), an integer, real or blob query is searched + /// by its SQL text form rather than rejected; NULL matches nothing and never reaches here. /// - private static void ValidateMatchArgument(SqlValue value) - { - if (value.Kind is SqlValueKind.Null or SqlValueKind.Text) - return; - - throw new EmbeddedSqlException("fts_match() requires a text query"); - } - - private static void ValidateScoreArgument(SqlValue value) - { - if (value.Kind is SqlValueKind.Null or SqlValueKind.Text) - return; - - throw new EmbeddedSqlException("fts_score() requires a text query"); - } - - /// Reads a query argument that already passed validation. public static string RequireQueryText(SqlValue value) - => value.Kind == SqlValueKind.Text - ? value.AsText() - : throw new EmbeddedSqlException("fts query must be text"); + => value.Kind == SqlValueKind.Null + ? throw new EmbeddedSqlException("fts query must not be NULL") + : EmbeddedDatabase.ToSqlText(value); /// /// Finds fts_match(cols…, query) among the conjuncts of a WHERE predicate, and reports diff --git a/src/Ahtola.Core/Storage/SqliteDatabaseHeader.cs b/src/Ahtola.Core/Storage/SqliteDatabaseHeader.cs index ff92def9..afa296f6 100644 --- a/src/Ahtola.Core/Storage/SqliteDatabaseHeader.cs +++ b/src/Ahtola.Core/Storage/SqliteDatabaseHeader.cs @@ -70,14 +70,25 @@ public static SqliteDatabaseHeader Parse(ReadOnlySpan source) { if (source.Length < Size) throw new InvalidDataException("SQLite database header is truncated."); + // SQLite's lockBtree "not a database" rules (bad magic, invalid page size, fewer than + // 480 usable bytes per page) report SQLITE_NOTADB rather than the field that tripped. if (!source[..Magic.Length].SequenceEqual(Magic)) - throw new InvalidDataException("Database does not contain an SQLite format 3 header."); + throw SqliteNotADatabaseException.Create("Database does not contain an SQLite format 3 header."); + + int pageSize; + try + { + pageSize = SqlitePageSize.Decode(BinaryPrimitives.ReadUInt16BigEndian(source[16..])); + } + catch (InvalidDataException exception) + { + throw SqliteNotADatabaseException.Create(exception.Message); + } - var pageSize = SqlitePageSize.Decode(BinaryPrimitives.ReadUInt16BigEndian(source[16..])); var writeVersion = ParseFileFormatVersion(source[18], "write"); var readVersion = ParseFileFormatVersion(source[19], "read"); var reservedSpace = source[20]; - ValidateUsableSpace(pageSize, reservedSpace, static message => new InvalidDataException(message)); + ValidateUsableSpace(pageSize, reservedSpace, static message => SqliteNotADatabaseException.Create(message)); if (source[21] != 64 || source[22] != 32 || source[23] != 32) throw new InvalidDataException("SQLite payload-fraction fields are invalid."); @@ -336,3 +347,25 @@ private static void ValidateUsableSpace( } } } + +/// +/// Marks a file whose plain SQLite header fails SQLite's "not a database" rules: bad magic, +/// an invalid page size, too few usable bytes per page, or a file too short to hold a header. +/// SQLite reports these as SQLITE_NOTADB ("file is not a database"). The engine still +/// raises an naming the specific rule; this marker rides as +/// its inner exception so the SQLite facade can report the result code. +/// +public sealed class SqliteNotADatabaseException : Exception +{ + /// SQLite's text for SQLITE_NOTADB. + public const string SqliteMessage = "file is not a database"; + + public SqliteNotADatabaseException() + : base(SqliteMessage) + { + } + + /// Creates the header-validation failure for . + internal static InvalidDataException Create(string detail) + => new(detail, new SqliteNotADatabaseException()); +} diff --git a/src/Ahtola.Core/Storage/SqlitePageStore.cs b/src/Ahtola.Core/Storage/SqlitePageStore.cs index 5856e8fe..687e3603 100644 --- a/src/Ahtola.Core/Storage/SqlitePageStore.cs +++ b/src/Ahtola.Core/Storage/SqlitePageStore.cs @@ -117,7 +117,15 @@ private static SqlitePageStore OpenCore( { var length = file.Length; if (length < SqliteDatabaseHeader.Size) - throw new InvalidDataException("File is too small to contain a SQLite database header."); + { + // SQLite zero-fills a short page 1 and then applies its header rules, so a file + // too short for a plain header is "not a database". A page codec may transform + // the magic, so it keeps the short-read diagnostic and judges the header itself. + const string tooSmall = "File is too small to contain a SQLite database header."; + throw pageCodec is null && encryption is null + ? SqliteNotADatabaseException.Create(tooSmall) + : new InvalidDataException(tooSmall); + } Span rawHeader = stackalloc byte[SqliteDatabaseHeader.Size]; if (file.Read(0, rawHeader) != SqliteDatabaseHeader.Size) diff --git a/src/Ahtola.Data.Sqlite/SqliteCommand.cs b/src/Ahtola.Data.Sqlite/SqliteCommand.cs index 67fc807e..9a02f4ad 100644 --- a/src/Ahtola.Data.Sqlite/SqliteCommand.cs +++ b/src/Ahtola.Data.Sqlite/SqliteCommand.cs @@ -6,6 +6,7 @@ using System.Text.RegularExpressions; using Ahtola; using Ahtola.Core; +using Ahtola.Core.Execution; using Ahtola.Core.Parsing; namespace Ahtola.Data.Sqlite; @@ -1761,12 +1762,7 @@ internal static SqliteException ToSqliteException(Exception ex, string? sql = nu return new SqliteException(Properties.Resources.SqliteNativeError(5, ex.Message), 5); if (TryGetSqliteErrorCode(ex) is { } sqliteErrorCode) - { - var codedMessage = UnwrapMessage(ex); - return new SqliteException( - Properties.Resources.SqliteNativeError(sqliteErrorCode, codedMessage), - sqliteErrorCode); - } + return CreateSqliteException(sqliteErrorCode, UnwrapMessage(ex)); var message = ex.Message; foreach (var prefix in new[] { "Unable to prepare statement: Parse error: ", "Parse error: " }) @@ -1789,14 +1785,57 @@ internal static SqliteException ToSqliteException(Exception ex, string? sql = nu && int.TryParse(message[sqliteErrorPrefix.Length..codeEnd], NumberStyles.Integer, CultureInfo.InvariantCulture, out var errorCode)) { var sqliteMessage = message[(codeEnd + 1)..]; - return new SqliteException(Properties.Resources.SqliteNativeError(errorCode, sqliteMessage), errorCode); + return CreateSqliteException(errorCode, sqliteMessage); } } if (sql is not null) message = PreserveNoSuchTableCase(message, sql); - return new SqliteException(Properties.Resources.SqliteNativeError(1, message), 1); + return CreateSqliteException(TryGetRemoteSqliteErrorCode(ex) ?? SqliteResultCode.Error, message); + } + + /// + /// Maps a remote (Hrana) failure's symbolic error.code to SQLite's numeric result + /// code. A bare primary SQLITE_CONSTRAINT is refined from the remote message the + /// way a local failure is, so a server that reports only the primary family still yields + /// the extended constraint kind. + /// + private static int? TryGetRemoteSqliteErrorCode(Exception ex) + { + for (var current = ex; current is not null; current = current.InnerException) + { + if (current is not AhtolaRemoteSqlException remote + || !SqliteResultCode.TryParseName(remote.RemoteErrorCode, out var code)) + { + continue; + } + + if (code == SqliteResultCode.Constraint + && SqliteResultCode.InferFromMessage(remote.RemoteErrorMessage) is { } inferred + && SqliteResultCode.Primary(inferred) == SqliteResultCode.Constraint) + { + return inferred; + } + + return code; + } + + return null; + } + + /// + /// Builds the facade exception the way Microsoft.Data.Sqlite does for a failed step: + /// SqliteErrorCode and the "SQLite Error N" text carry the primary result code, and + /// SqliteExtendedErrorCode the extended one (e.g. 19 / 2067 for a UNIQUE violation). + /// + internal static SqliteException CreateSqliteException(int resultCode, string message) + { + var primary = SqliteResultCode.Primary(resultCode); + return new SqliteException( + Properties.Resources.SqliteNativeError(primary, message), + primary, + resultCode); } private static int? TryGetSqliteErrorCode(Exception ex) diff --git a/src/Ahtola.Data.Sqlite/SqliteConnection.cs b/src/Ahtola.Data.Sqlite/SqliteConnection.cs index 15bc1a00..786e189d 100644 --- a/src/Ahtola.Data.Sqlite/SqliteConnection.cs +++ b/src/Ahtola.Data.Sqlite/SqliteConnection.cs @@ -6,6 +6,7 @@ using System.Security.Cryptography; using Ahtola; using Ahtola.Core; +using Ahtola.Core.Execution; using Ahtola.Core.Storage; namespace Ahtola.Data.Sqlite; @@ -344,6 +345,15 @@ public override void Open() CleanupFailedOpen(sharedMemoryPath); throw MapManagedEncryptionOpenFailure(ex, managedEncryption is not null || useManaged); } + catch (InvalidDataException ex) when (managedEncryption is null && PageCodec is null && IsPlainNotADatabase(ex)) + { + // An unkeyed, codec-less open of a garbage plain header is SQLite's SQLITE_NOTADB. + // A configured key or codec keeps the encrypted-or-not-a-database phrase below. + CleanupFailedOpen(sharedMemoryPath); + throw SqliteCommand.CreateSqliteException( + SqliteResultCode.NotADatabase, + SqliteNotADatabaseException.SqliteMessage); + } catch (InvalidDataException ex) { CleanupFailedOpen(sharedMemoryPath); @@ -1934,6 +1944,17 @@ private static void DeleteDatabaseSidecars(string databasePath) } } + private static bool IsPlainNotADatabase(Exception exception) + { + for (var current = exception; current is not null; current = current.InnerException) + { + if (current is SqliteNotADatabaseException) + return true; + } + + return false; + } + private static SqliteException ToSqliteException(AhtolaException exception) { var message = exception.Message; diff --git a/src/Ahtola.Data.Sqlite/SqliteRemoteException.cs b/src/Ahtola.Data.Sqlite/SqliteRemoteException.cs index 78da449c..f1842b2e 100644 --- a/src/Ahtola.Data.Sqlite/SqliteRemoteException.cs +++ b/src/Ahtola.Data.Sqlite/SqliteRemoteException.cs @@ -21,9 +21,10 @@ public sealed class SqliteRemoteException : SqliteException internal SqliteRemoteException( string message, int errorCode, + int extendedErrorCode, SqliteRemoteErrorClassification classification, HttpStatusCode? httpStatusCode) - : base(message, errorCode) + : base(message, errorCode, extendedErrorCode) { Classification = classification; HttpStatusCode = httpStatusCode; @@ -77,6 +78,11 @@ internal static SqliteRemoteException From(Exception source, SqliteException map { var statusCode = source is AhtolaException ahtola ? ahtola.RemoteStatusCode : null; var classification = Classify(source); - return new SqliteRemoteException(mapped.Message, mapped.SqliteErrorCode, classification, statusCode); + return new SqliteRemoteException( + mapped.Message, + mapped.SqliteErrorCode, + mapped.SqliteExtendedErrorCode, + classification, + statusCode); } } diff --git a/src/Ahtola.Tests/ManagedIndexMethodReviewRegressionTests.cs b/src/Ahtola.Tests/ManagedIndexMethodReviewRegressionTests.cs index c5450924..439b5c08 100644 --- a/src/Ahtola.Tests/ManagedIndexMethodReviewRegressionTests.cs +++ b/src/Ahtola.Tests/ManagedIndexMethodReviewRegressionTests.cs @@ -777,22 +777,87 @@ public void AGramIndexMatchesWholeWordQueries() // Finding 16: the indexed path preserves scalar type and error semantics. // --------------------------------------------------------------------------------------- - [Test] - public void ANumericQueryArgumentErrorsOnBothPaths() + // Turso d485aa6d8 ("fts: accept null and non-text search inputs"): a non-NULL query of any + // storage class is searched by its text form on both paths, mirroring + // test_fts_query_input_types / test_fts_bound_query_input_types. + [TestCase(false)] + [TestCase(true)] + public void NonTextQueryLiteralsAreSearchedAsTextOnBothPaths(bool indexed) { using var database = new EmbeddedDatabase(); using var connection = database.Connect(); - SeedSparseCorpus(connection); + SeedNumericBodies(connection, indexed); - // Indexed path. - ExplainDetail(connection, "SELECT id FROM docs WHERE fts_match(title, body, 'needle');") - .Should().Contain("INDEX METHOD"); - ShouldThrow(connection, "SELECT id FROM docs WHERE fts_match(title, body, 123);") - .Message.Should().Contain("requires a text query"); + foreach (var (query, expected) in new[] { ("NULL", Array.Empty()), ("42", [1L]), ("7.5", [3L]), ("x'3432'", [1L]) }) + { + foreach (var suffix in new[] { string.Empty, " LIMIT 10" }) + { + var sql = $"SELECT id FROM d WHERE fts_match(body, {query}){suffix};"; + QueryIntegers(connection, sql).Should().Equal(expected, sql); + } + } + } - Execute(connection, "DROP INDEX docs_fts;"); - ShouldThrow(connection, "SELECT id FROM docs WHERE fts_match(title, body, 123);") - .Message.Should().Contain("requires a text query"); + [TestCase(false)] + [TestCase(true)] + public void NonTextBoundQueriesAreSearchedAsTextOnBothPaths(bool indexed) + { + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + SeedNumericBodies(connection, indexed); + + string[] statements = + [ + "SELECT id FROM d WHERE fts_match(body, ?1)", + "SELECT id FROM d WHERE fts_match(body, ?1) LIMIT 10", + "SELECT id, fts_score(body, ?1) AS score FROM d WHERE fts_match(body, ?1)", + "SELECT id, fts_score(body, ?1) AS score FROM d WHERE fts_match(body, ?1) LIMIT 10", + "SELECT id, fts_score(body, ?1) AS score FROM d WHERE fts_match(body, ?1) ORDER BY score DESC", + "SELECT id, fts_score(body, ?1) AS score FROM d WHERE fts_match(body, ?1) ORDER BY score DESC LIMIT 10", + ]; + (SqlValue Query, long[] Expected)[] inputs = + [ + (SqlValue.Text("42"), [1L]), + (SqlValue.Null, []), + (SqlValue.Integer(42), [1L]), + (SqlValue.Real(7.5), [3L]), + (SqlValue.Blob("42"u8.ToArray()), [1L]), + (SqlValue.Null, []), + ]; + + foreach (var sql in statements) + { + using var statement = connection.Prepare(sql); + foreach (var (query, expected) in inputs) + { + statement.Reset(); + statement.Bind(1, query); + var ids = new List(); + while (statement.Step() == StatementStepResult.Row) + { + ids.Add(statement.GetValue(0).AsInteger()); + if (indexed && statement.ColumnCount > 1) + statement.GetValue(1).AsReal().Should().BeGreaterThan(0.0, sql); + } + + ids.Should().Equal(expected, $"{sql} with {query.Kind}"); + } + } + } + + private static void SeedNumericBodies(EmbeddedConnection connection, bool indexed) + { + Execute(connection, "CREATE TABLE d(id INTEGER PRIMARY KEY, body TEXT);"); + if (indexed) + Execute(connection, "CREATE INDEX fx ON d USING fts(body);"); + Execute(connection, "INSERT INTO d VALUES (1, '42'), (2, NULL), (3, '7.5');"); + for (var id = 4; id <= 400; id++) + Execute(connection, $"INSERT INTO d VALUES ({id}, 'filler text');"); + if (indexed) + { + ExplainDetail(connection, "SELECT id FROM d WHERE fts_match(body, '42');") + .Should().Contain("INDEX METHOD"); + } } [Test] diff --git a/src/Ahtola.Tests/MvccCheckpointStateMachineTests.cs b/src/Ahtola.Tests/MvccCheckpointStateMachineTests.cs index 32c5dafe..5ecaf107 100644 --- a/src/Ahtola.Tests/MvccCheckpointStateMachineTests.cs +++ b/src/Ahtola.Tests/MvccCheckpointStateMachineTests.cs @@ -80,6 +80,67 @@ public void PassiveCheckpointReportsBusyWhileConcurrentTxOpen() } } + // Turso a95d01284 / 7ad5d9de4: an explicit checkpoint must not overlap another root + // statement on the same connection, so a suspended reader keeps its cursors intact. + [TestCase(false)] + [TestCase(true)] + public void ExplicitCheckpointIsRefusedWhileAnotherStatementOnTheConnectionIsActive(bool mvcc) + { + using var db = new CheckpointFileDatabase(); + using var connection = db.Connect(); + if (mvcc) + connection.ExecuteNonQuery("PRAGMA journal_mode=mvcc;"); + connection.ExecuteNonQuery("INSERT INTO t VALUES (1), (2), (3);"); + + using (var select = connection.CreateCommand()) + { + select.CommandText = "SELECT v FROM t ORDER BY v;"; + using var reader = select.ExecuteReader(); + reader.Read().Should().BeTrue(); + + foreach (var mode in new[] { "PASSIVE", "TRUNCATE" }) + { + var checkpoint = () => connection.ExecuteNonQuery($"PRAGMA wal_checkpoint({mode});"); + var failure = checkpoint.Should().Throw().Which; + failure.SqliteErrorCode.Should().Be(5); + failure.SqliteExtendedErrorCode.Should().Be(5); + failure.Message.Should().Contain( + "cannot checkpoint while another statement is active - SQL statements in progress"); + } + + var remaining = new List(); + while (reader.Read()) + remaining.Add(reader.GetInt64(0)); + remaining.Should().Equal(2, 3); + } + + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = "PRAGMA wal_checkpoint(TRUNCATE);"; + using var reader = cmd.ExecuteReader(); + reader.Read().Should().BeTrue(); + Convert.ToInt64(reader.GetValue(0)).Should().Be(0L); + } + + Convert.ToInt64(Scalar(connection, "SELECT SUM(v) FROM t;")).Should().Be(6L); + } + + [TestCase(false)] + [TestCase(true)] + public void OpenBlobHandleDoesNotBlockExplicitCheckpoints(bool mvcc) + { + using var db = new CheckpointFileDatabase(); + using var connection = db.Connect(); + if (mvcc) + connection.ExecuteNonQuery("PRAGMA journal_mode=mvcc;"); + connection.ExecuteNonQuery("CREATE TABLE b(x BLOB);"); + connection.ExecuteNonQuery("INSERT INTO b VALUES (x'00112233');"); + + using var blob = new SqliteBlob(connection, "b", "x", 1, readOnly: true); + var checkpoint = () => connection.ExecuteNonQuery("PRAGMA wal_checkpoint(PASSIVE);"); + checkpoint.Should().NotThrow(); + } + [Test] public void TruncateAfterDeletesLeavesCatalogEmptyOnReopen() { diff --git a/src/Ahtola.Tests/SqliteErrorCodeDifferentialTests.cs b/src/Ahtola.Tests/SqliteErrorCodeDifferentialTests.cs new file mode 100644 index 00000000..81353105 --- /dev/null +++ b/src/Ahtola.Tests/SqliteErrorCodeDifferentialTests.cs @@ -0,0 +1,433 @@ +using AwesomeAssertions; +using ManagedSqlite = Ahtola.Data.Sqlite; +using NativeSqlite = Microsoft.Data.Sqlite; + +namespace Ahtola.Tests; + +/// +/// Pins the primary and extended result codes the Microsoft.Data.Sqlite-compatible +/// facade reports for engine failures against the bundled e_sqlite3 through +/// Microsoft.Data.Sqlite. Consumers branch on SqliteErrorCode / +/// SqliteExtendedErrorCode (for example 19/2067 for a UNIQUE violation), so every +/// failure class here must surface the same pair SQLite does, not a generic 1/1. +/// +[NonParallelizable] +public sealed class SqliteErrorCodeDifferentialTests +{ + public static IEnumerable StatementFailures() + { + yield return Case("unique-column", "CREATE TABLE t(a UNIQUE); INSERT INTO t VALUES (1);", "INSERT INTO t VALUES (1);", 19, 2067); + yield return Case("unique-index", "CREATE TABLE t(a); CREATE UNIQUE INDEX t_a ON t(a); INSERT INTO t VALUES (1);", "INSERT INTO t VALUES (1);", 19, 2067); + yield return Case("unique-update", "CREATE TABLE t(a UNIQUE); INSERT INTO t VALUES (1), (2);", "UPDATE t SET a = 1 WHERE a = 2;", 19, 2067); + yield return Case("rowid-primary-key", "CREATE TABLE t(id INTEGER PRIMARY KEY); INSERT INTO t VALUES (1);", "INSERT INTO t VALUES (1);", 19, 1555); + yield return Case("text-primary-key", "CREATE TABLE t(a TEXT PRIMARY KEY); INSERT INTO t VALUES ('x');", "INSERT INTO t VALUES ('x');", 19, 1555); + yield return Case("without-rowid-primary-key", "CREATE TABLE t(a TEXT PRIMARY KEY, b) WITHOUT ROWID; INSERT INTO t VALUES ('x', 1);", "INSERT INTO t VALUES ('x', 2);", 19, 1555); + yield return Case("not-null", "CREATE TABLE t(a NOT NULL);", "INSERT INTO t VALUES (NULL);", 19, 1299); + yield return Case("not-null-update", "CREATE TABLE t(a NOT NULL); INSERT INTO t VALUES (1);", "UPDATE t SET a = NULL;", 19, 1299); + yield return Case("check-column", "CREATE TABLE t(a CHECK (a > 0));", "INSERT INTO t VALUES (0);", 19, 275); + yield return Case("check-table", "CREATE TABLE t(a, b, CONSTRAINT positive CHECK (a < b));", "INSERT INTO t VALUES (2, 1);", 19, 275); + yield return Case( + "foreign-key-immediate", + "PRAGMA foreign_keys = ON; CREATE TABLE p(id INTEGER PRIMARY KEY); CREATE TABLE c(pid REFERENCES p(id));", + "INSERT INTO c VALUES (1);", + 19, + 787); + yield return Case( + "foreign-key-parent-delete", + "PRAGMA foreign_keys = ON; CREATE TABLE p(id INTEGER PRIMARY KEY); CREATE TABLE c(pid REFERENCES p(id)); INSERT INTO p VALUES (1); INSERT INTO c VALUES (1);", + "DELETE FROM p;", + 19, + 787); + yield return Case( + "foreign-key-deferred-commit", + "PRAGMA foreign_keys = ON; CREATE TABLE p(id INTEGER PRIMARY KEY); CREATE TABLE c(pid REFERENCES p(id) DEFERRABLE INITIALLY DEFERRED);", + "BEGIN; INSERT INTO c VALUES (1); COMMIT;", + 19, + 787); + yield return Case( + "trigger-raise-abort", + "CREATE TABLE t(a); CREATE TRIGGER tr BEFORE INSERT ON t BEGIN SELECT RAISE(ABORT, 'nope'); END;", + "INSERT INTO t VALUES (1);", + 19, + 1811); + yield return Case( + "trigger-raise-fail", + "CREATE TABLE t(a); CREATE TRIGGER tr BEFORE INSERT ON t BEGIN SELECT RAISE(FAIL, 'nope'); END;", + "INSERT INTO t VALUES (1);", + 19, + 1811); + yield return Case( + "trigger-raise-rollback", + "CREATE TABLE t(a); CREATE TRIGGER tr BEFORE INSERT ON t BEGIN SELECT RAISE(ROLLBACK, 'nope'); END;", + "INSERT INTO t VALUES (1);", + 19, + 1811); + yield return Case("hidden-rowid", "CREATE TABLE t(a); INSERT INTO t(rowid, a) VALUES (1, 1);", "INSERT INTO t(rowid, a) VALUES (1, 2);", 19, 2579); + yield return Case("insert-select-duplicate", "CREATE TABLE t(a UNIQUE); CREATE TABLE s(a); INSERT INTO s VALUES (1), (1);", "INSERT INTO t SELECT a FROM s;", 19, 2067); + yield return Case("or-fail-unique", "CREATE TABLE t(a UNIQUE); INSERT INTO t VALUES (1);", "INSERT OR FAIL INTO t VALUES (1);", 19, 2067); + yield return Case("or-rollback-primary-key", "CREATE TABLE t(id INTEGER PRIMARY KEY); INSERT INTO t VALUES (1);", "BEGIN; INSERT OR ROLLBACK INTO t VALUES (1);", 19, 1555); + yield return Case( + "upsert-update-unique", + "CREATE TABLE t(id INTEGER PRIMARY KEY, a UNIQUE); INSERT INTO t VALUES (1, 1), (2, 2);", + "INSERT INTO t VALUES (1, 5) ON CONFLICT(id) DO UPDATE SET a = 2;", + 19, + 2067); + yield return Case("composite-primary-key", "CREATE TABLE t(a, b, PRIMARY KEY (a, b)); INSERT INTO t VALUES (1, 2);", "INSERT INTO t VALUES (1, 2);", 19, 1555); + yield return Case("strict-datatype", "CREATE TABLE t(a INTEGER) STRICT;", "INSERT INTO t VALUES ('abc');", 19, 3091); + yield return Case("rowid-mismatch", "CREATE TABLE t(id INTEGER PRIMARY KEY);", "INSERT INTO t VALUES ('abc');", 20, 20); + yield return Case("syntax-error", string.Empty, "SELEC 1;", 1, 1); + yield return Case("no-such-table", string.Empty, "SELECT * FROM missing;", 1, 1); + yield return Case("no-such-column", "CREATE TABLE t(a);", "SELECT b FROM t;", 1, 1); + yield return Case("table-exists", "CREATE TABLE t(a);", "CREATE TABLE t(a);", 1, 1); + } + + [TestCaseSource(nameof(StatementFailures))] + public void StatementFailureCodesMatchSqlite(string setup, string failing, int expectedPrimary, int expectedExtended) + { + var native = CaptureNative(setup, failing); + var managed = CaptureManaged(setup, failing); + + native.Should().Be((expectedPrimary, expectedExtended), "the case must pin SQLite's own result codes"); + managed.Should().Be(native, "the managed facade must report SQLite's primary and extended result codes"); + } + + [TestCaseSource(nameof(StatementFailures))] + public void FileBackedStatementFailureCodesMatchSqlite(string setup, string failing, int expectedPrimary, int expectedExtended) + { + var nativePath = CreateEmptyPath(); + var managedPath = CreateEmptyPath(); + try + { + (int, int) native; + using (var connection = new NativeSqlite.SqliteConnection($"Data Source={nativePath};Pooling=False")) + { + connection.Open(); + if (setup.Length != 0) + ExecuteNative(connection, setup); + native = CaptureNative(connection, failing); + } + + (int, int) managed; + using (var connection = new ManagedSqlite.SqliteConnection($"Data Source={managedPath};Pooling=False")) + { + connection.Open(); + if (setup.Length != 0) + ExecuteManaged(connection, setup); + managed = CaptureManaged(connection, failing); + } + + native.Should().Be((expectedPrimary, expectedExtended), "the case must pin SQLite's own result codes"); + managed.Should().Be(native, "the page-backed managed engine must report SQLite's result codes"); + } + finally + { + DeleteDatabase(nativePath); + DeleteDatabase(managedPath); + } + } + + [TestCase("CREATE TABLE t(a UNIQUE); INSERT INTO t VALUES (1);", "INSERT INTO t VALUES (1);", 19, 2067)] + [TestCase("CREATE TABLE t(id INTEGER PRIMARY KEY); INSERT INTO t VALUES (1);", "INSERT INTO t VALUES (1);", 19, 1555)] + [TestCase("CREATE TABLE t(a NOT NULL);", "INSERT INTO t VALUES (NULL);", 19, 1299)] + [TestCase("CREATE TABLE t(a CHECK (a > 0));", "INSERT INTO t VALUES (0);", 19, 275)] + public void MvccStatementFailureCodesMatchSqlite(string setup, string failing, int expectedPrimary, int expectedExtended) + { + var path = CreateEmptyPath(); + try + { + using var connection = new ManagedSqlite.SqliteConnection($"Data Source={path};Pooling=False"); + connection.Open(); + ExecuteManaged(connection, "PRAGMA journal_mode = 'mvcc';"); + ExecuteManaged(connection, setup); + + CaptureManaged(connection, failing).Should().Be((expectedPrimary, expectedExtended)); + } + finally + { + DeleteDatabase(path); + } + } + + [Test] + public void UniqueViolationMessageMatchesSqlite() + { + const string setup = "CREATE TABLE t(a UNIQUE); INSERT INTO t VALUES (1);"; + const string failing = "INSERT INTO t VALUES (1);"; + + var nativeMessage = CaptureNativeMessage(setup, failing); + var managedMessage = CaptureManagedMessage(setup, failing); + + managedMessage.Should().Be(nativeMessage); + } + + [Test] + public void ReadOnlyWriteCodesMatchSqlite() + { + var path = CreateSeededFile(); + try + { + var native = CaptureNativeFile($"Data Source={path};Mode=ReadOnly;Pooling=False", "INSERT INTO t VALUES (2);"); + var managed = CaptureManagedFile($"Data Source={path};Mode=ReadOnly;Pooling=False", "INSERT INTO t VALUES (2);"); + + native.Should().Be((8, 8)); + managed.Should().Be(native); + } + finally + { + DeleteDatabase(path); + } + } + + [Test] + public void BusyWriterCodesMatchSqlite() + { + var nativePath = CreateSeededFile(); + var managedPath = CreateSeededFile(); + try + { + (int, int) native; + using (var holder = new NativeSqlite.SqliteConnection($"Data Source={nativePath};Pooling=False;Default Timeout=1")) + using (var contender = new NativeSqlite.SqliteConnection($"Data Source={nativePath};Pooling=False;Default Timeout=1")) + { + holder.Open(); + contender.Open(); + ExecuteNative(holder, "BEGIN IMMEDIATE; INSERT INTO t VALUES (2);"); + native = CaptureNative(contender, "BEGIN IMMEDIATE;"); + ExecuteNative(holder, "ROLLBACK;"); + } + + (int, int) managed; + using (var holder = new ManagedSqlite.SqliteConnection($"Data Source={managedPath};Pooling=False;Default Timeout=1")) + using (var contender = new ManagedSqlite.SqliteConnection($"Data Source={managedPath};Pooling=False;Default Timeout=1")) + { + holder.Open(); + contender.Open(); + ExecuteManaged(holder, "BEGIN IMMEDIATE; INSERT INTO t VALUES (2);"); + managed = CaptureManaged(contender, "BEGIN IMMEDIATE;"); + ExecuteManaged(holder, "ROLLBACK;"); + } + + native.Should().Be((5, 5)); + managed.Should().Be(native); + } + finally + { + DeleteDatabase(nativePath); + DeleteDatabase(managedPath); + } + } + + [Test] + public void GarbageHeaderReportsFileIsNotADatabase() + { + var path = Path.Combine(Path.GetTempPath(), $"ahtola-notadb-{Guid.NewGuid():N}.db"); + File.WriteAllBytes(path, Enumerable.Range(0, 4096).Select(i => (byte)(i * 7 + 3)).ToArray()); + try + { + var native = CaptureNativeFileWithMessage($"Data Source={path};Pooling=False", "SELECT * FROM sqlite_master;"); + var managed = CaptureManagedFileWithMessage($"Data Source={path};Pooling=False", "SELECT * FROM sqlite_master;"); + + native.Codes.Should().Be((26, 26)); + managed.Codes.Should().Be(native.Codes); + managed.Message.Should().Be(native.Message); + } + finally + { + DeleteDatabase(path); + } + } + + [TestCase("SQLITE_CONSTRAINT_UNIQUE", "UNIQUE constraint failed: t.a", 19, 2067)] + [TestCase("SQLITE_CONSTRAINT_PRIMARYKEY", "UNIQUE constraint failed: t.id", 19, 1555)] + [TestCase("SQLITE_CONSTRAINT_NOTNULL", "NOT NULL constraint failed: t.a", 19, 1299)] + [TestCase("SQLITE_CONSTRAINT_FOREIGNKEY", "FOREIGN KEY constraint failed", 19, 787)] + [TestCase("SQLITE_CONSTRAINT", "CHECK constraint failed: positive", 19, 275)] + [TestCase("SQLITE_CONSTRAINT", "constraint failed", 19, 19)] + [TestCase("SQLITE_BUSY", "database is locked", 5, 5)] + [TestCase("SQLITE_READONLY", "attempt to write a readonly database", 8, 8)] + [TestCase("SQLITE_IOERR_SHORT_READ", "disk I/O error", 10, 10)] + [TestCase("SQLITE_NOTADB", "file is not a database", 26, 26)] + [TestCase("SQL_PARSE_ERROR", "near \"SELEC\": syntax error", 1, 1)] + public void RemoteHranaErrorCodesMapToSqliteResultCodes(string remoteCode, string remoteMessage, int primary, int extended) + { + var remote = new AhtolaRemoteSqlException( + $"Remote SQL execution failed: {remoteMessage} ({remoteCode})", + remoteCode, + remoteMessage); + + var mapped = ManagedSqlite.SqliteCommand.ToSqliteException(remote); + + (mapped.SqliteErrorCode, mapped.SqliteExtendedErrorCode).Should().Be((primary, extended)); + mapped.Message.Should().StartWith($"SQLite Error {primary}: "); + + var facade = ManagedSqlite.SqliteRemoteExceptionClassifier.From(remote, mapped); + (facade.SqliteErrorCode, facade.SqliteExtendedErrorCode).Should().Be((primary, extended)); + } + + private static TestCaseData Case(string name, string setup, string failing, int primary, int extended) + => new TestCaseData(setup, failing, primary, extended).SetArgDisplayNames(name); + + private static string CreateEmptyPath() + => Path.Combine(Path.GetTempPath(), $"ahtola-errcode-{Guid.NewGuid():N}.db"); + + private static (int Primary, int Extended) CaptureNative(string setup, string failing) + { + using var connection = new NativeSqlite.SqliteConnection("Data Source=:memory:"); + connection.Open(); + if (setup.Length != 0) + ExecuteNative(connection, setup); + return CaptureNative(connection, failing); + } + + private static (int Primary, int Extended) CaptureNative(NativeSqlite.SqliteConnection connection, string failing) + { + try + { + ExecuteNative(connection, failing); + } + catch (NativeSqlite.SqliteException exception) + { + return (exception.SqliteErrorCode, exception.SqliteExtendedErrorCode); + } + + throw new AssertionException($"SQLite accepted '{failing}'."); + } + + private static string CaptureNativeMessage(string setup, string failing) + { + using var connection = new NativeSqlite.SqliteConnection("Data Source=:memory:"); + connection.Open(); + ExecuteNative(connection, setup); + try + { + ExecuteNative(connection, failing); + } + catch (NativeSqlite.SqliteException exception) + { + return exception.Message; + } + + throw new AssertionException($"SQLite accepted '{failing}'."); + } + + private static (int Primary, int Extended) CaptureManaged(string setup, string failing) + { + using var connection = new ManagedSqlite.SqliteConnection("Data Source=:memory:"); + connection.Open(); + if (setup.Length != 0) + ExecuteManaged(connection, setup); + return CaptureManaged(connection, failing); + } + + private static (int Primary, int Extended) CaptureManaged(ManagedSqlite.SqliteConnection connection, string failing) + { + try + { + ExecuteManaged(connection, failing); + } + catch (ManagedSqlite.SqliteException exception) + { + return (exception.SqliteErrorCode, exception.SqliteExtendedErrorCode); + } + + throw new AssertionException($"The managed engine accepted '{failing}'."); + } + + private static string CaptureManagedMessage(string setup, string failing) + { + using var connection = new ManagedSqlite.SqliteConnection("Data Source=:memory:"); + connection.Open(); + ExecuteManaged(connection, setup); + try + { + ExecuteManaged(connection, failing); + } + catch (ManagedSqlite.SqliteException exception) + { + return exception.Message; + } + + throw new AssertionException($"The managed engine accepted '{failing}'."); + } + + private static (int Primary, int Extended) CaptureNativeFile(string connectionString, string failing) + => CaptureNativeFileWithMessage(connectionString, failing).Codes; + + private static ((int Primary, int Extended) Codes, string Message) CaptureNativeFileWithMessage(string connectionString, string failing) + { + try + { + using var connection = new NativeSqlite.SqliteConnection(connectionString); + connection.Open(); + ExecuteNative(connection, failing); + } + catch (NativeSqlite.SqliteException exception) + { + return ((exception.SqliteErrorCode, exception.SqliteExtendedErrorCode), exception.Message); + } + + throw new AssertionException($"SQLite accepted '{failing}'."); + } + + private static (int Primary, int Extended) CaptureManagedFile(string connectionString, string failing) + => CaptureManagedFileWithMessage(connectionString, failing).Codes; + + private static ((int Primary, int Extended) Codes, string Message) CaptureManagedFileWithMessage(string connectionString, string failing) + { + try + { + using var connection = new ManagedSqlite.SqliteConnection(connectionString); + connection.Open(); + ExecuteManaged(connection, failing); + } + catch (ManagedSqlite.SqliteException exception) + { + return ((exception.SqliteErrorCode, exception.SqliteExtendedErrorCode), exception.Message); + } + + throw new AssertionException($"The managed engine accepted '{failing}'."); + } + + private static void ExecuteNative(NativeSqlite.SqliteConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + private static void ExecuteManaged(ManagedSqlite.SqliteConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + private static string CreateSeededFile() + { + var path = Path.Combine(Path.GetTempPath(), $"ahtola-errcode-{Guid.NewGuid():N}.db"); + using (var connection = new NativeSqlite.SqliteConnection($"Data Source={path};Pooling=False")) + { + connection.Open(); + ExecuteNative(connection, "CREATE TABLE t(a); INSERT INTO t VALUES (1);"); + } + + return path; + } + + private static void DeleteDatabase(string path) + { + ManagedSqlite.SqliteConnection.ClearAllPools(); + NativeSqlite.SqliteConnection.ClearAllPools(); + foreach (var candidate in new[] { path, path + "-wal", path + "-shm", path + "-journal" }) + { + try + { + if (File.Exists(candidate)) + File.Delete(candidate); + } + catch (IOException) + { + } + } + } +} diff --git a/src/Ahtola.Tests/VdbeScalarControlOpcodeTests.cs b/src/Ahtola.Tests/VdbeScalarControlOpcodeTests.cs index 8f408017..f1a5efe7 100644 --- a/src/Ahtola.Tests/VdbeScalarControlOpcodeTests.cs +++ b/src/Ahtola.Tests/VdbeScalarControlOpcodeTests.cs @@ -222,7 +222,9 @@ public void DecrJumpZeroRejectsNonIntegerRegisters() using var statement = new ResumableStatement(program); var error = Assert.Throws(() => statement.StepResumable()); - error!.SqliteErrorCode.Should().Be(SqliteResultCode.Constraint); + // Turso bails with a Constraint error here, but SQLite (and so the + // Microsoft.Data.Sqlite facade) reports "datatype mismatch" as SQLITE_MISMATCH. + error!.SqliteErrorCode.Should().Be(SqliteResultCode.Mismatch); error.Message.Should().Contain("datatype mismatch"); } } @@ -431,7 +433,7 @@ public void MustBeIntNoTargetConstraintMismatch() using var statement = new ResumableStatement(program); var error = Assert.Throws(() => statement.StepResumable()); - error!.SqliteErrorCode.Should().Be(SqliteResultCode.Constraint); + error!.SqliteErrorCode.Should().Be(SqliteResultCode.Mismatch); error.Message.Should().Contain("datatype mismatch"); } From e398391001fd97d15f930faeafef391014f36472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 15:21:22 -0400 Subject: [PATCH 05/20] Add replica parity: advanced keywords, explicit pull/push/checkpoint/stats, sync status, token provider Close the remaining gaps against turso-src v0.8.1 bindings/dotnet: - Connection strings (AhtolaConnectionStringBuilder and the SQLite facade, so EF Core UseAhtola reaches them): Sync Client Name, Sync Long Poll Timeout, Bootstrap If Empty, Partial Bootstrap Prefix/Query, Partial Sync Segment Size/Prefetch, Remote Encryption Cipher/Key, Push Operations Threshold, Pull Bytes Threshold, Force Logical MVCC Pull (only False is supported; True fails closed), Sync Experimental Features (validated, no effect) and the Ahtola-specific Automatic Sync Mode. They map onto AhtolaReplicaOptions and are rejected on local/direct-remote connections, matching Turso's HasAdvancedReplicaOptions. - Explicit sync operations on AhtolaConnection and SqliteConnection: Pull/PullAsync, Push/PushAsync (drains every batch pending at the call), Checkpoint/CheckpointAsync and GetSyncStatistics/GetSyncStatisticsAsync, reusing the managed replica engine. Pull-only and push-only requests coalesce per kind in ManagedReplicaSyncRegistry. Sync results now report real main/revert WAL sizes; statistics report pending journal changes. - Automatic sync: AutomaticSyncStatus and AutomaticSyncStatusChanged (ported from TursoAutomaticSyncCoordinator) make background failures observable. AhtolaAutomaticSyncMode.PullOnly matches Turso's pull-only loop; the default stays PushAndPull. - AuthTokenProvider hook (connection, facade, replica options) resolved per HTTP request, per Hrana WebSocket connect, and per replica sync request. - Hrana replication_index is treated as opaque per PROTOCOL.md 7.2.6: the numeric watermark is still tracked and echoed, other values are ignored instead of failing the batch. - Sync client ids may now carry a validated client-name prefix. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +- docs/dotnet-packages.md | 73 +++ src/Ahtola.Data.Sqlite/SqliteConnection.cs | 135 +++++- .../SqliteConnectionStringBuilder.cs | 240 +++++++++- src/Ahtola.Data/AhtolaAutomaticSync.cs | 228 +++++++++ src/Ahtola.Data/AhtolaConnection.cs | 380 ++++++++++++--- src/Ahtola.Data/AhtolaConnectionOptions.cs | 131 ++++++ .../AhtolaConnectionStringBuilder.cs | 200 ++++++++ src/Ahtola.Data/AhtolaRemoteClient.cs | 113 +++-- src/Ahtola.Data/AhtolaReplicaProvider.cs | 129 +++++- src/Ahtola.Data/AhtolaSyncOptions.cs | 42 +- .../Hrana/AhtolaHranaWebSocketTransport.cs | 12 +- .../Hrana/AhtolaRemoteClient.WebSocket.cs | 14 +- src/Ahtola.Data/ManagedReplicaBootstrapper.cs | 113 ++++- .../ManagedReplicaConnectionHost.cs | 233 +++++++++- src/Ahtola.Data/ManagedReplicaSyncRegistry.cs | 76 ++- ...anagedReplicaExplicitSyncOperationTests.cs | 437 ++++++++++++++++++ .../RemoteEfOptionsClassificationTests.cs | 3 + .../RemoteReplicationIndexTests.cs | 31 +- .../ReplicaConnectionStringOptionsTests.cs | 240 ++++++++++ 20 files changed, 2672 insertions(+), 163 deletions(-) create mode 100644 src/Ahtola.Data/AhtolaAutomaticSync.cs create mode 100644 src/Ahtola.Tests/ManagedReplicaExplicitSyncOperationTests.cs create mode 100644 src/Ahtola.Tests/ReplicaConnectionStringOptionsTests.cs diff --git a/README.md b/README.md index bf90dc70..e5b80f56 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,10 @@ Common connection-string keywords: `Data Source`, `Mode`, `Cache`, `Pooling`, `DateTimeKind`, `BinaryGUID`, or `Encryption Cipher` + `Encryption Key` (hex AES-GCM or AEGIS keys). Turso/Hrana URLs also accept `Auth Token`, `Replica Path`, `Sync Interval`, `Read Your Writes`, -and `Tls` through either ADO.NET facade. Default local provider is managed-only. +and `Tls` through either ADO.NET facade, plus Turso's advanced embedded-replica +keywords (`Sync Client Name`, `Sync Long Poll Timeout`, `Bootstrap If Empty`, +partial-bootstrap, remote-encryption and threshold keys) and Ahtola's +`Automatic Sync Mode` — see [docs/dotnet-packages.md](docs/dotnet-packages.md#connection-string-reference). Default local provider is managed-only. ### Standard SQLite files diff --git a/docs/dotnet-packages.md b/docs/dotnet-packages.md index 658ef5d7..9e55e56f 100644 --- a/docs/dotnet-packages.md +++ b/docs/dotnet-packages.md @@ -194,6 +194,26 @@ and `Ahtola.AhtolaConnectionStringBuilder`: Remote keywords accepted by both facades (see the Turso Cloud sections below): `Auth Token`, `Replica Path`, `Sync Interval`, `Read Your Writes`, `Tls`. +Embedded-replica keywords (each also accepts its space-free alias, e.g. +`SyncClientName`), named as in Turso's `TursoConnectionStringBuilder` so EF Core +`UseAhtola(...)` and the SQLite facade can reach every replica option. They +require `Replica Path`; on a local data source they throw +`InvalidOperationException`, on a direct remote connection `NotSupportedException`: + +| Keyword | Maps to | Notes | +| --- | --- | --- | +| `Sync Client Name` | `AhtolaReplicaOptions.ClientName` | Prefix of the sync client id recorded at bootstrap (`-`); 1–64 of `A-Z a-z 0-9 . _ -` | +| `Sync Long Poll Timeout` | `LongPollTimeout` | Milliseconds; `0` (default) disables long polling | +| `Bootstrap If Empty` | `BootstrapIfEmpty` | Default `true` | +| `Partial Bootstrap Prefix` / `Partial Bootstrap Query` | `PartialBootstrap` | Mutually exclusive; bytes / server-side SQL | +| `Partial Sync Segment Size` / `Partial Sync Prefetch` | `PartialBootstrap.SegmentSize` / `.Prefetch` | Require one of the two keywords above | +| `Remote Encryption Cipher` / `Remote Encryption Key` | `RemoteEncryption` | Turso cipher names (`aes256gcm`, `aegis256`, …) and a base64 key; both or neither | +| `Push Operations Threshold` | `PushOperationsThreshold` | `0` uses the default | +| `Pull Bytes Threshold` | `PullBytesThreshold` | `0` pulls the bootstrap image in one request | +| `Automatic Sync Mode` | `AutomaticSyncMode` | Ahtola-specific: `PushAndPull` (default) or `PullOnly` (Turso's behavior) | +| `Force Logical MVCC Pull` | — | Only `False` is supported; `True` fails closed because the managed provider always auto-detects the pull protocol | +| `Sync Experimental Features` | — | Validated comma-separated names; no effect on the managed engine, which does not gate features behind them | + Hrana WebSocket keywords, used only by `ws://`/`wss://` data sources (see [Hrana over WebSocket](#hrana-over-websocket-wswss)): @@ -436,6 +456,27 @@ policy. The native facade exposes `AhtolaException` / (`Data Source=...;Auth Token=***`) — it is never exposed in diagnostics. Never hardcode the token; read it from a secret manager or environment variable. +To rotate credentials without reopening, set `AuthTokenProvider` (on +`AhtolaConnection`, `SqliteConnection`, or `AhtolaReplicaOptions`) before +opening — the counterpart of Turso's serverless `with_auth_token_fn`. It takes +precedence over `Auth Token` and is awaited before every HTTP request (bootstrap, +pull, lazy page fetch, push, and each Hrana pipeline/cursor request) and before +every Hrana WebSocket (re)connect handshake; a `null` or blank result sends no +token. The HTTPS-only credential policy applies as if a token were configured. + +```csharp +using var cloud = new AhtolaConnection("Data Source=libsql://my-db.turso.io") +{ + AuthTokenProvider = async cancellationToken => await tokenCache.GetAsync(cancellationToken), +}; +``` + +Batch and step `replication_index` values are treated as opaque, per the +pinned Hrana spec (`turso-src/serverless/PROTOCOL.md` §7.2.6): the highest +non-negative integer watermark a libSQL server reports is still echoed on later +batches for read-your-writes against replica-backed servers, but any other +value is ignored instead of failing the request. + Expired Hrana streams are retried once automatically only for stateless commands. An active transaction is never replayed: it becomes unusable and commit/rollback preserves the original stream failure instead of masking it @@ -607,6 +648,38 @@ using var replica = new AhtolaConnection( ";Replica Path=./replica.db;Sync Interval=30"); ``` +Each tick runs a full push-then-pull `Sync` by default. Turso's .NET binding +instead runs a **pull-only** loop; add `Automatic Sync Mode=PullOnly` (or set +`AhtolaReplicaOptions.AutomaticSyncMode`) to match it and push explicitly. The +default is unchanged. Background failures are observable through +`AutomaticSyncStatus` (`Stopped`/`Waiting`/`Running`/`Retrying`/`Faulted`, with +`LastAttempt`, `LastSuccess`, `LastPullAppliedChanges`, `LastException`, +`NextAttempt`) and the `AutomaticSyncStatusChanged` event, raised on a +thread-pool thread in publication order. Transient transport failures are +retried up to three attempts per tick; anything else (for example a push +conflict) moves the loop to `Faulted`, stops it, and is rethrown by `Close`. + +The explicit operations mirror Turso's `TursoSyncDatabase` and exist on both +`AhtolaConnection` and `SqliteConnection` (sync and async): + +```csharp +AhtolaSyncResult pulled = replica.Pull(); // pull + apply only; pending local changes stay journaled +AhtolaSyncResult pushed = replica.Push(); // push every change pending at the call; no pull +replica.Checkpoint(); // fold the local WAL into the database file +AhtolaSyncStatistics stats = replica.GetSyncStatistics(); +Console.WriteLine(stats.CdcOperations); // local changes still waiting to be pushed +Console.WriteLine(stats.MainWalSize); // current -wal size; RevertWalSize for the recovery WAL +``` + +`Push` sends several batches when `Push Operations Threshold` caps them and +reports the pushed count in `Statistics.CdcOperations`. `Checkpoint` never +discards unpushed changes, and fails closed while a checkpoint-recovery bundle, +unknown push outcome, or unresolved conflict is pending, or while a partial +replica is still lazily materializing pages (`Sync` resolves the first two; +`Pull` fails closed on them too). `GetSyncStatistics` is a pure local read; +`LastPull`, `LastPush` and the pull-updates byte counters are tracked per replica +file within the current process and reset when every connection to it closes. + Bootstrap is a validated raw-page snapshot. For protocol-2 databases, incremental pull is Turso's MVCC logical stream: Ahtola validates the complete `lml3` response, replays the transaction atomically, filters this client's own diff --git a/src/Ahtola.Data.Sqlite/SqliteConnection.cs b/src/Ahtola.Data.Sqlite/SqliteConnection.cs index 15bc1a00..4a6ed0b5 100644 --- a/src/Ahtola.Data.Sqlite/SqliteConnection.cs +++ b/src/Ahtola.Data.Sqlite/SqliteConnection.cs @@ -27,6 +27,8 @@ public partial class SqliteConnection : private readonly IManagedDatabaseFactory? _managedDatabaseFactory; private AhtolaConnection? _ahtolaConnection; private bool _ahtolaConnectionWasOpen; + private Func>? _authTokenProvider; + private AhtolaAutomaticSyncStatus _lastAutomaticSyncStatus = AhtolaAutomaticSyncStatus.Stopped; private ManagedConnectionPoolLease? _managedPoolLease; private SqliteConnectionStringBuilder _connectionOptions = new(); private bool _disposed; @@ -236,7 +238,7 @@ public override void Open() var remoteOriginalState = State; try { - _ahtolaConnection = new AhtolaConnection(_connectionOptions.GetAhtolaConnectionString()); + _ahtolaConnection = CreateRemoteConnection(); _ahtolaConnection.Open(); _ahtolaConnectionWasOpen = true; _dataSource = _ahtolaConnection.DataSource; @@ -450,7 +452,7 @@ private async Task OpenRemoteAsync(CancellationToken cancellationToken) var remoteOpenOriginalState = State; try { - var connection = new AhtolaConnection(_connectionOptions.GetAhtolaConnectionString()); + var connection = CreateRemoteConnection(); _ahtolaConnection = connection; await connection.OpenAsync(cancellationToken).ConfigureAwait(false); _ahtolaConnectionWasOpen = true; @@ -472,6 +474,133 @@ private async Task OpenRemoteAsync(CancellationToken cancellationToken) } } + /// + /// Optional bearer-token provider for remote and embedded-replica data sources; see + /// . Takes precedence over Auth Token + /// and is invoked before every remote request. Must be set before the connection opens. + /// + public Func>? AuthTokenProvider + { + get => _authTokenProvider; + set + { + if (State == ConnectionState.Open) + throw new InvalidOperationException("AuthTokenProvider cannot be set while the connection is open."); + _authTokenProvider = value; + } + } + + /// + /// Gets the status of the embedded replica's automatic synchronization loop + /// (Sync Interval); see . + /// + public AhtolaAutomaticSyncStatus AutomaticSyncStatus + => _ahtolaConnection?.AutomaticSyncStatus ?? _lastAutomaticSyncStatus; + + /// + /// Raised on a thread-pool thread whenever changes; see + /// . The sender is this connection. + /// + public event EventHandler? AutomaticSyncStatusChanged; + + private AhtolaConnection CreateRemoteConnection() + { + var connection = new AhtolaConnection(_connectionOptions.GetAhtolaConnectionString()) + { + AuthTokenProvider = _authTokenProvider, + }; + connection.AutomaticSyncStatusChanged += (_, args) => + { + _lastAutomaticSyncStatus = args.Status; + AutomaticSyncStatusChanged?.Invoke(this, args); + }; + return connection; + } + + private AhtolaConnection RequireReplicaConnection() + => _ahtolaConnection + ?? throw new NotSupportedException("Sync requires an open embedded replica connection."); + + /// + /// Pulls and applies remote changes without pushing local changes; see + /// . + /// + public AhtolaSyncResult Pull() + => RunReplicaOperation(static connection => connection.Pull()); + + /// + public Task PullAsync(CancellationToken cancellationToken = default) + => RunReplicaOperationAsync(connection => connection.PullAsync(cancellationToken)); + + /// + /// Pushes pending local changes without pulling remote changes; see + /// . + /// + public AhtolaSyncResult Push() + => RunReplicaOperation(static connection => connection.Push()); + + /// + public Task PushAsync(CancellationToken cancellationToken = default) + => RunReplicaOperationAsync(connection => connection.PushAsync(cancellationToken)); + + /// + /// Checkpoints the embedded replica's local write-ahead log; see + /// . + /// + public void Checkpoint() + => RunReplicaOperation(static connection => + { + connection.Checkpoint(); + return true; + }); + + /// + public Task CheckpointAsync(CancellationToken cancellationToken = default) + => RunReplicaOperationAsync(async connection => + { + await connection.CheckpointAsync(cancellationToken).ConfigureAwait(false); + return true; + }); + + /// + /// Returns a snapshot of the embedded replica's synchronization statistics; see + /// . + /// + public AhtolaSyncStatistics GetSyncStatistics() + => RunReplicaOperation(static connection => connection.GetSyncStatistics()); + + /// + public Task GetSyncStatisticsAsync(CancellationToken cancellationToken = default) + => RunReplicaOperationAsync(connection => connection.GetSyncStatisticsAsync(cancellationToken)); + + private T RunReplicaOperation(Func operation) + { + var connection = RequireReplicaConnection(); + try + { + return operation(connection); + } + catch (Exception ex) when (ex is AhtolaException or HttpRequestException) + { + ObserveRemoteInvalidation(); + throw MapRemoteLifecycleException(ex); + } + } + + private async Task RunReplicaOperationAsync(Func> operation) + { + var connection = RequireReplicaConnection(); + try + { + return await operation(connection).ConfigureAwait(false); + } + catch (Exception ex) when (ex is AhtolaException or HttpRequestException) + { + ObserveRemoteInvalidation(); + throw MapRemoteLifecycleException(ex); + } + } + /// Synchronizes an embedded replica with its configured remote endpoint. public void Sync() { @@ -553,6 +682,7 @@ public override void Close() catch when (cleanupError is not null) { } + _lastAutomaticSyncStatus = connection.AutomaticSyncStatus; _ahtolaConnectionWasOpen = false; _dataSource = null; _readOnly = false; @@ -1183,6 +1313,7 @@ internal void ObserveRemoteInvalidation() finally { connection.Dispose(); + _lastAutomaticSyncStatus = connection.AutomaticSyncStatus; } } catch diff --git a/src/Ahtola.Data.Sqlite/SqliteConnectionStringBuilder.cs b/src/Ahtola.Data.Sqlite/SqliteConnectionStringBuilder.cs index 8f95e0f0..fd33f610 100644 --- a/src/Ahtola.Data.Sqlite/SqliteConnectionStringBuilder.cs +++ b/src/Ahtola.Data.Sqlite/SqliteConnectionStringBuilder.cs @@ -31,6 +31,20 @@ public class SqliteConnectionStringBuilder : DbConnectionStringBuilder "Replica Path", "Read Your Writes", "Sync Interval", + "Sync Client Name", + "Sync Long Poll Timeout", + "Bootstrap If Empty", + "Partial Bootstrap Prefix", + "Partial Bootstrap Query", + "Partial Sync Segment Size", + "Partial Sync Prefetch", + "Remote Encryption Cipher", + "Remote Encryption Key", + "Push Operations Threshold", + "Pull Bytes Threshold", + "Force Logical MVCC Pull", + "Sync Experimental Features", + "Automatic Sync Mode", "Tls", "Ws Keepalive Interval", "Ws Keepalive Timeout", @@ -82,6 +96,35 @@ public class SqliteConnectionStringBuilder : DbConnectionStringBuilder ["ReadYourWrites"] = "Read Your Writes", ["Sync Interval"] = "Sync Interval", ["SyncInterval"] = "Sync Interval", + // Advanced embedded-replica options, named as in Turso's SqliteConnectionStringBuilder. + ["Sync Client Name"] = "Sync Client Name", + ["SyncClientName"] = "Sync Client Name", + ["Sync Long Poll Timeout"] = "Sync Long Poll Timeout", + ["SyncLongPollTimeout"] = "Sync Long Poll Timeout", + ["Bootstrap If Empty"] = "Bootstrap If Empty", + ["BootstrapIfEmpty"] = "Bootstrap If Empty", + ["Partial Bootstrap Prefix"] = "Partial Bootstrap Prefix", + ["PartialBootstrapPrefix"] = "Partial Bootstrap Prefix", + ["Partial Bootstrap Query"] = "Partial Bootstrap Query", + ["PartialBootstrapQuery"] = "Partial Bootstrap Query", + ["Partial Sync Segment Size"] = "Partial Sync Segment Size", + ["PartialSyncSegmentSize"] = "Partial Sync Segment Size", + ["Partial Sync Prefetch"] = "Partial Sync Prefetch", + ["PartialSyncPrefetch"] = "Partial Sync Prefetch", + ["Remote Encryption Cipher"] = "Remote Encryption Cipher", + ["RemoteEncryptionCipher"] = "Remote Encryption Cipher", + ["Remote Encryption Key"] = "Remote Encryption Key", + ["RemoteEncryptionKey"] = "Remote Encryption Key", + ["Push Operations Threshold"] = "Push Operations Threshold", + ["PushOperationsThreshold"] = "Push Operations Threshold", + ["Pull Bytes Threshold"] = "Pull Bytes Threshold", + ["PullBytesThreshold"] = "Pull Bytes Threshold", + ["Force Logical MVCC Pull"] = "Force Logical MVCC Pull", + ["ForceLogicalMvccPull"] = "Force Logical MVCC Pull", + ["Sync Experimental Features"] = "Sync Experimental Features", + ["SyncExperimentalFeatures"] = "Sync Experimental Features", + ["Automatic Sync Mode"] = "Automatic Sync Mode", + ["AutomaticSyncMode"] = "Automatic Sync Mode", ["Tls"] = "Tls", ["TLS"] = "Tls", // Hrana WebSocket (ws/wss) transport tunables; ignored by the HTTP pipeline. @@ -262,6 +305,133 @@ public int SyncInterval } } + /// Client name prefixing the sync client identifier of a new embedded replica. + public string SyncClientName + { + get => GetString("Sync Client Name"); + set => SetString("Sync Client Name", value); + } + + /// Embedded-replica pull long-poll timeout in milliseconds; 0 disables long polling. + public int SyncLongPollTimeout + { + get => GetInt("Sync Long Poll Timeout", 0); + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + this["Sync Long Poll Timeout"] = value; + } + } + + /// Whether a missing embedded replica is bootstrapped from the remote (default true). + public bool BootstrapIfEmpty + { + get => GetBool("Bootstrap If Empty", true); + set => this["Bootstrap If Empty"] = value; + } + + /// Byte prefix of a partial embedded-replica bootstrap; 0 disables it. + public int PartialBootstrapPrefix + { + get => GetInt("Partial Bootstrap Prefix", 0); + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + this["Partial Bootstrap Prefix"] = value; + } + } + + /// Server-side query selecting the pages of a partial embedded-replica bootstrap. + public string PartialBootstrapQuery + { + get => GetString("Partial Bootstrap Query"); + set => SetString("Partial Bootstrap Query", value); + } + + /// Lazy-loading segment size in bytes of a partial bootstrap; 0 uses the default. + public long PartialSyncSegmentSize + { + get => GetLong("Partial Sync Segment Size", 0); + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + this["Partial Sync Segment Size"] = value; + } + } + + /// Whether a partial bootstrap prefetches adjacent pages during lazy loading. + public bool PartialSyncPrefetch + { + get => GetBool("Partial Sync Prefetch"); + set => this["Partial Sync Prefetch"] = value; + } + + /// Cipher of an encrypted remote database used by an embedded replica (e.g. aes256gcm). + public string RemoteEncryptionCipher + { + get => GetString("Remote Encryption Cipher"); + set => SetString("Remote Encryption Cipher", value); + } + + /// Base64-encoded key of an encrypted remote database used by an embedded replica. + public string RemoteEncryptionKey + { + get => GetString("Remote Encryption Key"); + set => SetString("Remote Encryption Key", value); + } + + /// Maximum change operations per embedded-replica push batch; 0 uses the default. + public long PushOperationsThreshold + { + get => GetLong("Push Operations Threshold", 0); + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + this["Push Operations Threshold"] = value; + } + } + + /// Target size in bytes of each initial bootstrap pull; 0 pulls the whole image. + public long PullBytesThreshold + { + get => GetLong("Pull Bytes Threshold", 0); + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + this["Pull Bytes Threshold"] = value; + } + } + + /// + /// Turso's forced MVCC logical-pull escape hatch. The managed provider always auto-detects + /// the pull protocol, so fails closed when the connection opens. + /// + public bool ForceLogicalMvccPull + { + get => GetBool("Force Logical MVCC Pull"); + set => this["Force Logical MVCC Pull"] = value; + } + + /// + /// Turso's experimental-feature list for the replica's local database; validated, but it + /// has no effect on the managed engine. + /// + public string SyncExperimentalFeatures + { + get => GetString("Sync Experimental Features"); + set => SetString("Sync Experimental Features", value); + } + + /// + /// What each tick does: + /// (the default) or Turso's . + /// + public AhtolaAutomaticSyncMode AutomaticSyncMode + { + get => GetEnum("Automatic Sync Mode", AhtolaAutomaticSyncMode.PushAndPull); + set => this["Automatic Sync Mode"] = value; + } + public bool? Tls { get => GetNullableBool("Tls"); @@ -421,6 +591,34 @@ internal string GetAhtolaConnectionString() builder["Read Your Writes"] = ReadYourWrites; if (base.ContainsKey("Sync Interval")) builder["Sync Interval"] = SyncInterval; + if (base.ContainsKey("Sync Client Name")) + builder["Sync Client Name"] = SyncClientName; + if (base.ContainsKey("Sync Long Poll Timeout")) + builder["Sync Long Poll Timeout"] = SyncLongPollTimeout; + if (base.ContainsKey("Bootstrap If Empty")) + builder["Bootstrap If Empty"] = BootstrapIfEmpty; + if (base.ContainsKey("Partial Bootstrap Prefix")) + builder["Partial Bootstrap Prefix"] = PartialBootstrapPrefix; + if (base.ContainsKey("Partial Bootstrap Query")) + builder["Partial Bootstrap Query"] = PartialBootstrapQuery; + if (base.ContainsKey("Partial Sync Segment Size")) + builder["Partial Sync Segment Size"] = PartialSyncSegmentSize; + if (base.ContainsKey("Partial Sync Prefetch")) + builder["Partial Sync Prefetch"] = PartialSyncPrefetch; + if (base.ContainsKey("Remote Encryption Cipher")) + builder["Remote Encryption Cipher"] = RemoteEncryptionCipher; + if (base.ContainsKey("Remote Encryption Key")) + builder["Remote Encryption Key"] = RemoteEncryptionKey; + if (base.ContainsKey("Push Operations Threshold")) + builder["Push Operations Threshold"] = PushOperationsThreshold; + if (base.ContainsKey("Pull Bytes Threshold")) + builder["Pull Bytes Threshold"] = PullBytesThreshold; + if (base.ContainsKey("Force Logical MVCC Pull")) + builder["Force Logical MVCC Pull"] = ForceLogicalMvccPull; + if (base.ContainsKey("Sync Experimental Features")) + builder["Sync Experimental Features"] = SyncExperimentalFeatures; + if (base.ContainsKey("Automatic Sync Mode")) + builder["Automatic Sync Mode"] = AutomaticSyncMode.ToString(); if (base.ContainsKey("Tls")) builder["Tls"] = Tls!.Value; if (base.ContainsKey("Ws Keepalive Interval")) @@ -539,6 +737,13 @@ private int GetInt(string keyword, int defaultValue) : defaultValue; } + private long GetLong(string keyword, long defaultValue) + { + return base.TryGetValue(keyword, out var value) + ? Convert.ToInt64(value, CultureInfo.InvariantCulture) + : defaultValue; + } + private TEnum GetEnum(string keyword, TEnum defaultValue) where TEnum : struct { @@ -575,13 +780,19 @@ private void SetNullable(string keyword, T? value) "Mode" => ConvertOpenMode(value), "Cache" => ConvertCacheMode(value), "Foreign Keys" => ConvertToNullableBoolean(value), - "Recursive Triggers" or "Pooling" or "BinaryGUID" or "Foreign Read Only" or "Read Your Writes" => Convert.ToBoolean(value, CultureInfo.InvariantCulture), + "Recursive Triggers" or "Pooling" or "BinaryGUID" or "Foreign Read Only" or "Read Your Writes" + or "Bootstrap If Empty" or "Partial Sync Prefetch" or "Force Logical MVCC Pull" + => Convert.ToBoolean(value, CultureInfo.InvariantCulture), "Tls" => ConvertToNullableBoolean(value), "Default Timeout" or "Version" or "Sync Interval" or "Ws Keepalive Interval" or "Ws Keepalive Timeout" or "Ws Half Open Timeout" or "Ws Max Message Bytes" or "Ws Connect Attempts" + or "Sync Long Poll Timeout" or "Partial Bootstrap Prefix" => Convert.ToInt32(value, CultureInfo.InvariantCulture), + "Partial Sync Segment Size" or "Push Operations Threshold" or "Pull Bytes Threshold" + => Convert.ToInt64(value, CultureInfo.InvariantCulture), "DateTimeKind" => ConvertDateTimeKind(value), "Local Provider" => ConvertLocalProvider(value), + "Automatic Sync Mode" => ConvertAutomaticSyncMode(value), _ => Convert.ToString(value, CultureInfo.InvariantCulture) ?? string.Empty, }; } @@ -597,6 +808,7 @@ private void SetNullable(string keyword, T? value) "Read Your Writes" => Convert.ToBoolean(value, CultureInfo.InvariantCulture), "DateTimeKind" => ConvertDateTimeKind(value), "Local Provider" => ConvertLocalProvider(value), + "Automatic Sync Mode" => ConvertAutomaticSyncMode(value), _ => value, }; } @@ -627,6 +839,20 @@ private void SetNullable(string keyword, T? value) "Replica Path" => string.Empty, "Read Your Writes" => true, "Sync Interval" => 0, + "Sync Client Name" => string.Empty, + "Sync Long Poll Timeout" => 0, + "Bootstrap If Empty" => true, + "Partial Bootstrap Prefix" => 0, + "Partial Bootstrap Query" => string.Empty, + "Partial Sync Segment Size" => 0L, + "Partial Sync Prefetch" => false, + "Remote Encryption Cipher" => string.Empty, + "Remote Encryption Key" => string.Empty, + "Push Operations Threshold" => 0L, + "Pull Bytes Threshold" => 0L, + "Force Logical MVCC Pull" => false, + "Sync Experimental Features" => string.Empty, + "Automatic Sync Mode" => AhtolaAutomaticSyncMode.PushAndPull, "Tls" => null!, "Ws Keepalive Interval" => 30, "Ws Keepalive Timeout" => 20, @@ -688,6 +914,18 @@ private static DateTimeKind ConvertDateTimeKind(object value) return kind; } + private static AhtolaAutomaticSyncMode ConvertAutomaticSyncMode(object value) + { + var mode = ConvertEnum(value); + if (!Enum.IsDefined(mode)) + throw new ArgumentOutOfRangeException( + nameof(value), + value, + Properties.Resources.InvalidEnumValue(typeof(AhtolaAutomaticSyncMode), mode)); + + return mode; + } + private static AhtolaLocalProvider ConvertLocalProvider(object value) { var provider = ConvertEnum(value); diff --git a/src/Ahtola.Data/AhtolaAutomaticSync.cs b/src/Ahtola.Data/AhtolaAutomaticSync.cs new file mode 100644 index 00000000..4acb1712 --- /dev/null +++ b/src/Ahtola.Data/AhtolaAutomaticSync.cs @@ -0,0 +1,228 @@ +namespace Ahtola; + +/// +/// Selects what the managed embedded-replica automatic synchronization loop +/// (Sync Interval) does on each tick. +/// +/// +/// Ahtola's historical behavior, and still the default, is : every tick +/// runs a full . +/// Turso's .NET binding runs a pull-only loop instead (TursoAutomaticSyncCoordinator calls +/// PullAsync); select to match it and push local changes explicitly +/// with . +/// +public enum AhtolaAutomaticSyncMode +{ + /// Push local changes, then pull and apply remote changes (the default). + PushAndPull, + + /// Only pull and apply remote changes; local changes are pushed explicitly. + PullOnly, +} + +/// +/// Identifies the current phase of the managed embedded-replica automatic synchronization loop. +/// +public enum AhtolaAutomaticSyncState +{ + /// No automatic synchronization loop is running. + Stopped, + + /// The loop is waiting for its next scheduled attempt. + Waiting, + + /// A scheduled attempt is in flight. + Running, + + /// A transient failure is being retried. + Retrying, + + /// + /// A non-retryable failure (or the last retry) stopped the loop. The failure is available in + /// and is rethrown by + /// . + /// + Faulted, +} + +/// +/// An immutable snapshot of the managed embedded-replica automatic synchronization loop. +/// +/// The current loop phase. +/// The 1-based attempt number of the current tick, or 0 while idle. +/// When the most recent attempt started. +/// When the most recent attempt completed successfully. +/// +/// Whether the most recent successful attempt applied remote changes, or +/// before the first success. +/// +/// The failure of the most recent failed attempt, if any. +/// When the next attempt is scheduled, if one is. +public sealed record AhtolaAutomaticSyncStatus( + AhtolaAutomaticSyncState State, + int Attempt, + DateTimeOffset? LastAttempt, + DateTimeOffset? LastSuccess, + bool? LastPullAppliedChanges, + Exception? LastException, + DateTimeOffset? NextAttempt) +{ + /// The status of a connection that has no automatic synchronization loop. + public static AhtolaAutomaticSyncStatus Stopped { get; } = new( + AhtolaAutomaticSyncState.Stopped, + Attempt: 0, + LastAttempt: null, + LastSuccess: null, + LastPullAppliedChanges: null, + LastException: null, + NextAttempt: null); +} + +/// +/// Carries the new status for . +/// +public sealed class AhtolaAutomaticSyncStatusChangedEventArgs(AhtolaAutomaticSyncStatus status) : EventArgs +{ + /// Gets the status that was just published. + public AhtolaAutomaticSyncStatus Status { get; } = status; +} + +/// +/// Runs one connection's managed embedded-replica automatic synchronization loop and publishes +/// every phase change. Mirrors Turso's TursoAutomaticSyncCoordinator +/// (turso-src/bindings/dotnet/src/Turso.Data/TursoAutomaticSyncCoordinator.cs), with the +/// tick operation selected by and Ahtola's own bounded +/// transient-retry policy. +/// +internal sealed class ManagedReplicaAutomaticSyncCoordinator +{ + private readonly Func> _operation; + private readonly TimeSpan _interval; + private readonly Action _publish; + private readonly CancellationTokenSource _cancellation = new(); + private AhtolaAutomaticSyncStatus _status; + private Task? _task; + private int _stopped; + + public ManagedReplicaAutomaticSyncCoordinator( + Func> operation, + TimeSpan interval, + AhtolaAutomaticSyncStatus initialStatus, + Action publish) + { + _operation = operation; + _interval = interval; + _status = initialStatus; + _publish = publish; + } + + public AhtolaAutomaticSyncStatus Status => Volatile.Read(ref _status); + + public void Start() + { + Publish(Status with + { + State = AhtolaAutomaticSyncState.Waiting, + Attempt = 0, + NextAttempt = DateTimeOffset.UtcNow + _interval, + }); + _task = RunAsync(_cancellation.Token); + } + + /// + /// Cancels the loop and waits for it. Returns the failure that stopped the loop, if any. + /// + public Exception? Stop() + { + if (Interlocked.Exchange(ref _stopped, 1) != 0) + return null; + if (_task is null) + { + _cancellation.Dispose(); + return null; + } + + try + { + _cancellation.Cancel(); + _task.GetAwaiter().GetResult(); + return null; + } + catch (OperationCanceledException) when (_cancellation.IsCancellationRequested) + { + return null; + } + catch (Exception exception) + { + return exception; + } + finally + { + _cancellation.Dispose(); + } + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + while (true) + { + await Task.Delay(_interval, cancellationToken).ConfigureAwait(false); + for (var attempt = 1; ; attempt++) + { + Publish(Status with + { + State = attempt == 1 ? AhtolaAutomaticSyncState.Running : AhtolaAutomaticSyncState.Retrying, + Attempt = attempt, + LastAttempt = DateTimeOffset.UtcNow, + NextAttempt = null, + }); + try + { + var result = await _operation(cancellationToken).ConfigureAwait(false); + var success = DateTimeOffset.UtcNow; + Publish(Status with + { + State = AhtolaAutomaticSyncState.Waiting, + Attempt = 0, + LastSuccess = success, + LastPullAppliedChanges = result.Outcome == AhtolaSyncOutcome.RemoteChangesApplied, + LastException = null, + NextAttempt = success + _interval, + }); + break; + } + catch (Exception exception) when ( + attempt < AhtolaConnection.AutomaticSyncMaximumAttempts + && AhtolaConnection.IsTransientAutomaticSyncFailure(exception, cancellationToken)) + { + var delay = AhtolaConnection.GetAutomaticSyncRetryDelay(attempt - 1); + Publish(Status with + { + State = AhtolaAutomaticSyncState.Retrying, + Attempt = attempt, + LastException = exception, + NextAttempt = DateTimeOffset.UtcNow + delay, + }); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + catch (Exception exception) when (!cancellationToken.IsCancellationRequested) + { + Publish(Status with + { + State = AhtolaAutomaticSyncState.Faulted, + Attempt = attempt, + LastException = exception, + NextAttempt = null, + }); + throw; + } + } + } + } + + private void Publish(AhtolaAutomaticSyncStatus status) + { + Volatile.Write(ref _status, status); + _publish(status); + } +} diff --git a/src/Ahtola.Data/AhtolaConnection.cs b/src/Ahtola.Data/AhtolaConnection.cs index 9cce6c5c..923f7f3a 100644 --- a/src/Ahtola.Data/AhtolaConnection.cs +++ b/src/Ahtola.Data/AhtolaConnection.cs @@ -32,8 +32,11 @@ public class AhtolaConnection : private AhtolaReplicaOptions? _replicaOptions; private HttpMessageHandler? _ownedReplicaHttpHandler; private readonly object _automaticSyncLock = new(); - private CancellationTokenSource? _automaticSyncCancellation; - private Task? _automaticSyncTask; + private ManagedReplicaAutomaticSyncCoordinator? _automaticSyncCoordinator; + private AhtolaAutomaticSyncStatus _automaticSyncStatus = AhtolaAutomaticSyncStatus.Stopped; + private readonly System.Collections.Concurrent.ConcurrentQueue _automaticSyncNotifications = new(); + private int _automaticSyncNotificationDrainScheduled; + private Func>? _authTokenProvider; private AhtolaEncryptionFileSystem? _managedEncryptionFileSystem; private AhtolaPageCodecFileSystem? _managedPageCodecFileSystem; private IPageCodec? _pageCodec; @@ -108,6 +111,52 @@ public IPageCodec? PageCodec } } + /// + /// Optional bearer-token provider for remote Hrana and embedded-replica connections, the + /// counterpart of Turso's serverless Builder::with_auth_token_fn. When set it takes + /// precedence over the Auth Token keyword and is invoked before every HTTP request + /// (and before every Hrana WebSocket connection handshake), so a rotated token takes effect + /// on the next request without reopening the connection. A or blank + /// result sends no token. Must be set before ; the provider must not call + /// back into this connection. + /// + /// + /// For a connection created by , configure + /// instead. + /// + public Func>? AuthTokenProvider + { + get => _authTokenProvider ?? _replicaOptions?.AuthTokenProvider; + set + { + if (State == ConnectionState.Open) + throw new InvalidOperationException("AuthTokenProvider cannot be set while the connection is open."); + if (_replicaOptions is not null) + { + throw new InvalidOperationException( + "Connections created by CreateReplica take their token provider from AhtolaReplicaOptions.AuthTokenProvider."); + } + + _authTokenProvider = value; + } + } + + /// + /// Gets the current status of the managed embedded-replica automatic synchronization loop + /// started by Sync Interval. when no + /// loop runs; after the loop stops, the last success/failure details are preserved. + /// + public AhtolaAutomaticSyncStatus AutomaticSyncStatus + => Volatile.Read(ref _automaticSyncCoordinator)?.Status ?? Volatile.Read(ref _automaticSyncStatus); + + /// + /// Raised, on a thread-pool thread and in publication order, whenever + /// changes, so background synchronization failures are + /// observable before rethrows them. Exceptions thrown by handlers are + /// ignored and never stop synchronization. + /// + public event EventHandler? AutomaticSyncStatusChanged; + protected override DbProviderFactory DbProviderFactory => AhtolaFactory.Instance; public AhtolaConnection() : this("") @@ -600,6 +649,169 @@ public Task SyncAsync( .SyncAsync(options, cancellationToken); } + /// + /// Pulls and applies remote changes without pushing local changes (Turso's + /// TursoSyncDatabase.Pull). Local changes that are still pending are preserved and + /// pushed by a later or . + /// + /// + /// The outcome; when remote changes were + /// applied. + /// + public AhtolaSyncResult Pull() + => PullAsync(new AhtolaSyncOptions(), CancellationToken.None).GetAwaiter().GetResult(); + + /// + /// Per-call synchronization options. + public AhtolaSyncResult Pull(AhtolaSyncOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return PullAsync(options, CancellationToken.None).GetAwaiter().GetResult(); + } + + /// + /// Cancels the pull. + public Task PullAsync(CancellationToken cancellationToken = default) + => PullAsync(new AhtolaSyncOptions(), cancellationToken); + + /// + /// Per-call synchronization options. + /// Cancels the pull. + public Task PullAsync(AhtolaSyncOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + ThrowIfCannotSynchronize(); + if (cancellationToken.IsCancellationRequested) + return Task.FromCanceled(cancellationToken); + if (_managedReplicaHost is { } managedReplicaHost) + { + return RunManagedReplicaSyncOperationAsync( + managedReplicaHost, + (host, token) => host.PullAsync(options, token), + cancellationToken); + } + + return GetRegisteredReplicaDatabase().PullAsync(options, cancellationToken); + } + + /// + /// Pushes every local change that is pending when the call starts, without pulling remote + /// changes (Turso's TursoSyncDatabase.Push). Batches stay bounded by + /// Push Operations Threshold; several are sent when needed. + /// + /// + /// A result whose is the number of changes + /// pushed. Its outcome is always because a push never + /// applies remote changes. + /// + public AhtolaSyncResult Push() + => PushAsync(new AhtolaSyncOptions(), CancellationToken.None).GetAwaiter().GetResult(); + + /// + /// Per-call synchronization options. + public AhtolaSyncResult Push(AhtolaSyncOptions options) + { + ArgumentNullException.ThrowIfNull(options); + return PushAsync(options, CancellationToken.None).GetAwaiter().GetResult(); + } + + /// + /// Cancels the push. + public Task PushAsync(CancellationToken cancellationToken = default) + => PushAsync(new AhtolaSyncOptions(), cancellationToken); + + /// + /// Per-call synchronization options. + /// Cancels the push. + public Task PushAsync(AhtolaSyncOptions options, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + ThrowIfCannotSynchronize(); + if (cancellationToken.IsCancellationRequested) + return Task.FromCanceled(cancellationToken); + if (_managedReplicaHost is { } managedReplicaHost) + { + return RunManagedReplicaSyncOperationAsync( + managedReplicaHost, + (host, token) => host.PushAsync(options, token), + cancellationToken); + } + + return GetRegisteredReplicaDatabase().PushAsync(options, cancellationToken); + } + + /// + /// Checkpoints the embedded replica's local write-ahead log into its main database file + /// (Turso's TursoSyncDatabase.Checkpoint). Unpushed local changes stay journaled and + /// are pushed by the next push or sync. A pending checkpoint-recovery bundle, an unknown push + /// outcome, an unresolved push conflict, or a partial replica that is still lazily loading + /// pages makes the call fail closed; synchronize first. + /// + public void Checkpoint() + => CheckpointAsync(CancellationToken.None).GetAwaiter().GetResult(); + + /// + /// Cancels the checkpoint. + public Task CheckpointAsync(CancellationToken cancellationToken = default) + { + ThrowIfCannotSynchronize(); + if (cancellationToken.IsCancellationRequested) + return Task.FromCanceled(cancellationToken); + if (_managedReplicaHost is { } managedReplicaHost) + { + return RunManagedReplicaSyncOperationAsync( + managedReplicaHost, + async (host, token) => + { + await host.CheckpointAsync(token).ConfigureAwait(false); + return true; + }, + cancellationToken); + } + + return GetRegisteredReplicaDatabase().CheckpointAsync(cancellationToken); + } + + /// + /// Returns a snapshot of the embedded replica's synchronization statistics (Turso's + /// TursoSyncDatabase.GetStats): pending unpushed local changes, current main and revert + /// WAL sizes, the synchronized server revision, and -- for the managed provider, tracked per + /// replica file within this process -- the last successful pull and push times and the + /// cumulative pull-updates traffic. Pure read: no network access and no local mutation. + /// + public AhtolaSyncStatistics GetSyncStatistics() + { + ThrowIfCannotSynchronize(); + return _managedReplicaHost is { } managedReplicaHost + ? managedReplicaHost.GetSyncStatistics() + : GetRegisteredReplicaDatabase().GetSyncStatisticsAsync(CancellationToken.None).GetAwaiter().GetResult(); + } + + /// + /// Cancels the read. + public Task GetSyncStatisticsAsync(CancellationToken cancellationToken = default) + { + ThrowIfCannotSynchronize(); + if (cancellationToken.IsCancellationRequested) + return Task.FromCanceled(cancellationToken); + return _managedReplicaHost is { } managedReplicaHost + ? Task.FromResult(managedReplicaHost.GetSyncStatistics()) + : GetRegisteredReplicaDatabase().GetSyncStatisticsAsync(cancellationToken); + } + + private void ThrowIfCannotSynchronize() + { + _replicaOptions?.ThrowIfApplicationHttpReentrant(closing: false); + ObjectDisposedException.ThrowIf(_disposed, this); + if (State != ConnectionState.Open) + throw new InvalidOperationException("Ahtola database is closed."); + if (_managedReplicaHost is null && !Capabilities.SupportsSync) + throw new NotSupportedException("Sync requires an embedded replica connection."); + } + + private AhtolaReplicaDatabase GetRegisteredReplicaDatabase() + => _replicaDatabase ?? throw new InvalidOperationException("Ahtola database is closed."); + /// /// Returns an immutable classification of the managed embedded replica's open push conflict, /// or when no conflict is recorded. Pure read: no network access and no @@ -713,10 +925,19 @@ internal async Task QuiesceManagedReplicaAsync( _managedDatabase = host.Database; } - private async Task SyncManagedReplicaAsync( + private Task SyncManagedReplicaAsync( ManagedReplicaConnectionHost host, AhtolaSyncOptions options, CancellationToken cancellationToken) + => RunManagedReplicaSyncOperationAsync( + host, + (replicaHost, token) => replicaHost.SyncAsync(options, token), + cancellationToken); + + private async Task RunManagedReplicaSyncOperationAsync( + ManagedReplicaConnectionHost host, + Func> operation, + CancellationToken cancellationToken) { if (_transaction is not null) throw new InvalidOperationException("Managed embedded replica sync cannot run while a transaction is active."); @@ -728,7 +949,7 @@ private async Task SyncManagedReplicaAsync( try { - return await host.SyncAsync(options, cancellationToken).ConfigureAwait(false); + return await operation(host, cancellationToken).ConfigureAwait(false); } finally { @@ -1493,6 +1714,8 @@ private void OpenRemote() } ValidateDirectRemoteLocalProvider(); + if (_connectionOptions.HasAdvancedReplicaOptions) + throw new NotSupportedException("Advanced sync options require an embedded replica connection."); var remoteEncryption = _connectionOptions.GetRemoteEncryptionOptions(); if (_connectionOptions.IsWebSocketRemote) { @@ -1503,20 +1726,22 @@ private void OpenRemote() _connectionOptions.AuthToken, _connectionOptions.GetWebSocketOptions(), remoteEncryption, - RemoteWebSocketConnectorFactory?.Invoke()); + RemoteWebSocketConnectorFactory?.Invoke(), + _authTokenProvider); return; } var endpoint = _connectionOptions.GetRemoteUri(); var handler = RemoteMessageHandlerFactory?.Invoke(); _remoteClient = handler is null - ? new AhtolaRemoteClient(endpoint, _connectionOptions.AuthToken, remoteEncryption) + ? new AhtolaRemoteClient(endpoint, _connectionOptions.AuthToken, remoteEncryption, _authTokenProvider) : new AhtolaRemoteClient( new HttpClient(handler, disposeHandler: false), endpoint, _connectionOptions.AuthToken, remoteEncryption, - disposeHttpClient: true); + disposeHttpClient: true, + authTokenProvider: _authTokenProvider); } private async Task OpenRemoteReplicaAsync( @@ -1597,21 +1822,16 @@ private AhtolaReplicaOptions GetReplicaOptions() || !string.IsNullOrWhiteSpace(_connectionOptions["Encryption Key"])) { throw new InvalidOperationException( - "Encryption Cipher and Encryption Key are local database options and cannot be used with remote Ahtola URLs."); + "Encryption Cipher and Encryption Key are local database options and cannot be used with remote Ahtola URLs. " + + "Use Remote Encryption Cipher and Remote Encryption Key for embedded replicas."); } if (_replicaOptions is not null) return _replicaOptions; - var handler = RemoteMessageHandlerFactory?.Invoke(); - return new AhtolaReplicaOptions( - _connectionOptions.ReplicaPath, - _connectionOptions.GetRemoteUri(), - _connectionOptions.AuthToken) - { - SyncInterval = _connectionOptions.SyncInterval, - HttpPolicy = new AhtolaSyncHttpPolicy(handler), - }; + return _connectionOptions.CreateReplicaOptions( + RemoteMessageHandlerFactory?.Invoke(), + _authTokenProvider); } private void SetReplicaDatabase(AhtolaReplicaDatabase replicaDatabase) @@ -1705,51 +1925,75 @@ private void StartAutomaticManagedReplicaSync(ManagedReplicaConnectionHost repli if (syncInterval <= 0 || !replicaHost.SupportsSync) return; - var cancellation = new CancellationTokenSource(); + var mode = _replicaOptions?.AutomaticSyncMode ?? _connectionOptions.AutomaticSyncMode; + Func> operation = mode == AhtolaAutomaticSyncMode.PullOnly + ? token => replicaHost.PullAsync(new AhtolaSyncOptions(), token) + : token => replicaHost.SyncAsync(new AhtolaSyncOptions(), token); lock (_automaticSyncLock) { - if (_automaticSyncTask is not null) - { - cancellation.Dispose(); + if (_automaticSyncCoordinator is not null) throw new InvalidOperationException("Automatic managed replica synchronization is already running."); - } - _automaticSyncCancellation = cancellation; - _automaticSyncTask = RunAutomaticManagedReplicaSyncAsync( - replicaHost, + var coordinator = new ManagedReplicaAutomaticSyncCoordinator( + operation, TimeSpan.FromSeconds(syncInterval), - cancellation.Token); + Volatile.Read(ref _automaticSyncStatus), + PublishAutomaticSyncStatus); + Volatile.Write(ref _automaticSyncCoordinator, coordinator); + coordinator.Start(); } } - private static async Task RunAutomaticManagedReplicaSyncAsync( - ManagedReplicaConnectionHost replicaHost, - TimeSpan interval, - CancellationToken cancellationToken) + private void PublishAutomaticSyncStatus(AhtolaAutomaticSyncStatus status) { - while (true) - { - await Task.Delay(interval, cancellationToken).ConfigureAwait(false); - await SynchronizeManagedReplicaWithRetryAsync(replicaHost, cancellationToken).ConfigureAwait(false); - } + Volatile.Write(ref _automaticSyncStatus, status); + if (AutomaticSyncStatusChanged is null) + return; + + _automaticSyncNotifications.Enqueue(status); + if (Interlocked.Exchange(ref _automaticSyncNotificationDrainScheduled, 1) != 0) + return; + + ThreadPool.UnsafeQueueUserWorkItem( + static connection => connection.DrainAutomaticSyncNotifications(), + this, + preferLocal: false); } - private static async Task SynchronizeManagedReplicaWithRetryAsync( - ManagedReplicaConnectionHost replicaHost, - CancellationToken cancellationToken) + private void DrainAutomaticSyncNotifications() { - for (var attempt = 1; ; attempt++) + try { - try + while (_automaticSyncNotifications.TryDequeue(out var status)) { - _ = await replicaHost.SyncAsync(new AhtolaSyncOptions(), cancellationToken).ConfigureAwait(false); - return; + var handlers = AutomaticSyncStatusChanged; + if (handlers is null) + continue; + + var args = new AhtolaAutomaticSyncStatusChangedEventArgs(status); + foreach (var callback in handlers.GetInvocationList()) + { + try + { + ((EventHandler)callback)(this, args); + } + catch + { + // Observers cannot stop synchronization. + } + } } - catch (Exception exception) when ( - attempt < AutomaticSyncMaximumAttempts - && IsTransientAutomaticSyncFailure(exception, cancellationToken)) + } + finally + { + Volatile.Write(ref _automaticSyncNotificationDrainScheduled, 0); + if (!_automaticSyncNotifications.IsEmpty + && Interlocked.Exchange(ref _automaticSyncNotificationDrainScheduled, 1) == 0) { - await Task.Delay(GetAutomaticSyncRetryDelay(attempt - 1), cancellationToken).ConfigureAwait(false); + ThreadPool.UnsafeQueueUserWorkItem( + static connection => connection.DrainAutomaticSyncNotifications(), + this, + preferLocal: false); } } } @@ -1777,37 +2021,25 @@ internal static bool IsTransientAutomaticSyncFailure(Exception exception, Cancel private Exception? StopAutomaticManagedReplicaSync() { - CancellationTokenSource? cancellation; - Task? syncTask; + ManagedReplicaAutomaticSyncCoordinator? coordinator; lock (_automaticSyncLock) - { - cancellation = _automaticSyncCancellation; - syncTask = _automaticSyncTask; - _automaticSyncCancellation = null; - _automaticSyncTask = null; - } + coordinator = Volatile.Read(ref _automaticSyncCoordinator); - if (cancellation is null || syncTask is null) + if (coordinator is null) return null; - try - { - cancellation.Cancel(); - syncTask.GetAwaiter().GetResult(); - return null; - } - catch (OperationCanceledException) when (cancellation.IsCancellationRequested) - { - return null; - } - catch (Exception exception) - { - return exception; - } - finally + var failure = coordinator.Stop(); + lock (_automaticSyncLock) + Volatile.Write(ref _automaticSyncCoordinator, null); + + // Keep the last success/failure details so a faulted loop stays diagnosable after Close. + PublishAutomaticSyncStatus(coordinator.Status with { - cancellation.Dispose(); - } + State = AhtolaAutomaticSyncState.Stopped, + Attempt = 0, + NextAttempt = null, + }); + return failure; } private void OpenCore() @@ -1870,6 +2102,10 @@ private void ValidateLocalOnlyOptions() throw new InvalidOperationException("Auth Token requires a remote Ahtola URL Data Source."); if (!string.IsNullOrWhiteSpace(_connectionOptions.ReplicaPath)) throw new InvalidOperationException("Replica Path requires a remote Ahtola URL Data Source."); + if (_connectionOptions.HasAdvancedReplicaOptions) + throw new InvalidOperationException("Advanced sync options require a remote embedded replica connection."); + if (_authTokenProvider is not null) + throw new InvalidOperationException("AuthTokenProvider requires a remote Ahtola URL Data Source."); if (_connectionOptions.Tls.HasValue) throw new InvalidOperationException("Tls requires a remote Ahtola URL Data Source."); } diff --git a/src/Ahtola.Data/AhtolaConnectionOptions.cs b/src/Ahtola.Data/AhtolaConnectionOptions.cs index bf65b850..a3475dae 100644 --- a/src/Ahtola.Data/AhtolaConnectionOptions.cs +++ b/src/Ahtola.Data/AhtolaConnectionOptions.cs @@ -45,8 +45,137 @@ public string? this[string keyword] public int SyncInterval => _builder.SyncInterval; + /// Gets what each automatic synchronization tick does. + public AhtolaAutomaticSyncMode AutomaticSyncMode => _builder.AutomaticSyncMode; + public bool? Tls => _builder.Tls; + private static readonly string[] AdvancedReplicaKeywords = + [ + "Sync Client Name", + "Sync Long Poll Timeout", + "Bootstrap If Empty", + "Partial Bootstrap Prefix", + "Partial Bootstrap Query", + "Partial Sync Segment Size", + "Partial Sync Prefetch", + "Remote Encryption Cipher", + "Remote Encryption Key", + "Push Operations Threshold", + "Pull Bytes Threshold", + "Force Logical MVCC Pull", + "Sync Experimental Features", + "Automatic Sync Mode", + ]; + + /// + /// True when any embedded-replica-only keyword (beyond Replica Path and + /// Sync Interval) is present, matching Turso's HasAdvancedReplicaOptions. + /// + internal bool HasAdvancedReplicaOptions + => Array.Exists(AdvancedReplicaKeywords, _builder.ContainsKey); + + /// + /// Maps the embedded-replica keywords of this connection string onto + /// , mirroring Turso's CreateReplicaOptions + /// (turso-src/bindings/dotnet/src/Turso.Data/TursoConnection.cs). + /// + internal AhtolaReplicaOptions CreateReplicaOptions( + HttpMessageHandler? messageHandler, + Func>? authTokenProvider) + { + if (_builder.ForceLogicalMvccPull) + { + throw new NotSupportedException( + "Force Logical MVCC Pull=True is not supported by the managed embedded replica: the pull " + + "protocol is always auto-detected from the remote and persisted in the replica metadata."); + } + + ValidateExperimentalFeatures(_builder.SyncExperimentalFeatures); + var clientName = _builder.SyncClientName; + var longPollMilliseconds = _builder.SyncLongPollTimeout; + var pushOperationsThreshold = _builder.PushOperationsThreshold; + var pullBytesThreshold = _builder.PullBytesThreshold; + return new AhtolaReplicaOptions( + ReplicaPath, + GetRemoteUri(), + AuthToken, + _builder.BootstrapIfEmpty) + { + SyncInterval = SyncInterval, + AutomaticSyncMode = AutomaticSyncMode, + ClientName = string.IsNullOrWhiteSpace(clientName) ? null : clientName, + LongPollTimeout = longPollMilliseconds == 0 ? null : TimeSpan.FromMilliseconds(longPollMilliseconds), + PartialBootstrap = GetPartialBootstrapOptions(), + RemoteEncryption = GetReplicaRemoteEncryptionOptions(), + PushOperationsThreshold = pushOperationsThreshold == 0 ? null : pushOperationsThreshold, + PullBytesThreshold = pullBytesThreshold == 0 ? null : pullBytesThreshold, + AuthTokenProvider = authTokenProvider, + HttpPolicy = new AhtolaSyncHttpPolicy(messageHandler), + }; + } + + private AhtolaPartialBootstrapOptions? GetPartialBootstrapOptions() + { + var prefix = _builder.PartialBootstrapPrefix; + var query = _builder.PartialBootstrapQuery; + var segmentSize = _builder.PartialSyncSegmentSize; + var prefetch = _builder.PartialSyncPrefetch; + var hasQuery = !string.IsNullOrWhiteSpace(query); + if (prefix != 0 && hasQuery) + { + throw new InvalidOperationException( + "Partial Bootstrap Prefix and Partial Bootstrap Query cannot be combined."); + } + + if (prefix == 0 && !hasQuery) + { + if (segmentSize != 0 || prefetch) + { + throw new InvalidOperationException( + "Partial Sync Segment Size and Partial Sync Prefetch require Partial Bootstrap Prefix or Partial Bootstrap Query."); + } + + return null; + } + + long? segment = segmentSize == 0 ? null : segmentSize; + return hasQuery + ? AhtolaPartialBootstrapOptions.QueryPages(query, segment, prefetch) + : AhtolaPartialBootstrapOptions.Prefix(prefix, segment, prefetch); + } + + private AhtolaRemoteEncryptionOptions? GetReplicaRemoteEncryptionOptions() + { + var cipher = _builder.RemoteEncryptionCipher; + var key = _builder.RemoteEncryptionKey; + if (string.IsNullOrWhiteSpace(cipher) && string.IsNullOrWhiteSpace(key)) + return null; + if (string.IsNullOrWhiteSpace(cipher) || string.IsNullOrWhiteSpace(key)) + { + throw new InvalidOperationException( + "Remote Encryption Cipher and Remote Encryption Key must be specified together."); + } + + return new AhtolaRemoteEncryptionOptions(key, AhtolaRemoteEncryptionOptions.ParseCipher(cipher)); + } + + private static void ValidateExperimentalFeatures(string features) + { + if (string.IsNullOrWhiteSpace(features)) + return; + + foreach (var feature in features.Split(',')) + { + var name = feature.Trim(); + if (name.Length == 0 || !name.All(static c => char.IsAsciiLetterOrDigit(c) || c == '_')) + { + throw new ArgumentException( + $"Sync Experimental Features must be a comma-separated list of feature names: {features}"); + } + } + } + public AhtolaLocalProvider LocalProvider => _builder.IsLocalProviderConfigured ? _builder.LocalProvider : IsRemote @@ -331,6 +460,8 @@ internal static AhtolaConnectionOptions FromReplica(AhtolaReplicaOptions options if (!string.IsNullOrWhiteSpace(options.AuthToken)) builder.AuthToken = options.AuthToken; builder.SyncInterval = options.SyncInterval; + if (options.AutomaticSyncMode != AhtolaAutomaticSyncMode.PushAndPull) + builder.AutomaticSyncMode = options.AutomaticSyncMode; return new AhtolaConnectionOptions(builder); } diff --git a/src/Ahtola.Data/AhtolaConnectionStringBuilder.cs b/src/Ahtola.Data/AhtolaConnectionStringBuilder.cs index d0b3f00b..6af4320a 100644 --- a/src/Ahtola.Data/AhtolaConnectionStringBuilder.cs +++ b/src/Ahtola.Data/AhtolaConnectionStringBuilder.cs @@ -38,6 +38,35 @@ public sealed class AhtolaConnectionStringBuilder : DbConnectionStringBuilder ["ReadYourWrites"] = "Read Your Writes", ["Sync Interval"] = "Sync Interval", ["SyncInterval"] = "Sync Interval", + // Advanced embedded-replica options, named as in Turso's TursoConnectionStringBuilder. + ["Sync Client Name"] = "Sync Client Name", + ["SyncClientName"] = "Sync Client Name", + ["Sync Long Poll Timeout"] = "Sync Long Poll Timeout", + ["SyncLongPollTimeout"] = "Sync Long Poll Timeout", + ["Bootstrap If Empty"] = "Bootstrap If Empty", + ["BootstrapIfEmpty"] = "Bootstrap If Empty", + ["Partial Bootstrap Prefix"] = "Partial Bootstrap Prefix", + ["PartialBootstrapPrefix"] = "Partial Bootstrap Prefix", + ["Partial Bootstrap Query"] = "Partial Bootstrap Query", + ["PartialBootstrapQuery"] = "Partial Bootstrap Query", + ["Partial Sync Segment Size"] = "Partial Sync Segment Size", + ["PartialSyncSegmentSize"] = "Partial Sync Segment Size", + ["Partial Sync Prefetch"] = "Partial Sync Prefetch", + ["PartialSyncPrefetch"] = "Partial Sync Prefetch", + ["Remote Encryption Cipher"] = "Remote Encryption Cipher", + ["RemoteEncryptionCipher"] = "Remote Encryption Cipher", + ["Remote Encryption Key"] = "Remote Encryption Key", + ["RemoteEncryptionKey"] = "Remote Encryption Key", + ["Push Operations Threshold"] = "Push Operations Threshold", + ["PushOperationsThreshold"] = "Push Operations Threshold", + ["Pull Bytes Threshold"] = "Pull Bytes Threshold", + ["PullBytesThreshold"] = "Pull Bytes Threshold", + ["Force Logical MVCC Pull"] = "Force Logical MVCC Pull", + ["ForceLogicalMvccPull"] = "Force Logical MVCC Pull", + ["Sync Experimental Features"] = "Sync Experimental Features", + ["SyncExperimentalFeatures"] = "Sync Experimental Features", + ["Automatic Sync Mode"] = "Automatic Sync Mode", + ["AutomaticSyncMode"] = "Automatic Sync Mode", ["Tls"] = "Tls", ["TLS"] = "Tls", ["Local Provider"] = "Local Provider", @@ -175,6 +204,170 @@ public int SyncInterval } } + /// + /// Gets or sets the client name that prefixes the sync client identifier of a newly + /// bootstrapped embedded replica (). + /// + public string SyncClientName + { + get => GetString("Sync Client Name"); + set => SetString("Sync Client Name", value); + } + + /// + /// Gets or sets the embedded-replica pull long-poll timeout in milliseconds; 0 (the default) + /// disables long polling (). + /// + public int SyncLongPollTimeout + { + get => GetInt("Sync Long Poll Timeout", 0); + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + this["Sync Long Poll Timeout"] = value; + } + } + + /// + /// Gets or sets whether a missing embedded replica is bootstrapped from the remote database + /// (default ; ). + /// + public bool BootstrapIfEmpty + { + get => GetBool("Bootstrap If Empty", defaultValue: true); + set => this["Bootstrap If Empty"] = value; + } + + /// + /// Gets or sets the byte prefix of a partial bootstrap + /// (); 0 disables it. + /// + public int PartialBootstrapPrefix + { + get => GetInt("Partial Bootstrap Prefix", 0); + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + this["Partial Bootstrap Prefix"] = value; + } + } + + /// + /// Gets or sets the server-side query of a partial bootstrap + /// (). + /// + public string PartialBootstrapQuery + { + get => GetString("Partial Bootstrap Query"); + set => SetString("Partial Bootstrap Query", value); + } + + /// + /// Gets or sets the lazy-loading segment size in bytes of a partial bootstrap; 0 uses the + /// default (). + /// + public long PartialSyncSegmentSize + { + get => GetLong("Partial Sync Segment Size", 0); + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + this["Partial Sync Segment Size"] = value; + } + } + + /// + /// Gets or sets whether a partial bootstrap prefetches adjacent pages + /// (). + /// + public bool PartialSyncPrefetch + { + get => GetBool("Partial Sync Prefetch"); + set => this["Partial Sync Prefetch"] = value; + } + + /// + /// Gets or sets the cipher of an encrypted remote database used by an embedded replica + /// (for example aes256gcm or aegis256; ). + /// + public string RemoteEncryptionCipher + { + get => GetString("Remote Encryption Cipher"); + set => SetString("Remote Encryption Cipher", value); + } + + /// + /// Gets or sets the base64-encoded key of an encrypted remote database used by an embedded + /// replica (). + /// + public string RemoteEncryptionKey + { + get => GetString("Remote Encryption Key"); + set => SetString("Remote Encryption Key", value); + } + + /// + /// Gets or sets the maximum number of change operations per push batch; 0 uses the default + /// (). + /// + public long PushOperationsThreshold + { + get => GetLong("Push Operations Threshold", 0); + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + this["Push Operations Threshold"] = value; + } + } + + /// + /// Gets or sets the target size in bytes of each initial bootstrap pull; 0 pulls the whole + /// image at once (). + /// + public long PullBytesThreshold + { + get => GetLong("Pull Bytes Threshold", 0); + set + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + this["Pull Bytes Threshold"] = value; + } + } + + /// + /// Gets or sets Turso's escape hatch that forces MVCC logical-log pulls. The managed provider + /// always auto-detects the pull protocol from the remote and persists it, so only + /// is supported; fails closed when the + /// connection opens. + /// + public bool ForceLogicalMvccPull + { + get => GetBool("Force Logical MVCC Pull"); + set => this["Force Logical MVCC Pull"] = value; + } + + /// + /// Gets or sets Turso's comma-separated experimental-feature list for the replica's local + /// database. Accepted for connection-string compatibility: the managed engine does not gate + /// features behind these names, so the value is validated but has no effect. + /// + public string SyncExperimentalFeatures + { + get => GetString("Sync Experimental Features"); + set => SetString("Sync Experimental Features", value); + } + + /// + /// Gets or sets what each automatic synchronization tick () does. + /// The default, , keeps Ahtola's full sync; + /// matches Turso's pull-only loop. + /// + public AhtolaAutomaticSyncMode AutomaticSyncMode + { + get => GetEnum("Automatic Sync Mode", AhtolaAutomaticSyncMode.PushAndPull); + set => this["Automatic Sync Mode"] = value; + } + public bool? Tls { get => GetNullableBool("Tls"); @@ -386,6 +579,13 @@ private int GetInt(string keyword, int defaultValue) : defaultValue; } + private long GetLong(string keyword, long defaultValue) + { + return TryGetValue(keyword, out var value) + ? Convert.ToInt64(value, CultureInfo.InvariantCulture) + : defaultValue; + } + private TEnum GetEnum(string keyword, TEnum defaultValue) where TEnum : struct, Enum { diff --git a/src/Ahtola.Data/AhtolaRemoteClient.cs b/src/Ahtola.Data/AhtolaRemoteClient.cs index 2b8b5ce8..0540e71c 100644 --- a/src/Ahtola.Data/AhtolaRemoteClient.cs +++ b/src/Ahtola.Data/AhtolaRemoteClient.cs @@ -19,6 +19,7 @@ internal sealed partial class AhtolaRemoteClient : IDisposable private readonly HttpClient? _httpClient; private readonly string? _authToken; + private readonly Func>? _authTokenProvider; private readonly string? _remoteEncryptionKey; private readonly bool _disposeHttpClient; private Uri _pipelineUri; @@ -33,24 +34,38 @@ internal sealed partial class AhtolaRemoteClient : IDisposable public AhtolaRemoteClient( Uri endpoint, string? authToken, - AhtolaRemoteEncryptionOptions? remoteEncryption = null) + AhtolaRemoteEncryptionOptions? remoteEncryption = null, + Func>? authTokenProvider = null) : this( AhtolaRemoteTransportSecurity.CreateRedirectSafeHttpClient(), endpoint, authToken, remoteEncryption, disposeHttpClient: true, - automaticRedirectsDisabled: true) + automaticRedirectsDisabled: true, + authTokenProvider) { } + /// The HTTP transport. + /// The Hrana endpoint. + /// A static bearer token, used when no provider is configured. + /// Optional remote encryption configuration. + /// Whether this client owns . + /// Whether the transport never follows redirects itself. + /// + /// Optional bearer-token provider. When set it takes precedence over + /// and is invoked once per HTTP request, so a rotated token is + /// picked up by the next request without reopening the connection. + /// internal AhtolaRemoteClient( HttpClient httpClient, Uri endpoint, string? authToken, AhtolaRemoteEncryptionOptions? remoteEncryption = null, bool disposeHttpClient = false, - bool automaticRedirectsDisabled = false) + bool automaticRedirectsDisabled = false, + Func>? authTokenProvider = null) { ArgumentNullException.ThrowIfNull(httpClient); ArgumentNullException.ThrowIfNull(endpoint); @@ -62,10 +77,11 @@ internal AhtolaRemoteClient( _protocolVersion == RemoteProtocolVersion.V2 ? "/v2/pipeline" : "/v3/pipeline"); _cursorUri = CreateProtocolUri(endpoint, "/v3/cursor"); _authToken = string.IsNullOrWhiteSpace(authToken) ? null : authToken; + _authTokenProvider = authTokenProvider; _remoteEncryptionKey = remoteEncryption?.Base64Key; AhtolaRemoteTransportSecurity.Validate( _pipelineUri, - _authToken, + CredentialValidationToken, remoteEncryptionConfigured: _remoteEncryptionKey is not null); AhtolaRemoteTransportSecurity.ValidateRedirectContract( automaticRedirectsDisabled, @@ -75,6 +91,32 @@ internal AhtolaRemoteClient( public bool HasOpenSession => _webSocketTransport?.HasOpenSession ?? _baton is not null; + /// + /// A non-null stand-in whenever a bearer credential is configured (statically or through a + /// provider), used only for the up-front HTTPS and redirect-origin credential policy checks. + /// + private string? CredentialValidationToken + => _authToken ?? (_authTokenProvider is null ? null : AuthTokenProviderPlaceholder); + + /// Stand-in credential used for transport policy checks when only a provider is set. + internal const string AuthTokenProviderPlaceholder = ""; + + /// + /// Resolves the bearer token for one request: the provider when configured (a blank result + /// means "send no token"), otherwise the static token. + /// + internal static async ValueTask ResolveAuthTokenAsync( + string? staticToken, + Func>? provider, + CancellationToken cancellationToken) + { + if (provider is null) + return string.IsNullOrWhiteSpace(staticToken) ? null : staticToken; + + var token = await provider(cancellationToken).ConfigureAwait(false); + return string.IsNullOrWhiteSpace(token) ? null : token; + } + public void ResetSession() { _webSocketTransport?.ResetSession(); @@ -643,17 +685,22 @@ private async Task SendProtocolRequestAsync( Uri requestUri, string json, CancellationToken cancellationToken) - => await AhtolaRemoteTransportSecurity + { + var httpClient = _httpClient ?? throw new InvalidOperationException( + "This remote client uses the Hrana WebSocket transport and has no HTTP pipeline."); + var authToken = await ResolveAuthTokenAsync(_authToken, _authTokenProvider, cancellationToken) + .ConfigureAwait(false); + return await AhtolaRemoteTransportSecurity .SendAsync( - _httpClient ?? throw new InvalidOperationException( - "This remote client uses the Hrana WebSocket transport and has no HTTP pipeline."), + httpClient, requestUri, - uri => CreateProtocolHttpRequest(uri, json), - _authToken, + uri => CreateProtocolHttpRequest(uri, json, authToken), + authToken ?? CredentialValidationToken, remoteEncryptionConfigured: _remoteEncryptionKey is not null, HttpCompletionOption.ResponseHeadersRead, cancellationToken) .ConfigureAwait(false); + } private static async Task ReadResponseBodyAsync( HttpResponseMessage response, @@ -715,15 +762,15 @@ private bool ShouldFallbackToV2(HttpResponseMessage response, string? requestBat return timeout; } - private HttpRequestMessage CreateProtocolHttpRequest(Uri requestUri, string json) + private HttpRequestMessage CreateProtocolHttpRequest(Uri requestUri, string json, string? authToken) { var request = new HttpRequestMessage(HttpMethod.Post, requestUri) { Content = new StringContent(json, Encoding.UTF8, "application/json"), }; - if (_authToken is not null) - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _authToken); + if (authToken is not null) + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", authToken); if (_remoteEncryptionKey is not null) request.Headers.TryAddWithoutValidation(EncryptionKeyHeaderName, _remoteEncryptionKey); return request; @@ -928,32 +975,30 @@ private IReadOnlyList ProcessBatchResult( replayedChangeContexts); } + /// + /// Tracks the highest batch/step replication_index watermark a libSQL (sqld) server + /// reported, so later batches can ask a replica-backed server to serve reads at least that + /// fresh. The pinned Hrana spec (turso-src/serverless/PROTOCOL.md section 7.2.6) now + /// says clients MUST NOT interpret the value and Turso servers send , so + /// this is best-effort: a value that is not a non-negative integer (string or legacy number) + /// is ignored rather than failing the request, and the previously tracked watermark is kept. + /// private void UpdateReplicationIndex(JsonElement encodedIndex) { - if (encodedIndex.ValueKind is JsonValueKind.Undefined or JsonValueKind.Null) - return; - - ulong index; - if (encodedIndex.ValueKind == JsonValueKind.String) + var parsed = encodedIndex.ValueKind switch { - if (!ulong.TryParse( + JsonValueKind.String => ulong.TryParse( encodedIndex.GetString(), NumberStyles.None, CultureInfo.InvariantCulture, - out index)) - { - throw new AhtolaException("Remote response returned an invalid replication_index."); - } - } - else if (encodedIndex.ValueKind == JsonValueKind.Number) - { - if (!encodedIndex.TryGetUInt64(out index)) - throw new AhtolaException("Remote response returned an invalid replication_index."); - } - else - { - throw new AhtolaException("Remote response returned an invalid replication_index."); - } + out var text) + ? text + : (ulong?)null, + JsonValueKind.Number => encodedIndex.TryGetUInt64(out var number) ? number : (ulong?)null, + _ => (ulong?)null, + }; + if (parsed is not { } index) + return; if (_replicationIndex is null || index > _replicationIndex.Value) _replicationIndex = index; @@ -1113,10 +1158,10 @@ private void UpdateBaseUrl(string? baseUrl) AhtolaRemoteTransportSecurity.ValidateRedirectOrigin( _pipelineUri, pipelineUri, - credentialsConfigured: _authToken is not null || _remoteEncryptionKey is not null); + credentialsConfigured: CredentialValidationToken is not null || _remoteEncryptionKey is not null); AhtolaRemoteTransportSecurity.Validate( pipelineUri, - _authToken, + CredentialValidationToken, remoteEncryptionConfigured: _remoteEncryptionKey is not null); _pipelineUri = pipelineUri; _cursorUri = cursorUri; diff --git a/src/Ahtola.Data/AhtolaReplicaProvider.cs b/src/Ahtola.Data/AhtolaReplicaProvider.cs index e67bff88..54d1b81a 100644 --- a/src/Ahtola.Data/AhtolaReplicaProvider.cs +++ b/src/Ahtola.Data/AhtolaReplicaProvider.cs @@ -133,18 +133,99 @@ public AhtolaReplicaOptions( /// public int SyncInterval { get; init; } + /// + /// Gets or initializes what each automatic synchronization tick does. The default, + /// , keeps Ahtola's historical full sync; + /// matches Turso's pull-only loop. + /// + public AhtolaAutomaticSyncMode AutomaticSyncMode { get; init; } + + /// + /// Gets or initializes a client name used as the prefix of the sync client identifier that a + /// newly bootstrapped replica records (<name>-<guid>), mirroring Turso's + /// client_name. keeps the plain GUID identifier. An already + /// bootstrapped replica keeps the identifier it was created with. Allowed characters are ASCII + /// letters, digits, ., _ and -, up to 64 characters. + /// + public string? ClientName { get; init; } + + /// + /// Gets or initializes an optional bearer-token provider. When set it takes precedence over + /// and is invoked before every sync HTTP request (bootstrap, pull, + /// lazy page fetch and push), so a rotated token takes effect without reopening the + /// connection. A or blank result sends no token. The provider must not + /// call back into the replica connection. + /// + public Func>? AuthTokenProvider { get; init; } + /// /// Gets or initializes the HTTP transport policy. /// public AhtolaSyncHttpPolicy HttpPolicy { get; init; } = new(); + /// Resolves the bearer token for one sync HTTP request. + internal ValueTask ResolveAuthTokenAsync(CancellationToken cancellationToken) + => AhtolaRemoteClient.ResolveAuthTokenAsync(AuthToken, AuthTokenProvider, cancellationToken); + + /// + /// Returns the sync client identifier for a new bootstrap: a GUID, prefixed by + /// when one is configured. + /// + internal string CreateClientId() + { + var id = Guid.NewGuid().ToString("N"); + return string.IsNullOrEmpty(ClientName) ? id : string.Concat(ClientName, "-", id); + } + + /// + /// Whether is a valid persisted sync client identifier: a + /// 32-digit GUID, optionally prefixed by a valid client name and -. + /// + internal static bool IsValidClientId(string clientId) + { + if (Guid.TryParseExact(clientId, "N", out _)) + return true; + + var separator = clientId.LastIndexOf('-'); + return separator > 0 + && IsValidClientName(clientId[..separator]) + && Guid.TryParseExact(clientId[(separator + 1)..], "N", out _); + } + + internal static bool IsValidClientName(string name) + { + if (name.Length is 0 or > 64) + return false; + + foreach (var character in name) + { + if (!char.IsAsciiLetterOrDigit(character) && character is not ('.' or '_' or '-')) + return false; + } + + return true; + } + internal void Validate() { ArgumentNullException.ThrowIfNull(HttpPolicy); AhtolaRemoteTransportSecurity.Validate( RemoteUri, - AuthToken, + AuthTokenProvider is null ? AuthToken : AhtolaRemoteClient.AuthTokenProviderPlaceholder, remoteEncryptionConfigured: RemoteEncryption is not null); + if (!Enum.IsDefined(AutomaticSyncMode)) + { + throw new ArgumentOutOfRangeException( + nameof(AutomaticSyncMode), + AutomaticSyncMode, + "Unknown automatic synchronization mode."); + } + if (ClientName is not null && !IsValidClientName(ClientName)) + { + throw new ArgumentException( + "Sync client names must be 1 to 64 ASCII letters, digits, '.', '_' or '-'.", + nameof(ClientName)); + } AhtolaRemoteTransportSecurity.ValidateRedirectContract( HttpPolicy.MessageHandler is null || HttpPolicy.MessageHandlerDisablesAutomaticRedirects, remoteEncryptionConfigured: RemoteEncryption is not null); @@ -233,6 +314,9 @@ internal AhtolaReplicaOptions CloneForConnection() PushOperationsThreshold = PushOperationsThreshold, PullBytesThreshold = PullBytesThreshold, SyncInterval = SyncInterval, + AutomaticSyncMode = AutomaticSyncMode, + ClientName = ClientName, + AuthTokenProvider = AuthTokenProvider, HttpPolicy = HttpPolicy, }; } @@ -260,6 +344,9 @@ internal AhtolaReplicaOptions WithoutLongPoll() PushOperationsThreshold = PushOperationsThreshold, PullBytesThreshold = PullBytesThreshold, SyncInterval = SyncInterval, + AutomaticSyncMode = AutomaticSyncMode, + ClientName = ClientName, + AuthTokenProvider = AuthTokenProvider, HttpPolicy = HttpPolicy, }; clone._applicationHttpScope = _applicationHttpScope; @@ -351,6 +438,46 @@ public virtual Task SyncAsync( "This embedded replica provider does not support result-bearing synchronization."); } + /// + /// Pulls and applies remote changes without pushing local changes. + /// + public virtual Task PullAsync( + AhtolaSyncOptions options, + CancellationToken cancellationToken) + { + throw new NotSupportedException( + "This embedded replica provider does not support pull-only synchronization."); + } + + /// + /// Pushes local changes without pulling remote changes. + /// + public virtual Task PushAsync( + AhtolaSyncOptions options, + CancellationToken cancellationToken) + { + throw new NotSupportedException( + "This embedded replica provider does not support push-only synchronization."); + } + + /// + /// Checkpoints the local replica's write-ahead log into its main database file. + /// + public virtual Task CheckpointAsync(CancellationToken cancellationToken) + { + throw new NotSupportedException( + "This embedded replica provider does not support explicit checkpoints."); + } + + /// + /// Returns a snapshot of the replica's synchronization statistics. + /// + public virtual Task GetSyncStatisticsAsync(CancellationToken cancellationToken) + { + throw new NotSupportedException( + "This embedded replica provider does not report synchronization statistics."); + } + internal virtual void EnsureCanClose() { } diff --git a/src/Ahtola.Data/AhtolaSyncOptions.cs b/src/Ahtola.Data/AhtolaSyncOptions.cs index f434dfb8..7f8e4a25 100644 --- a/src/Ahtola.Data/AhtolaSyncOptions.cs +++ b/src/Ahtola.Data/AhtolaSyncOptions.cs @@ -155,6 +155,32 @@ public AhtolaRemoteEncryptionOptions(string base64Key, AhtolaRemoteEncryptionCip /// public AhtolaRemoteEncryptionCipher Cipher { get; } + /// + /// Parses a Remote Encryption Cipher connection-string value using Turso's names + /// (aes256gcm, aes128gcm, chacha20poly1305, aegis128l, + /// aegis128x2, aegis128x4, aegis256, aegis256x2, + /// aegis256x4); - and _ separators are also accepted. + /// + internal static AhtolaRemoteEncryptionCipher ParseCipher(string value) + { + ArgumentException.ThrowIfNullOrWhiteSpace(value); + return value.Trim().Replace("-", string.Empty, StringComparison.Ordinal) + .Replace("_", string.Empty, StringComparison.Ordinal) + .ToLowerInvariant() switch + { + "aes256gcm" => AhtolaRemoteEncryptionCipher.Aes256Gcm, + "aes128gcm" => AhtolaRemoteEncryptionCipher.Aes128Gcm, + "chacha20poly1305" => AhtolaRemoteEncryptionCipher.ChaCha20Poly1305, + "aegis128l" => AhtolaRemoteEncryptionCipher.Aegis128L, + "aegis128x2" => AhtolaRemoteEncryptionCipher.Aegis128X2, + "aegis128x4" => AhtolaRemoteEncryptionCipher.Aegis128X4, + "aegis256" => AhtolaRemoteEncryptionCipher.Aegis256, + "aegis256x2" => AhtolaRemoteEncryptionCipher.Aegis256X2, + "aegis256x4" => AhtolaRemoteEncryptionCipher.Aegis256X4, + _ => throw new InvalidOperationException($"Unknown remote encryption cipher: {value}"), + }; + } + /// /// SQLite reserved bytes per page for this cipher: the 16-byte tag plus the /// cipher's nonce. Kept in agreement with the storage layer's own table by @@ -327,8 +353,22 @@ public enum AhtolaSyncOutcome } /// -/// Contains a snapshot of native sync-engine statistics. +/// Contains a snapshot of embedded-replica synchronization statistics. /// +/// +/// In an operation result: the number of change operations the operation pushed and/or applied. +/// From : the number of local changes still +/// waiting to be pushed. +/// +/// Current size in bytes of the replica's main write-ahead log. +/// +/// Current size in bytes of the replica's pending checkpoint-recovery (revert) WAL, or 0. +/// +/// When the last successful pull completed. +/// When the last successful push completed. +/// Bytes sent in pull-updates requests. +/// Bytes received in pull-updates responses. +/// The opaque server revision the replica is synchronized to. public sealed record AhtolaSyncStatistics( long CdcOperations, long MainWalSize, diff --git a/src/Ahtola.Data/Hrana/AhtolaHranaWebSocketTransport.cs b/src/Ahtola.Data/Hrana/AhtolaHranaWebSocketTransport.cs index 72dcf3d8..4e7c30ef 100644 --- a/src/Ahtola.Data/Hrana/AhtolaHranaWebSocketTransport.cs +++ b/src/Ahtola.Data/Hrana/AhtolaHranaWebSocketTransport.cs @@ -22,6 +22,7 @@ internal sealed class AhtolaHranaWebSocketTransport : IAsyncDisposable, IDisposa { private readonly Uri _endpoint; private readonly string? _authToken; + private readonly Func>? _authTokenProvider; private readonly IAhtolaWebSocketConnector _connector; /// @@ -52,13 +53,15 @@ public AhtolaHranaWebSocketTransport( Uri endpoint, string? authToken, AhtolaHranaWebSocketOptions options, - IAhtolaWebSocketConnector? connector = null) + IAhtolaWebSocketConnector? connector = null, + Func>? authTokenProvider = null) { ArgumentNullException.ThrowIfNull(endpoint); ArgumentNullException.ThrowIfNull(options); _endpoint = endpoint; _authToken = string.IsNullOrWhiteSpace(authToken) ? null : authToken; + _authTokenProvider = authTokenProvider; Options = options.Validate(); _connector = connector ?? AhtolaClientWebSocketConnector.Instance; } @@ -877,8 +880,13 @@ private async Task ConnectWithRetriesAsync(Cance var generation = Interlocked.Increment(ref _generation); try { + // A provider is consulted on every (re)connect, so the hello of each new physical + // connection carries the current token. + var authToken = await AhtolaRemoteClient + .ResolveAuthTokenAsync(_authToken, _authTokenProvider, cancellationToken) + .ConfigureAwait(false); return await AhtolaHranaWebSocketConnection - .ConnectAsync(_endpoint, _authToken, _connector, Options, generation, cancellationToken) + .ConnectAsync(_endpoint, authToken, _connector, Options, generation, cancellationToken) .ConfigureAwait(false); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) diff --git a/src/Ahtola.Data/Hrana/AhtolaRemoteClient.WebSocket.cs b/src/Ahtola.Data/Hrana/AhtolaRemoteClient.WebSocket.cs index 4b3ab4a4..26da524d 100644 --- a/src/Ahtola.Data/Hrana/AhtolaRemoteClient.WebSocket.cs +++ b/src/Ahtola.Data/Hrana/AhtolaRemoteClient.WebSocket.cs @@ -36,7 +36,8 @@ internal AhtolaRemoteClient( string? authToken, AhtolaHranaWebSocketOptions webSocketOptions, AhtolaRemoteEncryptionOptions? remoteEncryption = null, - IAhtolaWebSocketConnector? connector = null) + IAhtolaWebSocketConnector? connector = null, + Func>? authTokenProvider = null) { ArgumentNullException.ThrowIfNull(webSocketEndpoint); ArgumentNullException.ThrowIfNull(webSocketOptions); @@ -51,11 +52,15 @@ internal AhtolaRemoteClient( + $"the {EncryptionKeyHeaderName} header on every pipeline request."); } - AhtolaRemoteTransportSecurity.Validate(webSocketEndpoint, authToken, remoteEncryptionConfigured: false); + _authToken = string.IsNullOrWhiteSpace(authToken) ? null : authToken; + _authTokenProvider = authTokenProvider; + AhtolaRemoteTransportSecurity.Validate( + webSocketEndpoint, + CredentialValidationToken, + remoteEncryptionConfigured: false); _httpClient = null; _disposeHttpClient = false; - _authToken = string.IsNullOrWhiteSpace(authToken) ? null : authToken; _remoteEncryptionKey = null; _pipelineUri = webSocketEndpoint; _cursorUri = webSocketEndpoint; @@ -65,7 +70,8 @@ internal AhtolaRemoteClient( webSocketEndpoint, _authToken, webSocketOptions, - connector); + connector, + authTokenProvider); } /// True when this client speaks Hrana over a persistent WebSocket. diff --git a/src/Ahtola.Data/ManagedReplicaBootstrapper.cs b/src/Ahtola.Data/ManagedReplicaBootstrapper.cs index eba4b34d..bd79b5b0 100644 --- a/src/Ahtola.Data/ManagedReplicaBootstrapper.cs +++ b/src/Ahtola.Data/ManagedReplicaBootstrapper.cs @@ -177,6 +177,7 @@ await WriteMetadataAsync( download.Protocol, tableMap, cancellationToken, + clientId: options.CreateClientId(), remoteBaseSha256: remoteBaseSha256).ConfigureAwait(false); ManagedReplicaFaultInjection.Hit(ManagedReplicaDurableBoundary.BootstrapStagedDatabase); @@ -612,7 +613,7 @@ private static (string Revision, string Fingerprint) DecodeCommonFields(string e try { var revision = StrictUtf8.GetString(Convert.FromBase64String(encodedRevision)); - if (revision.Length == 0 || !Guid.TryParseExact(clientId, "N", out _)) + if (revision.Length == 0 || !AhtolaReplicaOptions.IsValidClientId(clientId)) throw new InvalidDataException("Managed embedded replica metadata is invalid."); if (!IsSha256Hex(fingerprint)) throw new InvalidDataException("Managed embedded replica metadata is invalid."); @@ -1074,8 +1075,8 @@ internal static async Task WaitForRemoteChangesAsyn using var scope = options.EnterApplicationHttpScope(); using var client = options.HttpPolicy.CreateHttpClient(options.RemoteEncryption is not null); client.Timeout = Timeout.InfiniteTimeSpan; - var token = string.IsNullOrWhiteSpace(options.AuthToken) ? null : options.AuthToken; var effectiveToken = timeout?.Token ?? cancellationToken; + var token = await options.ResolveAuthTokenAsync(effectiveToken).ConfigureAwait(false); using var response = await AhtolaRemoteTransportSecurity .SendAsync( client, @@ -1358,7 +1359,7 @@ private static async Task ApplyStagedNoOpAsync( syncOptions.Progress?.Report(new AhtolaSyncProgress(AhtolaSyncProgressStage.Completed)); return new ManagedReplicaPullOutcome( new AhtolaSyncResult(AhtolaSyncOutcome.UpToDate, - new AhtolaSyncStatistics(0, 0, 0, DateTimeOffset.UtcNow, null, staged.RequestPayloadLength, staged.ResponseBytesRead, metadata.Revision)), + new AhtolaSyncStatistics(0, GetMainWalSize(options.Path), GetRevertWalSize(options.Path), DateTimeOffset.UtcNow, null, staged.RequestPayloadLength, staged.ResponseBytesRead, metadata.Revision)), ReplayedLocalChangeCount: 0); } @@ -1438,7 +1439,7 @@ await ApplyIncrementalPagesAsync( syncOptions.Progress?.Report(new AhtolaSyncProgress(AhtolaSyncProgressStage.Completed)); return new ManagedReplicaPullOutcome( new AhtolaSyncResult(AhtolaSyncOutcome.RemoteChangesApplied, - new AhtolaSyncStatistics(0, 0, 0, DateTimeOffset.UtcNow, null, staged.RequestPayloadLength, staged.ResponseBytesRead, header.Revision)), + new AhtolaSyncStatistics(0, GetMainWalSize(options.Path), GetRevertWalSize(options.Path), DateTimeOffset.UtcNow, null, staged.RequestPayloadLength, staged.ResponseBytesRead, header.Revision)), ReplayedLocalChangeCount: 0); } @@ -1753,7 +1754,7 @@ private static bool HaveSamePendingLocalChangeSequence( // below. syncOptions.Progress?.Report(new AhtolaSyncProgress(AhtolaSyncProgressStage.Completed)); return (AhtolaSyncOutcome.UpToDate, - new AhtolaSyncStatistics(0, 0, 0, DateTimeOffset.UtcNow, null, networkSentBytes, networkReceivedBytes, metadata.Revision), + new AhtolaSyncStatistics(0, GetMainWalSize(options.Path), GetRevertWalSize(options.Path), DateTimeOffset.UtcNow, null, networkSentBytes, networkReceivedBytes, metadata.Revision), 0); } if (pendingLocalChanges.Count != 0 || quarantineActive) @@ -1952,7 +1953,7 @@ await WriteMetadataAsync( syncOptions.Progress?.Report(new AhtolaSyncProgress(AhtolaSyncProgressStage.Completed)); return (AhtolaSyncOutcome.RemoteChangesApplied, - new AhtolaSyncStatistics(operationCount, 0, 0, DateTimeOffset.UtcNow, null, networkSentBytes, networkReceivedBytes, header.Revision), + new AhtolaSyncStatistics(operationCount, GetMainWalSize(options.Path), GetRevertWalSize(options.Path), DateTimeOffset.UtcNow, null, networkSentBytes, networkReceivedBytes, header.Revision), 0); } @@ -2114,8 +2115,8 @@ private static (AhtolaSyncOutcome Outcome, AhtolaSyncStatistics Statistics, int AhtolaSyncOutcome.RemoteChangesApplied, new AhtolaSyncStatistics( operationCount, - 0, - 0, + GetMainWalSize(options.Path), + GetRevertWalSize(options.Path), DateTimeOffset.UtcNow, null, networkSentBytes, @@ -2848,7 +2849,6 @@ private static async Task DownloadDatabaseAsync( using var scope = options.EnterApplicationHttpScope(); using var client = options.HttpPolicy.CreateHttpClient(options.RemoteEncryption is not null); client.Timeout = Timeout.InfiniteTimeSpan; - var authToken = string.IsNullOrWhiteSpace(options.AuthToken) ? null : options.AuthToken; var querySelector = GetBootstrapQuerySelector(options.PartialBootstrap); // Query bootstrap is never chunked. The server -- not the client -- decides which pages the @@ -2885,7 +2885,6 @@ private static async Task DownloadDatabaseAsync( var header = await PullBootstrapChunkAsync( client, options, - authToken, staging, CreateBootstrapPullRequest( serverRevision: null, @@ -2914,7 +2913,6 @@ private static async Task DownloadDatabaseAsync( _ = await PullBootstrapChunkAsync( client, options, - authToken, staging, CreateBootstrapPullRequest( header.Revision, @@ -2975,7 +2973,6 @@ private static async Task DownloadDatabaseAsync( private static async Task PullBootstrapChunkAsync( HttpClient client, AhtolaReplicaOptions options, - string? authToken, FileStream staging, byte[] requestPayload, PullHeader? expectedHeader, @@ -2987,6 +2984,7 @@ private static async Task PullBootstrapChunkAsync( { using var timeout = CreateTimeout(options.HttpPolicy.RequestTimeout, cancellationToken); var effectiveCancellationToken = timeout?.Token ?? cancellationToken; + var authToken = await options.ResolveAuthTokenAsync(effectiveCancellationToken).ConfigureAwait(false); using var response = await AhtolaRemoteTransportSecurity .SendAsync( client, @@ -3158,7 +3156,7 @@ internal static async Task FetchPagesAsync( using var scope = options.EnterApplicationHttpScope(); using var client = options.HttpPolicy.CreateHttpClient(options.RemoteEncryption is not null); client.Timeout = Timeout.InfiniteTimeSpan; - var authToken = string.IsNullOrWhiteSpace(options.AuthToken) ? null : options.AuthToken; + var authToken = await options.ResolveAuthTokenAsync(effectiveCancellationToken).ConfigureAwait(false); using var response = await AhtolaRemoteTransportSecurity .SendAsync( client, @@ -4264,6 +4262,95 @@ private static void CheckFileDivergence(string databasePath, ManagedReplicaMetad throw new NotSupportedException("Managed embedded replica local divergence was detected; incremental pull cannot replace local changes."); } + /// Current size in bytes of the replica's main write-ahead log, or 0 when absent. + internal static long GetMainWalSize(string databasePath) => GetFileLength(databasePath + "-wal"); + + /// + /// Current size in bytes of the replica's revert (checkpoint-recovery) WAL sidecar, or 0 when + /// no recovery bundle is pending. This is Ahtola's counterpart of Turso's revert WAL. + /// + internal static long GetRevertWalSize(string databasePath) + => GetFileLength(databasePath + ManagedReplicaRevertWal.Suffix); + + private static long GetFileLength(string path) + { + try + { + var info = new FileInfo(path); + return info.Exists ? info.Length : 0; + } + catch (IOException) + { + return 0; + } + catch (UnauthorizedAccessException) + { + return 0; + } + } + + /// + /// Folds the committed frames of the replica's main WAL into its database file and republishes + /// the local fingerprint, exactly like the pending-local-changes checkpoint that an incremental + /// page apply already performs (see ). The change journal + /// is untouched, so every unpushed local change is still pushed by the next push or sync. The + /// caller must hold exclusive managed-replica publication (every host closed). + /// + /// + /// Fails closed, instead of guessing, while a checkpoint-recovery bundle or push outcome is + /// pending (a sync resolves those), while a push conflict is unresolved, and while a partial + /// replica is still lazily materializing pages. + /// + internal static async Task CheckpointLocalWalAsync( + AhtolaReplicaOptions options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + await using var pushLease = await ManagedReplicaPushLock + .AcquireExclusiveAsync(options.Path, cancellationToken) + .ConfigureAwait(false); + await using var applyLease = await ManagedReplicaApplyLock + .AcquireExclusiveAsync(options.Path, cancellationToken) + .ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + + ManagedReplicaConflictState.ThrowIfPending(options.Path); + var metadata = LoadMetadata(options.Path) + ?? throw new NotSupportedException( + "Managed embedded replica checkpoint requires bootstrap metadata."); + ManagedReplicaRevertWal.ValidateSynchronizationReady(options.Path, metadata); + if (File.Exists(options.Path + ManagedReplicaPageMaterializingFileSystem.StateSuffix)) + { + throw new NotSupportedException( + "Managed embedded replica checkpoint is not supported while a partial replica is still " + + "lazily materializing pages; synchronize to complete the image first."); + } + + if (GetMainWalSize(options.Path) <= 32) + return; + + using (var opened = ManagedReplicaEncryption.OpenDatabase(options.Path, options.RemoteEncryption)) + ExecuteNonQuery(opened.Database.Connect(), "PRAGMA wal_checkpoint(TRUNCATE)"); + cancellationToken.ThrowIfCancellationRequested(); + + var fingerprint = ComputeDatabaseFingerprint(options.Path); + if (string.Equals(fingerprint, metadata.DatabaseSha256, StringComparison.Ordinal)) + return; + + var metadataPath = options.Path + MetadataSuffix; + var metadataStagingPath = Path.Combine( + Path.GetDirectoryName(Path.GetFullPath(options.Path))!, + $".{Path.GetFileName(metadataPath)}.checkpoint-{Guid.NewGuid():N}.tmp"); + try + { + WriteMetadata(metadataStagingPath, metadataPath, metadata with { DatabaseSha256 = fingerprint }); + } + finally + { + DeleteIfExists(metadataStagingPath); + } + } + /// Determines whether a bootstrapped managed embedded replica is present at /// using only local filesystem state (the database file, /// metadata sidecar, and optional lazy-page state) — never opens a connection or contacts the remote endpoint. Opening diff --git a/src/Ahtola.Data/ManagedReplicaConnectionHost.cs b/src/Ahtola.Data/ManagedReplicaConnectionHost.cs index a003f5a9..ffa298d0 100644 --- a/src/Ahtola.Data/ManagedReplicaConnectionHost.cs +++ b/src/Ahtola.Data/ManagedReplicaConnectionHost.cs @@ -733,16 +733,39 @@ private async Task SynchronizeAsync( .ConfigureAwait(false); metadata = metadataAfterPush; _metadata = metadata; + var pushedAt = pushedChangeCount == 0 ? (DateTimeOffset?)null : DateTimeOffset.UtcNow; + if (pushedAt is not null) + _syncEntry.RecordCompletedOperation(pulledAt: null, pushedAt, 0, 0); + var result = await WaitAndApplyRemoteChangesAsync(replicaOptions, metadata, syncOptions, cancellationToken) + .ConfigureAwait(false); + return result with + { + Statistics = result.Statistics with + { + CdcOperations = checked(result.Statistics.CdcOperations + pushedChangeCount), + LastPush = pushedAt ?? result.Statistics.LastPush, + }, + }; + } + + /// + /// The pull half of a sync cycle: waits for remote changes and applies them, retrying + /// (bounded, with backoff) when the staged response turns out to be stale relative to local + /// state. The wait itself runs entirely outside any publication gate -- see + /// ManagedReplicaBootstrapper.WaitAndApplyRemoteChangesAsync -- while the apply runs gated, + /// since it mutates the local database file. Mirrors Turso's wait_changes_from_remote -> + /// apply_changes_from_remote split (turso-src/sync/engine/src/database_sync_engine.rs). + /// + private async Task WaitAndApplyRemoteChangesAsync( + AhtolaReplicaOptions replicaOptions, + ManagedReplicaBootstrapper.ManagedReplicaMetadata metadata, + AhtolaSyncOptions syncOptions, + CancellationToken cancellationToken) + { var pendingLocalChanges = _changeJournal.ReadBatch(int.MaxValue).Changes; var acknowledgedLocalChanges = _changeJournal.ReadAcknowledged(metadata.JournalBaseWatermark); - // Wait for remote changes and apply them, retrying (bounded, with backoff) when the staged - // response turns out to be stale relative to local state. The wait itself runs entirely - // outside any publication gate -- see ManagedReplicaBootstrapper.WaitAndApplyRemoteChangesAsync - // -- while the apply runs gated, since it mutates the local database file. Mirrors Turso's - // wait_changes_from_remote -> apply_changes_from_remote split - // (turso-src/sync/engine/src/database_sync_engine.rs). var outcome = await ManagedReplicaBootstrapper.WaitAndApplyRemoteChangesAsync( replicaOptions, metadata, syncOptions, pendingLocalChanges, acknowledgedLocalChanges, (staged, token) => _syncEntry.PublishExclusiveAsync( @@ -756,16 +779,199 @@ private async Task SynchronizeAsync( _metadata = ManagedReplicaBootstrapper.LoadMetadata(replicaOptions.Path); if (result.Outcome == AhtolaSyncOutcome.RemoteChangesApplied && _metadata is { } published) _changeJournal.PruneAcknowledged(published.JournalBaseWatermark); - return result with - { - Statistics = result.Statistics with + _syncEntry.RecordCompletedOperation( + result.Statistics.LastPull ?? DateTimeOffset.UtcNow, + pushedAt: null, + result.Statistics.NetworkSentBytes, + result.Statistics.NetworkReceivedBytes); + return result; + } + + /// + /// Pulls and applies remote changes without pushing local changes, mirroring Turso's + /// TursoSyncDatabase.PullAsync. Pending local changes are preserved exactly as a full + /// sync preserves the ones a capped push left behind (see + /// ManagedReplicaBootstrapper.CheckForUpdatesAsync). A pending checkpoint-recovery + /// bundle or push outcome is not resolved here: it fails closed until a full sync (whose push + /// phase performs that recovery) runs. + /// + public Task PullAsync( + AhtolaSyncOptions options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + if (_metadata is null) + return Task.FromException(CreateMissingMetadataException()); + + return _syncEntry.SynchronizeAsync( + this, + ManagedReplicaSyncKind.PullOnly, + async token => { - CdcOperations = checked(result.Statistics.CdcOperations + pushedChangeCount), - LastPush = pushedChangeCount == 0 ? result.Statistics.LastPush : DateTimeOffset.UtcNow, + var metadata = LoadRequiredMetadata(); + ThrowIfReplicaConflictIsPending(_options.Path); + metadata = PrepareSynchronizationMetadata(metadata); + + // A partial replica still catching up completes its pinned image first, exactly as + // in a full sync; that step can mutate the local file, so it runs gated. + metadata = await _syncEntry.PublishExclusiveAsync( + gateToken => ManagedReplicaBootstrapper.CompletePartialReplicaAsync( + _options, + metadata, + allowTrackedLocalMutations: HasTrackedLocalChanges(metadata), + ReadRetainedMaterializer(), + gateToken), + token) + .ConfigureAwait(false); + _metadata = metadata; + return await WaitAndApplyRemoteChangesAsync(_options, metadata, options, token).ConfigureAwait(false); }, - }; + cancellationToken); + } + + /// + /// Pushes every local change that was pending when the call started, without pulling remote + /// changes, mirroring Turso's TursoSyncDatabase.PushAsync. Batches are still bounded + /// by ; several batches are sent when + /// needed. The result's is the number of + /// changes pushed, and its outcome is always because + /// no remote change is applied. + /// + public Task PushAsync( + AhtolaSyncOptions options, + CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(options); + if (_metadata is null) + return Task.FromException(CreateMissingMetadataException()); + + return _syncEntry.SynchronizeAsync( + this, + ManagedReplicaSyncKind.PushOnly, + token => PushAllPendingAsync(options, token), + cancellationToken); + } + + private async Task PushAllPendingAsync( + AhtolaSyncOptions syncOptions, + CancellationToken cancellationToken) + { + var metadata = LoadRequiredMetadata(); + ThrowIfReplicaConflictIsPending(_options.Path); + metadata = PrepareSynchronizationMetadata(metadata); + + var pending = ManagedReplicaChangeJournal.Open(_options.Path).ReadBatch(int.MaxValue).Changes; + var targetSequence = pending.Count == 0 ? 0 : pending[^1].Sequence; + long pushedChangeCount = 0; + + // Each round pushes at least one change or stops, so the pending count at entry bounds the + // loop; changes committed after this call started are left for the next push or sync. + for (var round = 0; round <= pending.Count; round++) + { + var roundMetadata = metadata; + var push = await _syncEntry.PublishExclusiveAsync( + token => PushLocalChangesAsync( + _options, + roundMetadata, + syncOptions, + ReadRetainedMaterializer(), + token), + cancellationToken) + .ConfigureAwait(false); + metadata = push.Metadata; + _metadata = metadata; + pushedChangeCount = checked(pushedChangeCount + push.ChangeCount); + if (push.ChangeCount == 0) + break; + + var next = _changeJournal.ReadBatch(1).Changes; + if (next.Count == 0 || next[0].Sequence > targetSequence) + break; + } + + var pushedAt = pushedChangeCount == 0 ? (DateTimeOffset?)null : DateTimeOffset.UtcNow; + if (pushedAt is not null) + _syncEntry.RecordCompletedOperation(pulledAt: null, pushedAt, 0, 0); + syncOptions.Progress?.Report(new AhtolaSyncProgress(AhtolaSyncProgressStage.Completed)); + return new AhtolaSyncResult( + AhtolaSyncOutcome.UpToDate, + new AhtolaSyncStatistics( + pushedChangeCount, + ManagedReplicaBootstrapper.GetMainWalSize(_options.Path), + ManagedReplicaBootstrapper.GetRevertWalSize(_options.Path), + LastPull: null, + LastPush: pushedAt, + NetworkSentBytes: 0, + NetworkReceivedBytes: 0, + metadata.Revision)); } + /// + /// Folds the local replica's committed WAL frames into its main database file. See + /// for the exact contract. + /// + public Task CheckpointAsync(CancellationToken cancellationToken) + { + if (_metadata is null) + return Task.FromException(CreateMissingMetadataException()); + + return _syncEntry.PublishExclusiveAsync( + async token => + { + await ManagedReplicaBootstrapper.CheckpointLocalWalAsync(_options, token).ConfigureAwait(false); + _metadata = ManagedReplicaBootstrapper.LoadMetadata(_options.Path); + return true; + }, + cancellationToken); + } + + /// + /// Returns a snapshot of this replica's synchronization statistics: the number of local + /// changes still waiting to be pushed, the current main and revert WAL sizes, the durable + /// server revision, and the in-process history of the last pull, last push and cumulative + /// pull-updates traffic. Pure read: no network access and no local mutation. + /// + public AhtolaSyncStatistics GetSyncStatistics() + { + var metadata = ManagedReplicaBootstrapper.LoadMetadata(_options.Path) + ?? _metadata + ?? throw CreateMissingMetadataException(); + var pending = ManagedReplicaChangeJournal.Open(_options.Path).ReadBatch(int.MaxValue).Changes.Count; + var history = _syncEntry.GetStatistics(); + return new AhtolaSyncStatistics( + pending, + ManagedReplicaBootstrapper.GetMainWalSize(_options.Path), + ManagedReplicaBootstrapper.GetRevertWalSize(_options.Path), + history.LastPull, + history.LastPush, + history.NetworkSentBytes, + history.NetworkReceivedBytes, + metadata.Revision); + } + + private ManagedReplicaBootstrapper.ManagedReplicaMetadata LoadRequiredMetadata() + { + var metadata = ManagedReplicaBootstrapper.LoadMetadata(_options.Path) + ?? throw CreateMissingMetadataException(); + _metadata = metadata; + return metadata; + } + + private ManagedReplicaBootstrapper.ManagedReplicaMetadata PrepareSynchronizationMetadata( + ManagedReplicaBootstrapper.ManagedReplicaMetadata metadata) + { + if (metadata.JournalBaseWatermark < _changeJournal.RetentionBase) + metadata = metadata with { JournalBaseWatermark = _changeJournal.RetentionBase }; + return ManagedReplicaBootstrapper.EnsureLegacyRemoteBaseSnapshot(_options.Path, metadata); + } + + private bool HasTrackedLocalChanges(ManagedReplicaBootstrapper.ManagedReplicaMetadata metadata) + => _changeJournal.ReadBatch(int.MaxValue).Changes.Count != 0 + || _changeJournal.ReadAcknowledged(metadata.JournalBaseWatermark).Count != 0; + + private static NotSupportedException CreateMissingMetadataException() + => new("Managed embedded replica synchronization requires bootstrap metadata."); + /// /// Runs the gated first half of one sync cycle: push local changes to the remote, then (when a /// partial bootstrap is still catching up) complete the partial image. Split out of @@ -1225,7 +1431,8 @@ private async Task PushLocalChangesAsync( replicaOptions.AuthToken, replicaOptions.RemoteEncryption, disposeHttpClient: false, - automaticRedirectsDisabled: true); + automaticRedirectsDisabled: true, + replicaOptions.AuthTokenProvider); var sourcePullGeneration = selectionMetadata.PushState?.SourcePullGeneration ?? 0; if (recoveringUnknownPush) diff --git a/src/Ahtola.Data/ManagedReplicaSyncRegistry.cs b/src/Ahtola.Data/ManagedReplicaSyncRegistry.cs index 6abeba27..076c2830 100644 --- a/src/Ahtola.Data/ManagedReplicaSyncRegistry.cs +++ b/src/Ahtola.Data/ManagedReplicaSyncRegistry.cs @@ -7,6 +7,16 @@ namespace Ahtola; /// Local work holds a shared lease; publication waits for those leases and then closes /// and reopens every registered host while no local work can begin. /// +/// +/// The kinds of network synchronization a managed embedded replica coalesces independently. +/// +internal enum ManagedReplicaSyncKind +{ + PushAndPull = 0, + PullOnly = 1, + PushOnly = 2, +} + internal static class ManagedReplicaSyncRegistry { private static readonly ConcurrentDictionary Entries = @@ -38,7 +48,14 @@ internal sealed class Entry private readonly string _path; private readonly HashSet _hosts = []; private TaskCompletionSource _stateChanged = NewStateChangedSource(); - private Task? _inFlightSync; + // One coalescing slot per ManagedReplicaSyncKind: concurrent callers of the SAME kind share + // one in-flight operation, while a pull-only or push-only request never joins (and so never + // silently reports the result of) an operation of a different kind. + private readonly Task?[] _inFlightSyncs = new Task?[3]; + private DateTimeOffset? _lastPull; + private DateTimeOffset? _lastPush; + private long _networkSentBytes; + private long _networkReceivedBytes; private int _references; private int _activeLocalOperations; private bool _publicationPending; @@ -118,6 +135,13 @@ public Task SynchronizeAsync( ManagedReplicaConnectionHost initiator, Func> stagedOperation, CancellationToken cancellationToken) + => SynchronizeAsync(initiator, ManagedReplicaSyncKind.PushAndPull, stagedOperation, cancellationToken); + + public Task SynchronizeAsync( + ManagedReplicaConnectionHost initiator, + ManagedReplicaSyncKind kind, + Func> stagedOperation, + CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(initiator); ArgumentNullException.ThrowIfNull(stagedOperation); @@ -125,23 +149,55 @@ public Task SynchronizeAsync( Task syncTask; lock (_gate) { - if (_inFlightSync is null) + if (_inFlightSyncs[(int)kind] is not { } inFlight) { var completion = new TaskCompletionSource( TaskCreationOptions.RunContinuationsAsynchronously); - _inFlightSync = completion.Task; + _inFlightSyncs[(int)kind] = completion.Task; syncTask = completion.Task; - _ = CompleteSyncAsync(completion, stagedOperation, cancellationToken); + _ = CompleteSyncAsync(kind, completion, stagedOperation, cancellationToken); } else { - syncTask = _inFlightSync; + syncTask = inFlight; } } return WaitForSynchronizationAsync(syncTask, cancellationToken); } + /// + /// Records the outcome of a completed pull and/or push so + /// can report it to every connection of this replica. + /// + public void RecordCompletedOperation( + DateTimeOffset? pulledAt, + DateTimeOffset? pushedAt, + long networkSentBytes, + long networkReceivedBytes) + { + lock (_gate) + { + if (pulledAt is { } pull) + _lastPull = pull; + if (pushedAt is { } push) + _lastPush = push; + _networkSentBytes = checked(_networkSentBytes + Math.Max(0, networkSentBytes)); + _networkReceivedBytes = checked(_networkReceivedBytes + Math.Max(0, networkReceivedBytes)); + } + } + + /// + /// Returns the in-process synchronization history of this replica: last successful pull + /// and push times and the cumulative pull-updates traffic. + /// + public (DateTimeOffset? LastPull, DateTimeOffset? LastPush, long NetworkSentBytes, long NetworkReceivedBytes) + GetStatistics() + { + lock (_gate) + return (_lastPull, _lastPush, _networkSentBytes, _networkReceivedBytes); + } + private static async Task WaitForSynchronizationAsync( Task syncTask, CancellationToken cancellationToken) @@ -169,6 +225,7 @@ private static async Task WaitForSynchronizationAsync( } private async Task CompleteSyncAsync( + ManagedReplicaSyncKind kind, TaskCompletionSource completion, Func> stagedOperation, CancellationToken cancellationToken) @@ -211,8 +268,8 @@ private async Task CompleteSyncAsync( // condition standing between an already-unreferenced entry and retirement. lock (_gate) { - if (ReferenceEquals(_inFlightSync, completion.Task)) - _inFlightSync = null; + if (ReferenceEquals(_inFlightSyncs[(int)kind], completion.Task)) + _inFlightSyncs[(int)kind] = null; SignalStateChangedNoLock(); RetireIfUnusedNoLock(); } @@ -241,7 +298,8 @@ public Task PublishAsync( /// /// Runs as one exclusive publication unit and returns its - /// result. Unlike this never coalesces with an in-flight sync: + /// result. Unlike + /// this never coalesces with an in-flight sync: /// callers such as explicit conflict resolution choose a specific action, so joining someone /// else's already-running operation and reporting its result would be wrong. /// @@ -382,7 +440,7 @@ private void RetireIfUnusedNoLock() || _activeLocalOperations != 0 || _publicationPending || _publicationActive - || _inFlightSync is not null) + || Array.Exists(_inFlightSyncs, static inFlight => inFlight is not null)) { return; } diff --git a/src/Ahtola.Tests/ManagedReplicaExplicitSyncOperationTests.cs b/src/Ahtola.Tests/ManagedReplicaExplicitSyncOperationTests.cs new file mode 100644 index 00000000..9adb4374 --- /dev/null +++ b/src/Ahtola.Tests/ManagedReplicaExplicitSyncOperationTests.cs @@ -0,0 +1,437 @@ +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http.Headers; +using System.Text.Json; +using AwesomeAssertions; +using Ahtola.Data.Sqlite; + +namespace Ahtola.Tests; + +/// +/// Covers the explicit pull-only, push-only, checkpoint and statistics operations of a managed +/// embedded replica (the counterparts of Turso's TursoSyncDatabase.Pull/Push/Checkpoint/GetStats), +/// the automatic-synchronization status surface, and per-request auth-token providers. +/// +public sealed partial class ManagedEmbeddedReplicaConnectionTests +{ + [Test] + public async Task PullAppliesRemoteChangesWithoutPushingPendingLocalChanges() + { + var path = NewReplicaPath("managed-replica-explicit-pull"); + var image = CreateJournalDatabaseImage(path + ".source"); + var handler = new ExplicitSyncHandler( + [ + CreatePullResponse("revision-42", image, protocol: 2), + CreateLogicalPullResponse("revision-42", body: []), + ]); + try + { + using var connection = AhtolaConnection.CreateReplica(CreateOptions(path, handler)); + connection.Open(); + connection.ExecuteNonQuery("INSERT INTO journal_events VALUES (1);"); + var pullsBefore = handler.PullCallCount; + + var result = await connection.PullAsync(); + + result.Outcome.Should().Be(AhtolaSyncOutcome.UpToDate); + handler.PullCallCount.Should().Be(pullsBefore + 1); + handler.PushCallCount.Should().Be(0, "a pull-only operation never pushes"); + var statistics = connection.GetSyncStatistics(); + statistics.CdcOperations.Should().Be(1, "the unpushed local change is still pending"); + statistics.LastPull.Should().NotBeNull(); + statistics.LastPush.Should().BeNull(); + statistics.Revision.Should().Be("revision-42"); + statistics.NetworkReceivedBytes.Should().BePositive(); + ReadJournalEventValues(connection).Should().Equal(1); + } + finally + { + DeleteReplicaFiles(path); + } + } + + [Test] + public async Task PushSendsEveryPendingBatchWithoutPulling() + { + var path = NewReplicaPath("managed-replica-explicit-push"); + var image = CreateJournalDatabaseImage(path + ".source"); + var handler = new ExplicitSyncHandler( + [ + CreatePullResponse("revision-42", image, protocol: 2), + CreateLogicalPullResponse("revision-42", body: []), + ]); + try + { + using var connection = AhtolaConnection.CreateReplica( + CreateOptions(path, handler, pushOperationsThreshold: 1)); + connection.Open(); + connection.ExecuteNonQuery("INSERT INTO journal_events VALUES (1);"); + connection.ExecuteNonQuery("INSERT INTO journal_events VALUES (2);"); + connection.ExecuteNonQuery("INSERT INTO journal_events VALUES (3);"); + connection.GetSyncStatistics().CdcOperations.Should().Be(3); + var pullsBefore = handler.PullCallCount; + + var result = await connection.PushAsync(); + + result.Outcome.Should().Be(AhtolaSyncOutcome.UpToDate); + result.Statistics.CdcOperations.Should().Be(3); + result.Statistics.LastPush.Should().NotBeNull(); + handler.PushCallCount.Should().Be(3, "the threshold of one change caps every batch"); + handler.PullCallCount.Should().Be(pullsBefore, "a push-only operation never pulls"); + var statistics = connection.GetSyncStatistics(); + statistics.CdcOperations.Should().Be(0); + statistics.LastPush.Should().NotBeNull(); + + // A later push with nothing pending is a no-op, and a normal sync still works. + connection.Push().Statistics.CdcOperations.Should().Be(0); + handler.PushCallCount.Should().Be(3); + (await connection.SyncAsync(new AhtolaSyncOptions(), CancellationToken.None)) + .Outcome.Should().Be(AhtolaSyncOutcome.UpToDate); + } + finally + { + DeleteReplicaFiles(path); + } + } + + [TestCase(1UL)] + [TestCase(2UL)] + public async Task CheckpointFoldsTheLocalWalAndKeepsPendingChangesPushable(ulong protocol) + { + var path = NewReplicaPath("managed-replica-explicit-checkpoint"); + var image = CreateJournalDatabaseImage(path + ".source"); + byte[][] pulls = protocol == 2 + ? + [ + CreatePullResponse("revision-42", image, protocol: 2), + CreateLogicalPullResponse("revision-42", body: []), + ] + : + [ + CreatePullResponse("revision-42", image), + CreatePullResponse("revision-42", [], declaredPages: 1), + ]; + var handler = new ExplicitSyncHandler(pulls); + try + { + using var connection = AhtolaConnection.CreateReplica(CreateOptions(path, handler)); + connection.Open(); + connection.ExecuteNonQuery("INSERT INTO journal_events VALUES (7);"); + + await connection.CheckpointAsync(); + + var statistics = connection.GetSyncStatistics(); + statistics.MainWalSize.Should().BeLessThanOrEqualTo(32, "the WAL was checkpointed and truncated"); + statistics.RevertWalSize.Should().Be(0); + statistics.CdcOperations.Should().Be(1, "a checkpoint never discards unpushed local changes"); + ReadJournalEventValues(connection).Should().Equal(7); + handler.PushCallCount.Should().Be(0); + + var result = await connection.SyncAsync(new AhtolaSyncOptions(), CancellationToken.None); + result.Outcome.Should().Be(AhtolaSyncOutcome.UpToDate); + handler.PushCallCount.Should().Be(1); + connection.GetSyncStatistics().CdcOperations.Should().Be(0); + ReadJournalEventValues(connection).Should().Equal(7); + } + finally + { + DeleteReplicaFiles(path); + } + } + + [Test] + public void ExplicitSyncOperationsRequireAnEmbeddedReplica() + { + using var local = new AhtolaConnection("Data Source=:memory:;Local Provider=Managed"); + local.Open(); + + Assert.Throws(() => local.Pull()); + Assert.Throws(() => local.Push()); + Assert.Throws(() => local.Checkpoint()); + Assert.Throws(() => local.GetSyncStatistics()); + local.AutomaticSyncStatus.Should().Be(AhtolaAutomaticSyncStatus.Stopped); + + using var closed = new AhtolaConnection(); + Assert.Throws(() => closed.Pull()); + } + + [Test] + public async Task AutomaticPullOnlySyncPublishesStatusAndNeverPushes() + { + var path = NewReplicaPath("managed-replica-automatic-pull-only"); + var image = CreateJournalDatabaseImage(path + ".source"); + var handler = new ExplicitSyncHandler( + [ + CreatePullResponse("revision-42", image, protocol: 2), + CreateLogicalPullResponse("revision-42", body: []), + ]); + var baseOptions = CreateOptions(path, handler); + var options = new AhtolaReplicaOptions(path, baseOptions.RemoteUri, baseOptions.AuthToken) + { + SyncInterval = 1, + AutomaticSyncMode = AhtolaAutomaticSyncMode.PullOnly, + HttpPolicy = baseOptions.HttpPolicy, + }; + var observed = new ConcurrentQueue(); + var succeeded = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + try + { + using var connection = AhtolaConnection.CreateReplica(options); + connection.AutomaticSyncStatusChanged += (sender, args) => + { + sender.Should().BeSameAs(connection); + observed.Enqueue(args.Status); + if (args.Status is { State: AhtolaAutomaticSyncState.Waiting, LastSuccess: not null }) + succeeded.TrySetResult(args.Status); + }; + connection.Open(); + connection.AutomaticSyncStatus.State.Should().BeOneOf( + AhtolaAutomaticSyncState.Waiting, + AhtolaAutomaticSyncState.Running); + connection.ExecuteNonQuery("INSERT INTO journal_events VALUES (1);"); + + var success = await succeeded.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + success.LastPullAppliedChanges.Should().BeFalse(); + success.LastException.Should().BeNull(); + success.NextAttempt.Should().NotBeNull(); + observed.Select(status => status.State).Should().Contain(AhtolaAutomaticSyncState.Running); + handler.PushCallCount.Should().Be(0, "pull-only automatic sync never pushes"); + connection.GetSyncStatistics().CdcOperations.Should().Be(1); + + connection.Close(); + connection.AutomaticSyncStatus.State.Should().Be(AhtolaAutomaticSyncState.Stopped); + connection.AutomaticSyncStatus.LastSuccess.Should().NotBeNull(); + } + finally + { + DeleteReplicaFiles(path); + } + } + + [Test] + public async Task AutomaticSyncStatusExposesAFaultedBackgroundLoopBeforeClose() + { + var path = NewReplicaPath("managed-replica-automatic-faulted"); + var image = CreateJournalDatabaseImage(path + ".source"); + var handler = new ExplicitSyncHandler( + [CreatePullResponse("revision-42", image)], + _ => ReplicaPushHandler.BatchErrorResponse(5, 2, "conflicting local change", "SQLITE_CONSTRAINT")); + var faulted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var connection = AhtolaConnection.CreateReplica(CreateOptions(path, handler, syncInterval: 1)); + try + { + connection.AutomaticSyncStatusChanged += (_, args) => + { + if (args.Status.State == AhtolaAutomaticSyncState.Faulted) + faulted.TrySetResult(args.Status); + }; + connection.Open(); + connection.ExecuteNonQuery("INSERT INTO journal_events VALUES (10);"); + + var status = await faulted.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + status.LastException.Should().BeOfType(); + connection.AutomaticSyncStatus.State.Should().Be(AhtolaAutomaticSyncState.Faulted); + Assert.Throws(() => connection.Close()); + connection.AutomaticSyncStatus.State.Should().Be(AhtolaAutomaticSyncState.Stopped); + connection.AutomaticSyncStatus.LastException.Should().BeOfType(); + } + finally + { + connection.Dispose(); + DeleteReplicaFiles(path); + } + } + + [Test] + public async Task ReplicaAuthTokenProviderIsConsultedForEverySyncRequest() + { + var path = NewReplicaPath("managed-replica-token-provider"); + var image = CreateJournalDatabaseImage(path + ".source"); + var handler = new ExplicitSyncHandler( + [ + CreatePullResponse("revision-42", image, protocol: 2), + CreateLogicalPullResponse("revision-42", body: []), + ]); + var baseOptions = CreateOptions(path, handler); + var issued = 0; + var options = new AhtolaReplicaOptions(path, baseOptions.RemoteUri, authToken: "static-token") + { + AuthTokenProvider = _ => ValueTask.FromResult( + "rotated-" + Interlocked.Increment(ref issued).ToString(System.Globalization.CultureInfo.InvariantCulture)), + HttpPolicy = baseOptions.HttpPolicy, + }; + try + { + using var connection = AhtolaConnection.CreateReplica(options); + connection.AuthTokenProvider.Should().BeSameAs(options.AuthTokenProvider); + connection.Open(); + connection.ExecuteNonQuery("INSERT INTO journal_events VALUES (1);"); + await connection.SyncAsync(new AhtolaSyncOptions(), CancellationToken.None); + + var headers = handler.AuthorizationHeaders.ToArray(); + headers.Length.Should().Be(handler.PullCallCount + handler.PushCallCount); + handler.PushCallCount.Should().Be(1); + headers.Should().OnlyHaveUniqueItems(); + headers.Should().AllSatisfy(header => header.Should().StartWith("rotated-")); + headers.Should().NotContain("static-token"); + } + finally + { + DeleteReplicaFiles(path); + } + } + + [Test] + public void ReplicaClientNamePrefixesTheBootstrappedSyncClientIdentifier() + { + var path = NewReplicaPath("managed-replica-client-name"); + var image = CreateJournalDatabaseImage(path + ".source"); + var handler = new ExplicitSyncHandler([CreatePullResponse("revision-42", image)]); + var baseOptions = CreateOptions(path, handler); + var options = new AhtolaReplicaOptions(path, baseOptions.RemoteUri, baseOptions.AuthToken) + { + ClientName = "ahtola.tests_app", + HttpPolicy = baseOptions.HttpPolicy, + }; + try + { + using (var connection = AhtolaConnection.CreateReplica(options)) + connection.Open(); + + var clientId = ManagedReplicaBootstrapper.LoadMetadata(path)!.Value.ClientId; + clientId.Should().StartWith("ahtola.tests_app-"); + Guid.TryParseExact(clientId["ahtola.tests_app-".Length..], "N", out _).Should().BeTrue(); + + // The persisted identifier is accepted when the replica is reopened. + using var reopened = AhtolaConnection.CreateReplica(options); + reopened.Open(); + reopened.GetSyncStatistics().Revision.Should().Be("revision-42"); + } + finally + { + DeleteReplicaFiles(path); + } + + Assert.Throws(() => AhtolaConnection.CreateReplica( + new AhtolaReplicaOptions(path, new Uri("https://example.test"), authToken: null) + { + ClientName = "has space", + })); + } + + [Test] + public async Task SqliteFacadeExposesExplicitReplicaOperationsFromTheConnectionString() + { + var path = NewReplicaPath("sqlite-facade-explicit-sync"); + var image = CreateJournalDatabaseImage(path + ".source"); + var handler = new ExplicitSyncHandler( + [ + CreatePullResponse("revision-42", image, protocol: 2), + CreateLogicalPullResponse("revision-42", body: []), + ]); + var priorFactory = Ahtola.Data.Sqlite.SqliteConnection.RemoteMessageHandlerFactory; + Ahtola.Data.Sqlite.SqliteConnection.RemoteMessageHandlerFactory = () => handler; + var tokens = 0; + try + { + using var replica = new Ahtola.Data.Sqlite.SqliteConnection( + $"Data Source=https://example.test/cluster;Replica Path={path};Local Provider=Managed;Pooling=False;" + + "Sync Client Name=facade;Sync Long Poll Timeout=250;Push Operations Threshold=1;" + + "Automatic Sync Mode=PullOnly;Sync Experimental Features=views,strict") + { + AuthTokenProvider = _ => ValueTask.FromResult( + "facade-" + Interlocked.Increment(ref tokens).ToString(System.Globalization.CultureInfo.InvariantCulture)), + }; + await replica.OpenAsync(); + using (var insert = replica.CreateCommand()) + { + insert.CommandText = "INSERT INTO journal_events VALUES (1); INSERT INTO journal_events VALUES (2);"; + insert.ExecuteNonQuery(); + } + + (await replica.PullAsync()).Outcome.Should().Be(AhtolaSyncOutcome.UpToDate); + handler.PushCallCount.Should().Be(0); + replica.GetSyncStatistics().CdcOperations.Should().Be(2); + replica.Checkpoint(); + (await replica.PushAsync()).Statistics.CdcOperations.Should().Be(2); + handler.PushCallCount.Should().Be(2); + (await replica.GetSyncStatisticsAsync()).CdcOperations.Should().Be(0); + replica.AutomaticSyncStatus.State.Should().Be(AhtolaAutomaticSyncState.Stopped); + ManagedReplicaBootstrapper.LoadMetadata(path)!.Value.ClientId.Should().StartWith("facade-"); + handler.AuthorizationHeaders.Should().AllSatisfy(header => header.Should().StartWith("facade-")); + } + finally + { + Ahtola.Data.Sqlite.SqliteConnection.RemoteMessageHandlerFactory = priorFactory; + DeleteReplicaFiles(path); + } + } + + private static IReadOnlyList ReadJournalEventValues(AhtolaConnection connection) + { + using var command = connection.CreateCommand(); + command.CommandText = "SELECT value FROM journal_events ORDER BY value;"; + using var reader = command.ExecuteReader(); + var values = new List(); + while (reader.Read()) + values.Add(checked((int)reader.GetInt64(0))); + return values; + } + + /// + /// Serves queued pull-updates responses (repeating the last one once the queue is drained, so + /// it must be a no-op), acknowledges every Hrana push batch with the exact step count it + /// carried unless a custom response is supplied, and records every bearer token it receives. + /// + private sealed class ExplicitSyncHandler( + IEnumerable pullResponses, + Func? pushResponse = null) : HttpMessageHandler + { + private readonly Queue _pullResponses = new(pullResponses); + private byte[]? _lastPullResponse; + private int _pullCallCount; + private int _pushCallCount; + + public int PullCallCount => Volatile.Read(ref _pullCallCount); + + public int PushCallCount => Volatile.Read(ref _pushCallCount); + + public ConcurrentQueue AuthorizationHeaders { get; } = new(); + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + AuthorizationHeaders.Enqueue(request.Headers.Authorization?.Parameter); + if (request.RequestUri!.AbsolutePath.EndsWith("/pull-updates", StringComparison.Ordinal)) + { + Interlocked.Increment(ref _pullCallCount); + byte[] payload; + lock (_pullResponses) + { + payload = _pullResponses.Count != 0 + ? _lastPullResponse = _pullResponses.Dequeue() + : _lastPullResponse!; + } + + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(payload), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue("application/protobuf"); + return response; + } + + Interlocked.Increment(ref _pushCallCount); + if (pushResponse is not null) + return pushResponse(request); + + using var document = JsonDocument.Parse( + await request.Content!.ReadAsStringAsync(cancellationToken).ConfigureAwait(false)); + var steps = document.RootElement.GetProperty("requests")[0].GetProperty("batch").GetProperty("steps"); + return ReplicaPushHandler.SuccessfulBatchResponse(steps.GetArrayLength()); + } + } +} diff --git a/src/Ahtola.Tests/RemoteEfOptionsClassificationTests.cs b/src/Ahtola.Tests/RemoteEfOptionsClassificationTests.cs index 148dda0e..69130718 100644 --- a/src/Ahtola.Tests/RemoteEfOptionsClassificationTests.cs +++ b/src/Ahtola.Tests/RemoteEfOptionsClassificationTests.cs @@ -38,6 +38,9 @@ public void UseAhtola_ClassifiesDirectRemoteUrlsAsRemoteHranaServices(string con [TestCase("Data Source=libsql://database.example;Replica Path=replica.db;Auth Token=token")] [TestCase("Data Source=turso://database.example;Replica Path=replica.db;Auth Token=token")] + [TestCase("Data Source=libsql://database.example;Replica Path=replica.db;Auth Token=token;" + + "Sync Client Name=ef;Sync Long Poll Timeout=1000;Bootstrap If Empty=True;" + + "Partial Bootstrap Prefix=4096;Push Operations Threshold=5;Automatic Sync Mode=PullOnly")] public void UseAhtola_ClassifiesEmbeddedReplicaAsReplicaServices(string connectionString) { using var context = CreateContext(connectionString); diff --git a/src/Ahtola.Tests/RemoteReplicationIndexTests.cs b/src/Ahtola.Tests/RemoteReplicationIndexTests.cs index 1ee01ef8..0575cfcc 100644 --- a/src/Ahtola.Tests/RemoteReplicationIndexTests.cs +++ b/src/Ahtola.Tests/RemoteReplicationIndexTests.cs @@ -40,28 +40,39 @@ await client.ExecuteBatchAsync( handler.RequestReplicationIndexes.Should().Equal(null, "42"); } + // turso-src/serverless/PROTOCOL.md (v0.8.1) section 7.2.6: clients MUST NOT interpret the + // replication_index value. An uninterpretable value is therefore ignored -- the request + // succeeds and the previously tracked watermark is kept -- instead of failing the batch. [TestCase("\"not-an-index\"")] + [TestCase("\"-1\"")] [TestCase("1.5")] + [TestCase("-3")] [TestCase("{}")] - public void RemoteBatchRejectsAnInvalidReplicationIndex(string encodedIndex) + [TestCase("[]")] + [TestCase("true")] + public async Task RemoteBatchIgnoresAnUninterpretableReplicationIndex(string encodedIndex) { using var handler = new ReplicationIndexHandler( """ - {"results":[{"type":"ok","response":{"type":"batch","result":{"step_results":[{"cols":[],"rows":[],"affected_row_count":0}],"step_errors":[null],"replication_index":"__INDEX__"}}}]} - """.Replace("\"__INDEX__\"", encodedIndex, StringComparison.Ordinal)); + {"results":[{"type":"ok","response":{"type":"batch","result":{"step_results":[{"cols":[],"rows":[],"affected_row_count":0}],"step_errors":[null],"replication_index":"5"}}}]} + """, + """ + {"results":[{"type":"ok","response":{"type":"batch","result":{"step_results":[{"cols":[],"rows":[],"affected_row_count":0,"replication_index":__INDEX__}],"step_errors":[null],"replication_index":__INDEX__}}}]} + """.Replace("__INDEX__", encodedIndex, StringComparison.Ordinal), + """ + {"results":[{"type":"ok","response":{"type":"batch","result":{"step_results":[{"cols":[],"rows":[],"affected_row_count":0}],"step_errors":[null]}}}]} + """); using var httpClient = new HttpClient(handler); using var client = new AhtolaRemoteClient( httpClient, new Uri("https://example.com"), authToken: null); + var commands = new[] { new AhtolaBatchCommand("SELECT 1") }; - Assert.ThrowsAsync(() => client.ExecuteBatchAsync( - [new AhtolaBatchCommand("SELECT 1")], - commandTimeout: 30, - wantRows: true, - closeAfter: true, - CancellationToken.None))! - .Message.Should().Be("Remote response returned an invalid replication_index."); + for (var request = 0; request < 3; request++) + await client.ExecuteBatchAsync(commands, 30, wantRows: true, closeAfter: true, CancellationToken.None); + + handler.RequestReplicationIndexes.Should().Equal(null, "5", "5"); } [Test] diff --git a/src/Ahtola.Tests/ReplicaConnectionStringOptionsTests.cs b/src/Ahtola.Tests/ReplicaConnectionStringOptionsTests.cs new file mode 100644 index 00000000..07bf6a79 --- /dev/null +++ b/src/Ahtola.Tests/ReplicaConnectionStringOptionsTests.cs @@ -0,0 +1,240 @@ +using System.Net; +using System.Text; +using AwesomeAssertions; + +namespace Ahtola.Tests; + +/// +/// Pins the embedded-replica connection-string keywords Ahtola accepts for parity with Turso's +/// TursoConnectionStringBuilder (turso-src/bindings/dotnet/src/Turso.Data), their +/// mapping onto , and the per-request auth-token provider. +/// +public sealed class ReplicaConnectionStringOptionsTests +{ + private const string AdvancedReplicaConnectionString = + "Data Source=libsql://db.example.test;Auth Token=secret;Replica Path=replica.db;" + + "SyncClientName=my-app;Sync Long Poll Timeout=1500;BootstrapIfEmpty=False;" + + "Partial Bootstrap Prefix=8192;Partial Sync Segment Size=16384;PartialSyncPrefetch=True;" + + "Push Operations Threshold=10;Sync Interval=5;AutomaticSyncMode=PullOnly;" + + "Remote Encryption Cipher=aes256gcm;Remote Encryption Key=c2VjcmV0;" + + "Force Logical MVCC Pull=False;Sync Experimental Features=views, strict"; + + [Test] + public void AhtolaBuilderAcceptsTursoReplicaKeywordsAndAliases() + { + var builder = new AhtolaConnectionStringBuilder(AdvancedReplicaConnectionString); + + builder.SyncClientName.Should().Be("my-app"); + builder.SyncLongPollTimeout.Should().Be(1500); + builder.BootstrapIfEmpty.Should().BeFalse(); + builder.PartialBootstrapPrefix.Should().Be(8192); + builder.PartialBootstrapQuery.Should().BeEmpty(); + builder.PartialSyncSegmentSize.Should().Be(16384); + builder.PartialSyncPrefetch.Should().BeTrue(); + builder.PushOperationsThreshold.Should().Be(10); + builder.PullBytesThreshold.Should().Be(0); + builder.RemoteEncryptionCipher.Should().Be("aes256gcm"); + builder.RemoteEncryptionKey.Should().Be("c2VjcmV0"); + builder.ForceLogicalMvccPull.Should().BeFalse(); + builder.SyncExperimentalFeatures.Should().Be("views, strict"); + builder.AutomaticSyncMode.Should().Be(AhtolaAutomaticSyncMode.PullOnly); + + var defaults = new AhtolaConnectionStringBuilder(); + defaults.BootstrapIfEmpty.Should().BeTrue(); + defaults.AutomaticSyncMode.Should().Be(AhtolaAutomaticSyncMode.PushAndPull); + defaults.PullBytesThreshold = 4096; + defaults.ConnectionString.Should().Be("Pull Bytes Threshold=4096"); + } + + [Test] + public void ConnectionStringKeywordsMapOntoReplicaOptions() + { + var options = AhtolaConnectionOptions.Parse(AdvancedReplicaConnectionString) + .CreateReplicaOptions(messageHandler: null, authTokenProvider: null); + + options.Path.Should().Be("replica.db"); + options.RemoteUri.Should().Be(new Uri("https://db.example.test/")); + options.AuthToken.Should().Be("secret"); + options.ClientName.Should().Be("my-app"); + options.LongPollTimeout.Should().Be(TimeSpan.FromMilliseconds(1500)); + options.BootstrapIfEmpty.Should().BeFalse(); + options.PartialBootstrap.Should().NotBeNull(); + options.PartialBootstrap!.Kind.Should().Be(AhtolaPartialBootstrapKind.Prefix); + options.PartialBootstrap.PrefixLength.Should().Be(8192); + options.PartialBootstrap.SegmentSize.Should().Be(16384); + options.PartialBootstrap.Prefetch.Should().BeTrue(); + options.PushOperationsThreshold.Should().Be(10); + options.PullBytesThreshold.Should().BeNull(); + options.SyncInterval.Should().Be(5); + options.AutomaticSyncMode.Should().Be(AhtolaAutomaticSyncMode.PullOnly); + options.RemoteEncryption.Should().NotBeNull(); + options.RemoteEncryption!.Cipher.Should().Be(AhtolaRemoteEncryptionCipher.Aes256Gcm); + options.RemoteEncryption.Base64Key.Should().Be("c2VjcmV0"); + + var query = AhtolaConnectionOptions.Parse( + "Data Source=https://db.example.test;Replica Path=replica.db;Partial Bootstrap Query=SELECT * FROM t;Pull Bytes Threshold=65536") + .CreateReplicaOptions(null, null); + query.PartialBootstrap!.Kind.Should().Be(AhtolaPartialBootstrapKind.Query); + query.PartialBootstrap.Query.Should().Be("SELECT * FROM t"); + query.PullBytesThreshold.Should().Be(65536); + query.BootstrapIfEmpty.Should().BeTrue(); + query.LongPollTimeout.Should().BeNull(); + query.ClientName.Should().BeNull(); + query.AutomaticSyncMode.Should().Be(AhtolaAutomaticSyncMode.PushAndPull); + } + + [TestCase("aes128gcm", AhtolaRemoteEncryptionCipher.Aes128Gcm)] + [TestCase("AES-256-GCM", AhtolaRemoteEncryptionCipher.Aes256Gcm)] + [TestCase("chacha20poly1305", AhtolaRemoteEncryptionCipher.ChaCha20Poly1305)] + [TestCase("aegis128l", AhtolaRemoteEncryptionCipher.Aegis128L)] + [TestCase("aegis_256x4", AhtolaRemoteEncryptionCipher.Aegis256X4)] + public void RemoteEncryptionCipherNamesFollowTurso(string name, AhtolaRemoteEncryptionCipher expected) + => AhtolaRemoteEncryptionOptions.ParseCipher(name).Should().Be(expected); + + [TestCase("Partial Bootstrap Prefix=4096;Partial Bootstrap Query=SELECT 1", "cannot be combined")] + [TestCase("Partial Sync Segment Size=4096", "require Partial Bootstrap Prefix")] + [TestCase("Partial Sync Prefetch=True", "require Partial Bootstrap Prefix")] + [TestCase("Remote Encryption Cipher=aes256gcm", "must be specified together")] + [TestCase("Remote Encryption Key=c2VjcmV0", "must be specified together")] + [TestCase("Remote Encryption Cipher=rot13;Remote Encryption Key=c2VjcmV0", "Unknown remote encryption cipher")] + public void InvalidReplicaKeywordCombinationsFailClosed(string keywords, string message) + { + var options = AhtolaConnectionOptions.Parse( + "Data Source=https://db.example.test;Replica Path=replica.db;" + keywords); + + Assert.Throws(() => options.CreateReplicaOptions(null, null))! + .Message.Should().Contain(message); + } + + [Test] + public void UnsupportedTursoEscapeHatchesFailClosedOrValidate() + { + var forced = AhtolaConnectionOptions.Parse( + "Data Source=https://db.example.test;Replica Path=replica.db;Force Logical MVCC Pull=True"); + Assert.Throws(() => forced.CreateReplicaOptions(null, null))! + .Message.Should().Contain("Force Logical MVCC Pull"); + + var features = AhtolaConnectionOptions.Parse( + "Data Source=https://db.example.test;Replica Path=replica.db;Sync Experimental Features='views,,x'"); + Assert.Throws(() => features.CreateReplicaOptions(null, null)); + } + + [Test] + public void AdvancedReplicaKeywordsRequireAnEmbeddedReplica() + { + using var local = new AhtolaConnection("Data Source=:memory:;Local Provider=Managed;Sync Client Name=x"); + Assert.Throws(() => local.Open())! + .Message.Should().Be("Advanced sync options require a remote embedded replica connection."); + + using var remote = new AhtolaConnection("Data Source=https://db.example.test;Automatic Sync Mode=PullOnly"); + Assert.Throws(() => remote.Open())! + .Message.Should().Be("Advanced sync options require an embedded replica connection."); + + using var tokenOnLocal = new AhtolaConnection("Data Source=:memory:;Local Provider=Managed") + { + AuthTokenProvider = _ => ValueTask.FromResult("t"), + }; + Assert.Throws(() => tokenOnLocal.Open()); + } + + [Test] + public void SqliteFacadeForwardsReplicaKeywordsToTheAhtolaConnectionString() + { + var facade = new Ahtola.Data.Sqlite.SqliteConnectionStringBuilder(AdvancedReplicaConnectionString); + facade.SyncClientName.Should().Be("my-app"); + facade.PartialSyncSegmentSize.Should().Be(16384); + facade.AutomaticSyncMode.Should().Be(AhtolaAutomaticSyncMode.PullOnly); + facade.Keys.Cast().Should().Contain( + ["Sync Client Name", "Remote Encryption Key", "Force Logical MVCC Pull", "Automatic Sync Mode"]); + facade["Pull Bytes Threshold"].Should().Be(0L); + + var forwarded = new AhtolaConnectionStringBuilder(facade.GetAhtolaConnectionString()); + forwarded.SyncClientName.Should().Be("my-app"); + forwarded.SyncLongPollTimeout.Should().Be(1500); + forwarded.BootstrapIfEmpty.Should().BeFalse(); + forwarded.PartialBootstrapPrefix.Should().Be(8192); + forwarded.PartialSyncSegmentSize.Should().Be(16384); + forwarded.PartialSyncPrefetch.Should().BeTrue(); + forwarded.PushOperationsThreshold.Should().Be(10); + forwarded.RemoteEncryptionCipher.Should().Be("aes256gcm"); + forwarded.RemoteEncryptionKey.Should().Be("c2VjcmV0"); + forwarded.SyncExperimentalFeatures.Should().Be("views, strict"); + forwarded.AutomaticSyncMode.Should().Be(AhtolaAutomaticSyncMode.PullOnly); + forwarded.LocalProvider.Should().Be(AhtolaLocalProvider.Managed); + } + + [Test] + public async Task RemoteClientResolvesTheTokenProviderForEveryRequest() + { + using var handler = new AuthorizationRecordingHandler(); + using var httpClient = new HttpClient(handler); + var issued = 0; + using var client = new AhtolaRemoteClient( + httpClient, + new Uri("https://example.com"), + authToken: "static-token", + authTokenProvider: _ => ValueTask.FromResult( + ++issued == 2 ? " " : "token-" + issued.ToString(System.Globalization.CultureInfo.InvariantCulture))); + var commands = new[] { new AhtolaBatchCommand("SELECT 1") }; + + for (var request = 0; request < 3; request++) + await client.ExecuteBatchAsync(commands, 30, wantRows: true, closeAfter: true, CancellationToken.None); + + handler.Authorizations.Should().Equal("Bearer token-1", null, "Bearer token-3"); + } + + [Test] + public void TokenProviderStillRequiresHttpsForNonLoopbackHosts() + { + using var httpClient = new HttpClient(new AuthorizationRecordingHandler()); + Assert.Throws(() => _ = new AhtolaRemoteClient( + httpClient, + new Uri("http://example.com"), + authToken: null, + authTokenProvider: _ => ValueTask.FromResult("token"))); + + Assert.Throws(() => AhtolaConnection.CreateReplica( + new AhtolaReplicaOptions("replica.db", new Uri("http://example.com"), authToken: null) + { + AuthTokenProvider = _ => ValueTask.FromResult("token"), + })); + } + + [Test] + public async Task WebSocketHelloUsesTheTokenProviderOnEveryConnect() + { + var server = new FakeHranaServer(); + var issued = 0; + using var client = new AhtolaRemoteClient( + new Uri("wss://database.example"), + authToken: "static-token", + AhtolaHranaWebSocketOptions.Default, + remoteEncryption: null, + server, + _ => ValueTask.FromResult( + "ws-token-" + Interlocked.Increment(ref issued).ToString(System.Globalization.CultureInfo.InvariantCulture))); + + _ = await HranaWebSocketTransportTests.ExecuteAsync(client, "SELECT 1"); + + server.ObservedJwt.Should().Be("ws-token-1"); + } + + private sealed class AuthorizationRecordingHandler : HttpMessageHandler + { + public List Authorizations { get; } = []; + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + Authorizations.Add(request.Headers.Authorization?.ToString()); + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent( + """{"results":[{"type":"ok","response":{"type":"batch","result":{"step_results":[{"cols":[],"rows":[],"affected_row_count":0}],"step_errors":[null]}}}]}""", + Encoding.UTF8, + "application/json"), + }); + } + } +} From fb2a094eba4fde0d8fbd22d0a7e452c4a97eafa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 15:39:34 -0400 Subject: [PATCH 06/20] Report page-1 auto_vacuum mode and reject MVCC ALTER COLUMN index rebuilds PRAGMA auto_vacuum (Turso v0.8.1 translate/pragma.rs without the experimental autovacuum feature, which Ahtola does not have): - The read form reports the mode page 1 declares (largest-root-page and incremental-vacuum header fields), so an auto-vacuum database SQLite created reports FULL (1) / INCREMENTAL (2) instead of a hard-coded 0. - The set form resolves the mode like upstream's requested_mode: names in any case, or a numeric literal valued 0..2 (so 00 and 0x0 are NONE). NONE stays accepted; FULL, INCREMENTAL and unrecognized values keep upstream's --experimental-autovacuum diagnostic. ALTER COLUMN in MVCC mode (upstream validate_indexes_can_be_rewritten): a rewrite that changes stored values (becoming generated, toggling VIRTUAL, an affinity change) or may change a VIRTUAL generated value, and that touches an index observing the column or a generated column derived from it, is rejected with 'cannot ALTER COLUMN "x": rebuilding affected indexes is not supported in MVCC mode'. The two alter_column *-mvcc corpus cases now fail only on the CLI "Parse error:" prefix, like the other prefix-only entries; their ledger summaries are updated accordingly. Co-Authored-By: Claude Opus 5.5 --- src/Ahtola.Core/EmbeddedDatabase.cs | 121 ++++++++++++++++-- src/Ahtola.Core/EmbeddedFileStore.cs | 8 ++ .../managed-sqltest-expected-failures.txt | 4 +- .../ManagedAlterTableAlterColumnTests.cs | 47 +++++++ .../ManagedDocumentedBoundaryTests.cs | 6 + .../NativeSqliteChurnInteropTests.cs | 2 + 6 files changed, 177 insertions(+), 11 deletions(-) diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index 0e81c809..08552917 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -4075,6 +4075,13 @@ private void EnforceMaxPageCountForCatalogChange(int additionalPages) throw new EmbeddedSqlException("database or disk is full"); } + /// SQLite's auto-vacuum mode from page 1: 0 NONE, 1 FULL, 2 INCREMENTAL. + internal int GetAutoVacuumMode() + { + lock (_gate) + return _fileStore?.AutoVacuumMode ?? 0; + } + internal uint GetFreelistCount() { lock (_gate) @@ -9069,6 +9076,77 @@ context with /// Decides whether an ALTER COLUMN is legal and computes the rows the program rewrites, the /// dependent definitions its ParseSchema adopts, and the AUTOINCREMENT state it retires. /// + /// + /// Turso's validate_indexes_can_be_rewritten MVCC arm (translate/alter.rs): an ALTER + /// COLUMN whose rewrite changes a stored value (becoming generated, toggling VIRTUAL, or an + /// affinity change) or may change a VIRTUAL generated value must rebuild every index that + /// observes the column or a generated column derived from it, which MVCC mode does not + /// support. + /// + private void ThrowIfMvccAlterColumnRebuildsIndexes( + string columnName, + EmbeddedTable original, + EmbeddedTable replacement, + int columnIndex) + { + if (!IsMvccEnabled || original.Indexes.Count == 0) + return; + + var oldColumn = original.ColumnDefinitions[columnIndex]; + var newColumn = replacement.ColumnDefinitions[columnIndex]; + var oldVirtual = oldColumn.IsGenerated && !oldColumn.GeneratedStored; + var newVirtual = newColumn.IsGenerated && !newColumn.GeneratedStored; + var rewritesPhysicalLayout = (!oldColumn.IsGenerated && newColumn.IsGenerated) + || oldVirtual != newVirtual + || original.GetColumnAffinity(oldColumn) != replacement.GetColumnAffinity(newColumn); + if (!rewritesPhysicalLayout && !oldVirtual && !newVirtual) + return; + + var affected = new HashSet(StringComparer.OrdinalIgnoreCase); + CollectColumnsAffectedByUpdate(original, columnIndex, affected); + CollectColumnsAffectedByUpdate(replacement, columnIndex, affected); + if (!original.Indexes.Any(index => IndexObservesAnyColumn(index, affected))) + return; + + throw new EmbeddedSqlException( + $"cannot ALTER COLUMN \"{columnName}\": rebuilding affected indexes is not supported in MVCC mode"); + + static void CollectColumnsAffectedByUpdate(EmbeddedTable table, int columnIndex, HashSet names) + { + names.Add(table.Columns[columnIndex]); + for (var changed = true; changed;) + { + changed = false; + foreach (var column in table.ColumnDefinitions) + { + if (column.GenerationExpression is not { } expression || names.Contains(column.Name)) + continue; + + var references = new HashSet(StringComparer.OrdinalIgnoreCase); + EmbeddedTable.CollectColumnReferences(expression, references); + if (references.Overlaps(names)) + changed |= names.Add(column.Name); + } + } + } + + static bool IndexObservesAnyColumn(EmbeddedIndex index, HashSet names) + { + var references = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var column in index.Columns) + { + if (column.Expression is { } expression) + EmbeddedTable.CollectColumnReferences(expression, references); + else + references.Add(column.Name); + } + + if (index.Where is { } where) + EmbeddedTable.CollectColumnReferences(where, references); + return references.Overlaps(names); + } + } + private (string CurrentName, int ColumnIndex, CompiledAlterTablePlan Plan) PlanAlterTableAlterColumn( AlterTableAlterColumnStatement statement, SchemaCatalog catalog, @@ -9095,6 +9173,7 @@ context with statement.ColumnName, statement.Column, context.CancellationToken); + ThrowIfMvccAlterColumnRebuildsIndexes(statement.ColumnName, table, replacement, columnIndex); var candidateTables = new Dictionary( catalog.Tables, StringComparer.OrdinalIgnoreCase) @@ -65269,16 +65348,20 @@ private ExecutionResult ExecutePragmaAutoVacuum(PragmaAutoVacuumStatement statem { ValidatePragmaSchema(statement.Schema); if (statement.Value is null) - return new ExecutionResult(["auto_vacuum"], [[SqlValue.Integer(0)]], 0); + { + // Turso reports the mode the pager read from page 1 (largest-root-page and + // incremental-vacuum header fields), so an auto-vacuum database SQLite created + // reports FULL (1) or INCREMENTAL (2) even though this engine never enables it. + var database = ResolvePragmaDatabase(statement.Schema); + return new ExecutionResult(["auto_vacuum"], [[SqlValue.Integer(database.GetAutoVacuumMode())]], 0); + } - // Auto-vacuum is always off: Ahtola has no `--experimental-autovacuum` flag/engine - // support to turn it on. SQLite spells the mode either by name or by number (0/NONE, - // 1/FULL, 2/INCREMENTAL); requesting NONE only restates the state the database is - // already in, so it is always accepted, while requesting FULL/INCREMENTAL (or any - // unrecognized value) fails the same way it would if the flag existed but was unset. - var isNone = string.Equals(statement.Value, "none", StringComparison.OrdinalIgnoreCase) - || string.Equals(statement.Value, "0", StringComparison.Ordinal); - if (!isNone) + // Turso v0.8.1 translate/pragma.rs without the experimental autovacuum feature, which + // Ahtola does not have: SQLite spells the mode by name or by number (0/NONE, 1/FULL, + // 2/INCREMENTAL) and treats both the same; NONE only restates the state the database + // is already in, so it is accepted (and, like SQLite once page 1 exists, has no effect), + // while FULL, INCREMENTAL and any unrecognized value fail with the flag diagnostic. + if (ParseAutoVacuumMode(statement.Value) != 0) { throw new EmbeddedSqlException( "Autovacuum is not enabled. Use --experimental-autovacuum flag to enable it."); @@ -65287,6 +65370,26 @@ private ExecutionResult ExecutePragmaAutoVacuum(PragmaAutoVacuumStatement statem return ExecutionResult.Empty; } + /// + /// Upstream's requested_mode: a mode name (any case), or a numeric literal whose value + /// is 0, 1 or 2 (so 00 and 0x0 are NONE). Anything else is . + /// + private static int? ParseAutoVacuumMode(string value) + { + if (value.Equals("none", StringComparison.OrdinalIgnoreCase)) + return 0; + if (value.Equals("full", StringComparison.OrdinalIgnoreCase)) + return 1; + if (value.Equals("incremental", StringComparison.OrdinalIgnoreCase)) + return 2; + + long number; + var parsed = value.StartsWith("0x", StringComparison.OrdinalIgnoreCase) + ? long.TryParse(value.AsSpan(2), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out number) + : long.TryParse(value, NumberStyles.None, CultureInfo.InvariantCulture, out number); + return parsed && number is >= 0 and <= 2 ? (int)number : null; + } + private ExecutionResult ExecutePragmaDataSyncRetry(PragmaDataSyncRetryStatement statement) { ValidatePragmaSchema(statement.Schema); diff --git a/src/Ahtola.Core/EmbeddedFileStore.cs b/src/Ahtola.Core/EmbeddedFileStore.cs index dc6c9b4e..0b099eb9 100644 --- a/src/Ahtola.Core/EmbeddedFileStore.cs +++ b/src/Ahtola.Core/EmbeddedFileStore.cs @@ -336,6 +336,14 @@ private byte[] ReadPageForLoad(uint pageNumber, object? pinnedTransaction = null private uint GetCommittedPageCountForLoad(object? pinnedTransaction = null) => _pager.CommittedPageCount; + /// + /// The auto-vacuum mode page 1 declares, as Turso's pager derives it at open: a nonzero + /// largest-root-page field means auto-vacuum, and the incremental-vacuum field picks + /// INCREMENTAL (2) over FULL (1). + /// + internal int AutoVacuumMode + => _header.LargestRootBtreePage == 0 ? 0 : _header.IncrementalVacuumEnabled != 0 ? 2 : 1; + private SqliteOverflowChainReader CreateOverflowReaderForLoad(object? pinnedTransaction = null) => new(_pager, _header); diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index 3d47aa06..3125b828 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -232,8 +232,8 @@ scalar-functions.sqltest::min-float-equals-int | database variant 1/2 (:default: scalar-functions.sqltest::min-int-equals-float | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '1.0' but was '1' simple-count-optimization.sqltest::simple-count-unusable-partial-index-column-correctness | database variant 1/1 (:memory:): expected success but got error: no query solution sql-error-message-prefix.sqltest::database-full-error-has-no-prefix | database variant 1/1 (:memory:): expected an error but the statement succeeded -turso/alter_column.sqltest::alter-column-rejects-index-only-rewrite-mvcc | database variant 1/1 (:memory:): expected an error but the statement succeeded -turso/alter_column.sqltest::alter-column-rejects-index-rewrite-mvcc | database variant 1/1 (:memory:): expected an error but the statement succeeded +turso/alter_column.sqltest::alter-column-rejects-index-only-rewrite-mvcc | error message lacks the CLI "Parse error:" prefix the Turso corpus pins (the MVCC index-rebuild rejection text is otherwise identical) +turso/alter_column.sqltest::alter-column-rejects-index-rewrite-mvcc | error message lacks the CLI "Parse error:" prefix the Turso corpus pins (the MVCC index-rebuild rejection text is otherwise identical) turso/alter_column.sqltest::alter-column-rewrites-expression-index-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' turso/alter_column.sqltest::alter-column-rewrites-generated-index-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' turso/alter_column.sqltest::alter-column-rewrites-indexed-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' diff --git a/src/Ahtola.Tests/ManagedAlterTableAlterColumnTests.cs b/src/Ahtola.Tests/ManagedAlterTableAlterColumnTests.cs index 39b2ff6c..7928e96c 100644 --- a/src/Ahtola.Tests/ManagedAlterTableAlterColumnTests.cs +++ b/src/Ahtola.Tests/ManagedAlterTableAlterColumnTests.cs @@ -30,6 +30,53 @@ public void AlterColumnReplacingAnAutoincrementRowidAliasClearsItsSequence() .Equal("1\u001fx", "2\u001fy", "\u001fz"); } + // turso/alter_column.sqltest::alter-column-rejects-index-rewrite-mvcc and + // ::alter-column-rejects-index-only-rewrite-mvcc: MVCC mode cannot rebuild the indexes + // an ALTER COLUMN rewrite invalidates, so upstream rejects the statement up front. + [TestCase( + "CREATE TABLE t(x NUMERIC); CREATE INDEX idx_x ON t(x); INSERT INTO t VALUES(10),(2),(30);", + "ALTER TABLE t ALTER COLUMN x TO y TEXT;", + "x")] + [TestCase( + "CREATE TABLE t(a TEXT, g TEXT GENERATED ALWAYS AS ('old:' || a) VIRTUAL); CREATE INDEX idx_g ON t(g); INSERT INTO t(a) VALUES('a'),('b');", + "ALTER TABLE t ALTER COLUMN g TO h TEXT GENERATED ALWAYS AS ('new:' || a) VIRTUAL;", + "g")] + [TestCase( + "CREATE TABLE t(a NUMERIC, g GENERATED ALWAYS AS (a + 1) VIRTUAL); CREATE INDEX idx_g ON t(g); INSERT INTO t(a) VALUES(1);", + "ALTER TABLE t ALTER COLUMN a TO a TEXT;", + "a")] + public void MvccAlterColumnThatMustRebuildIndexesIsRejected(string setup, string alter, string column) + { + using var connection = new ManagedSqliteConnection("Data Source=:memory:;Local Provider=Managed"); + connection.Open(); + Execute(connection, "PRAGMA journal_mode = 'mvcc';"); + Execute(connection, setup); + var schemaBefore = Scalar(connection, "SELECT group_concat(sql, ';') FROM sqlite_schema;"); + + var act = () => Execute(connection, alter); + + act.Should().Throw().WithMessage( + $"*cannot ALTER COLUMN \"{column}\": rebuilding affected indexes is not supported in MVCC mode*"); + Scalar(connection, "SELECT group_concat(sql, ';') FROM sqlite_schema;").Should().Be(schemaBefore); + } + + [Test] + public void MvccAlterColumnThatLeavesIndexesValidStillSucceeds() + { + using var connection = new ManagedSqliteConnection("Data Source=:memory:;Local Provider=Managed"); + connection.Open(); + Execute(connection, "PRAGMA journal_mode = 'mvcc';"); + Execute(connection, """ + CREATE TABLE t(x TEXT, y NUMERIC); + CREATE INDEX idx_x ON t(x); + INSERT INTO t VALUES('a', 1); + ALTER TABLE t ALTER COLUMN x TO x2 TEXT; + ALTER TABLE t ALTER COLUMN y TO y2 TEXT; + """); + + ReadRows(connection, "SELECT x2, y2 FROM t;").Should().Equal("a\u001f1"); + } + private static void Execute(ManagedSqliteConnection connection, string sql) { using var command = connection.CreateCommand(); diff --git a/src/Ahtola.Tests/ManagedDocumentedBoundaryTests.cs b/src/Ahtola.Tests/ManagedDocumentedBoundaryTests.cs index 96786a5a..26476593 100644 --- a/src/Ahtola.Tests/ManagedDocumentedBoundaryTests.cs +++ b/src/Ahtola.Tests/ManagedDocumentedBoundaryTests.cs @@ -45,6 +45,10 @@ public void BeginConcurrentRequiresMvccAndSucceedsWhenEnabled() [Test] [TestCase("PRAGMA auto_vacuum")] [TestCase("PRAGMA auto_vacuum=NONE")] + [TestCase("PRAGMA auto_vacuum='none'")] + [TestCase("PRAGMA auto_vacuum=0")] + [TestCase("PRAGMA auto_vacuum=00")] + [TestCase("PRAGMA auto_vacuum=0x0")] [TestCase("PRAGMA incremental_vacuum")] [TestCase("PRAGMA incremental_vacuum(10)")] public void AutoVacuumFamilyIsAcceptedNoOp(string sql) @@ -66,6 +70,8 @@ public void AutoVacuumFamilyIsAcceptedNoOp(string sql) [TestCase("PRAGMA auto_vacuum=INCREMENTAL")] [TestCase("PRAGMA auto_vacuum=1")] [TestCase("PRAGMA auto_vacuum=2")] + [TestCase("PRAGMA auto_vacuum=3")] + [TestCase("PRAGMA auto_vacuum=bogus")] public void AutoVacuumNonNoneModeFailsClosedWithoutExperimentalFlag(string sql) { using var connection = Open(); diff --git a/src/Ahtola.Tests/NativeSqliteChurnInteropTests.cs b/src/Ahtola.Tests/NativeSqliteChurnInteropTests.cs index 019ab289..2baa5ee7 100644 --- a/src/Ahtola.Tests/NativeSqliteChurnInteropTests.cs +++ b/src/Ahtola.Tests/NativeSqliteChurnInteropTests.cs @@ -456,6 +456,8 @@ public void ManagedReaderOpensAutoVacuumDatabaseReadOnlyAndRefusesWrites() // documented part of the format SQLite accepts. ScalarString(managed, "PRAGMA quick_check;").Should().Be("ok", "the managed reader must accept the auto-vacuum database"); + ScalarLong(managed, "PRAGMA auto_vacuum;").Should().Be(1, + "like Turso, the mode is read from page 1 rather than assumed to be NONE"); var diagnostics = "native churn interop (auto-vacuum)"; TypedSqliteOracle.AssertEquivalent( managed, From 672eb97f76b2d24fa305cd22864c7dc482d11628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 15:54:27 -0400 Subject: [PATCH 07/20] Close v0.8.1 schema/DML/pragma parity gaps Ports the Turso v0.8.1 (turso-src 8549c1659) fixes behind 19 ledger entries recorded by the v0.8.1 corpus adoption, plus non-corpus parity: - REPLACE defaults now land before virtual generated columns are recomputed (Turso ea7b14a0a), so CHECK, UNIQUE keys, NOT NULL, STRICT typing and RETURNING see the substituted value; generated-column NOT NULL is enforced after the default pass. - Invalid generated-column clauses report SQLite's messages (be3c173b7): a second generation clause, a clause after DEFAULT, or an unknown type token fail with `error in generated column ""`. - UPSERT honors the INSERT target alias (859c91ade): the alias names the target row, hides the base name, and shadows `excluded`. - UPSERT DO UPDATE keeps columns a BEFORE UPDATE trigger rewrote and recomputes generated columns (a99d061a5). - User INSERT/UPDATE/DELETE on the __turso_internal_ namespace fails with `table X may not be modified` regardless of case (67262f427); managed backup/snapshot replay is exempt like upstream nested statements. - ALTER TABLE RENAME onto a view reports `there is already another table or index with this name` (799e4737c). - PRAGMA temp.synchronous reads OFF and ignores writes, like SQLite. - CREATE TABLE / ADD COLUMN enforce SQLITE_MAX_COLUMN 2000 with SQLite's messages (a81efd85b). - Named parameters tokenize with SQLite's CC_VARALPHA rule: bare `$`, `:`, `@` (or `$::`, unclosed `$a(b`) are `unrecognized token`, and TCL names `$::v`, `$ns::v`, `$arr(e)` are single parameters (22a0a0f6e, a45cd87ff). - PRAGMA count_changes uses SQLite's column names, reports zero-row statements, leaves RETURNING rows alone, and counts only rows an UPSERT inserted. Co-Authored-By: Claude Opus 5.5 --- src/Ahtola.Core/EmbeddedDatabase.Triggers.cs | 17 +- src/Ahtola.Core/EmbeddedDatabase.cs | 147 +++++- src/Ahtola.Core/ManagedLocalAdapters.cs | 3 + src/Ahtola.Core/Parsing/SqlLexer.cs | 72 ++- src/Ahtola.Core/Parsing/SqlParameterMap.cs | 36 +- src/Ahtola.Core/Parsing/SqlParser.cs | 44 +- ...lterTableMigrationReviewRegressionTests.cs | 4 +- .../managed-sqltest-expected-failures.txt | 19 - .../SchemaDmlPragmaParityTests.cs | 465 ++++++++++++++++++ 9 files changed, 710 insertions(+), 97 deletions(-) create mode 100644 src/Ahtola.Tests/SchemaDmlPragmaParityTests.cs diff --git a/src/Ahtola.Core/EmbeddedDatabase.Triggers.cs b/src/Ahtola.Core/EmbeddedDatabase.Triggers.cs index 622e7016..956ed7eb 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.Triggers.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.Triggers.cs @@ -2172,7 +2172,7 @@ private ExecutionResult PerformRowTriggeredUpsert( TriggerMutationKind.Update, plan); ValidateUpsertUpdateExpressions( - statement.TableName, + statement.TargetAlias ?? statement.TableName, update.Assignments, update.Where, allowTriggerQualifiers: context.InsideTrigger); @@ -2237,6 +2237,7 @@ private ExecutionResult PerformRowTriggeredUpsert( var returningRows = new List(); string[]? returningColumns = null; var rowsAffected = 0; + var rowsInserted = 0; long? lastInsertRowId = null; foreach (var values in inputRows) @@ -2356,6 +2357,7 @@ private ExecutionResult PerformRowTriggeredUpsert( if (!inserted) continue; rowsAffected++; + rowsInserted++; context.TriggerState!.Changed = true; if (table.HasRowid) { @@ -2406,7 +2408,7 @@ private ExecutionResult PerformRowTriggeredUpsert( var original = table.Rows[conflictPosition].ToArray(); var originalRowId = table.HasRowid ? table.RowIds[conflictPosition] : conflictPosition + 1; var source = CreateUpsertSourceRow( - statement.TableName, + statement.TargetAlias ?? statement.TableName, table, original, originalRowId, @@ -2437,6 +2439,16 @@ private ExecutionResult PerformRowTriggeredUpsert( conflictPosition = FindTriggerRowPosition(table, identity); if (conflictPosition < 0) continue; + // A BEFORE UPDATE trigger may have rewritten the conflicting row. Like a plain + // UPDATE (and Turso's "refresh index deletion values after BEFORE UPDATE trigger by + // UPSERT"), keep the trigger's values for every column DO UPDATE does not SET, then + // recompute generated columns from the refreshed row. + updated = ReloadColumnsTheUpdateDoesNotAssign( + table, + updatePlan!, + updated, + table.Rows[conflictPosition]); + ComputeGeneratedColumns(table, statement.TableName, updated, parameters, doUpdateContext); ValidateCheckConstraints( statement.TableName, table, @@ -2498,6 +2510,7 @@ private ExecutionResult PerformRowTriggeredUpsert( rowsAffected > 0 || context.TriggerState!.Changed) { LastInsertRowId = lastInsertRowId, + CountChangesRows = rowsInserted, }; } diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index b1e16831..bf7d49c1 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -7966,6 +7966,11 @@ private ExecutionResult ExecuteDropTrigger( throw new EmbeddedSqlException("virtual tables may not be altered"); if (!context.Tables.TryGetValue(statement.TableName, out var table)) throw new EmbeddedSqlException($"no such table: {statement.TableName}"); + // sqlite3AddColumn enforces SQLITE_MAX_COLUMN against the scratch copy that + // sqlite3AlterBeginAddColumn names "sqlite_altertab_
", so SQLite's message + // carries that name (Turso's "enforce 2,000 max columns limit on tables"). + if (table.Columns.Length >= SqlParser.MaxColumns) + throw new EmbeddedSqlException($"too many columns on sqlite_altertab_{table.Name}"); if (statement.Column.GeneratedStored) throw new EmbeddedSqlException("cannot add a STORED column"); @@ -9723,7 +9728,7 @@ private void ValidateTriggerInsert( if (upsert.Action is DoUpdateUpsertAction update) { var upsertRow = CreateUpsertSourceRow( - statement.TableName, + statement.TargetAlias ?? statement.TableName, table, Enumerable.Repeat(SqlValue.Null, table.Columns.Length).ToArray(), targetRowId: 0, @@ -11234,7 +11239,7 @@ private ExecutionResult ExecuteVirtualTableMutations( private ExecutionResult ExecuteInsert(InsertStatement statement, SqlValue[] parameters, QueryContext context) { - RejectInternalTypeTableMutation(statement.TableName); + RejectInternalTableDml(statement.TableName); if (context.InsideTrigger && context.TriggerConflictAlgorithm is { } triggerConflictAlgorithm) statement = statement with { ConflictAlgorithm = triggerConflictAlgorithm }; else if (context.InsideTrigger @@ -11870,6 +11875,7 @@ void InsertCandidate( return; ResolveNotNullReplaceDefaults(statement, table, row, triggerContext); + EnforceGeneratedNotNullConstraints(table, statement.TableName, row); if (deferredRowId is not null) { rowId = FinalizeAutomaticRowId( @@ -12508,7 +12514,7 @@ private ExecutionResult PerformUpsertEvaluated( Where: null); plan = PrepareUpdate(updateStatement, table, context); ValidateUpsertUpdateExpressions( - statement.TableName, + statement.TargetAlias ?? statement.TableName, update.Assignments, update.Where, allowTriggerQualifiers: context.InsideTrigger); @@ -12558,6 +12564,9 @@ private ExecutionResult PerformUpsertEvaluated( var affectedRows = new List(); var affectedRowIds = new List(); var affectedLastInsertRowIds = new List(); + // PRAGMA count_changes reports "rows inserted": SQLite's insert counter never + // counts a row DO UPDATE changed instead of inserting. + var insertedRowCount = 0; var insertTriggers = GetMatchingTriggers(context, statement.TableName, TriggerEvent.Insert); var deleteTriggers = context.RecursiveTriggersEnabled ? GetMatchingTriggers(context, statement.TableName, TriggerEvent.Delete) @@ -12699,6 +12708,7 @@ private ExecutionResult PerformUpsertEvaluated( affectedRows.Add(candidate); affectedRowIds.Add(candidateRowId); + insertedRowCount++; if (table.HasRowid) { lastInsertRowId = candidateRowId; @@ -12726,7 +12736,7 @@ private ExecutionResult PerformUpsertEvaluated( var original = table.Rows[conflictPosition]; var originalRowId = table.RowIds[conflictPosition]; var source = CreateUpsertSourceRow( - statement.TableName, + statement.TargetAlias ?? statement.TableName, table, original, originalRowId, @@ -12870,7 +12880,10 @@ private ExecutionResult PerformUpsertEvaluated( rowsAffected: affectedRows.Count, changed: affectedRows.Count > 0, lastInsertRowId: lastInsertRowId, - affectedLastInsertRowIds: affectedLastInsertRowIds); + affectedLastInsertRowIds: affectedLastInsertRowIds) with + { + CountChangesRows = insertedRowCount, + }; } catch (EmbeddedSqlException exception) when ( lastInsertRowId.HasValue @@ -13451,8 +13464,11 @@ private int FindUpsertConflictPosition( return (updated, updatedRowId); } + // targetName is the INSERT target alias when one is written, else the table name: an + // alias hides the base table name, and an alias spelled `excluded` shadows the pseudo-row + // (SQLite name resolution; Turso's "honor INSERT target aliases in UPSERT"). private static SourceRow CreateUpsertSourceRow( - string tableName, + string targetName, EmbeddedTable table, SqlValue[] target, long targetRowId, @@ -13462,10 +13478,12 @@ private static SourceRow CreateUpsertSourceRow( Array.Copy(target, values, target.Length); Array.Copy(excluded, 0, values, target.Length, excluded.Length); var qualified = new Dictionary(StringComparer.OrdinalIgnoreCase); + var targetShadowsExcluded = targetName.Equals("excluded", StringComparison.OrdinalIgnoreCase); for (var index = 0; index < table.Columns.Length; index++) { - qualified.Add($"{tableName}.{table.Columns[index]}", index); - qualified.Add($"excluded.{table.Columns[index]}", table.Columns.Length + index); + qualified.Add($"{targetName}.{table.Columns[index]}", index); + if (!targetShadowsExcluded) + qualified.Add($"excluded.{table.Columns[index]}", table.Columns.Length + index); } return new SourceRow( @@ -13473,7 +13491,7 @@ private static SourceRow CreateUpsertSourceRow( values, qualified, RowId: table.HasRowid ? targetRowId : null, - RowIdQualifier: tableName, + RowIdQualifier: targetName, ColumnDefinitions: table.ColumnDefinitions .Cast() .Concat(table.ColumnDefinitions) @@ -14455,8 +14473,16 @@ private InsertPlan PrepareInsert( row[plan.AliasIndex] = SqlValue.Integer(rowid); // Generated columns are computed after the base columns (and any rowid alias) - // are final, so they can reference every stored column value. - ComputeGeneratedColumns(table, statement.TableName, row, parameters, context); + // are final, so they can reference every stored column value. When REPLACE default + // substitution is deferred past BEFORE triggers, so is generated-column NOT NULL + // enforcement: the caller recomputes once the defaults land (SQLite's constraint pass). + ComputeGeneratedColumns( + table, + statement.TableName, + row, + parameters, + context, + enforceNotNull: resolveNotNullReplace); if (validateCheckConstraints) ValidateCheckConstraints(statement.TableName, table, row, rowid, parameters, context); @@ -14500,8 +14526,15 @@ private void ResolveNotNullReplaceDefaults( changed = true; } - if (changed) - table.ApplyAffinities(row); + if (!changed) + return; + + table.ApplyAffinities(row); + // Virtual generated columns were computed from the NULL the default replaced; like + // SQLite (and Turso's "apply REPLACE defaults before computing virtual generated + // columns"), recompute them so CHECK, unique keys, NOT NULL and RETURNING see the + // substituted value. Callers enforce generated-column NOT NULL afterwards. + ComputeGeneratedColumns(table, table.Name, row, EmptyParameters, context, enforceNotNull: false); } // Validates the pending inserts against the whole table, then appends them and @@ -14966,7 +14999,7 @@ private ExecutionResult ExecuteUpdate( SqlValue[] parameters, QueryContext context) { - RejectInternalTypeTableMutation(statement.TableName); + RejectInternalTableDml(statement.TableName); if (TryGetVirtualTable(context, new NamedTableSource(statement.TableName, statement.Alias), out var virtualTable)) return ExecuteVirtualTableUpdate(statement, virtualTable, parameters, context); @@ -17583,7 +17616,7 @@ private ExecutionResult ExecuteDelete( SqlValue[] parameters, QueryContext context) { - RejectInternalTypeTableMutation(statement.TableName); + RejectInternalTableDml(statement.TableName); if (TryGetVirtualTable(context, new NamedTableSource(statement.TableName, statement.Alias), out var virtualTable)) return ExecuteVirtualTableDelete(statement, virtualTable, parameters, context); @@ -28530,8 +28563,11 @@ private static void ValidateAlterTableRenameName( throw new EmbeddedSqlException($"object name reserved for internal use: {statement.NewName}"); if (tables.ContainsKey(statement.NewName) || virtualTables?.ContainsKey(statement.NewName) == true) throw new EmbeddedSqlException($"table {statement.NewName} already exists"); + // sqlite3AlterRenameTable finds views through sqlite3FindTable, so a view name collision + // reports the table-or-index message (Turso's "refuse ALTER TABLE RENAME onto an + // existing view name"). if (views?.ContainsKey(statement.NewName) == true) - throw new EmbeddedSqlException($"there is already a view named {statement.NewName}"); + throw new EmbeddedSqlException($"there is already another table or index with this name: {statement.NewName}"); if (triggers?.ContainsKey(statement.NewName) == true) throw new EmbeddedSqlException($"there is already a trigger named {statement.NewName}"); if (TryFindIndex(tables, statement.NewName, out _, out _)) @@ -39365,6 +39401,23 @@ internal static bool IsReservedObjectName(string name) || IsAutoIncrementSequenceBackingTable(name) || Indexing.ManagedIndexMethodNames.IsReserved(name); + private static readonly AsyncLocal s_internalTableDmlScopes = new(); + + /// + /// Lets engine-internal statements on the current flow write the reserved + /// __turso_internal_ tables (Turso's nested-statement exemption) until disposed. + /// + internal static InternalTableDmlScope AllowInternalTableDml() + { + s_internalTableDmlScopes.Value++; + return default; + } + + internal readonly struct InternalTableDmlScope : IDisposable + { + public void Dispose() => s_internalTableDmlScopes.Value--; + } + private static void RejectInternalTypeTableMutation(string name) { if (ManagedSchemaName.TrySplit(name, out _, out var localName)) @@ -39373,6 +39426,23 @@ private static void RejectInternalTypeTableMutation(string name) throw new EmbeddedSqlException("The internal type registry cannot be modified directly."); } + // Mirrors Turso's allow_user_dml: user INSERT/UPDATE/DELETE may not target the reserved + // __turso_internal_ namespace (sequence backing tables, ...), matched ASCII + // case-insensitively like every other identifier lookup ("protect internal tables + // regardless of identifier case"). Engine-internal statements that replay a whole image + // (managed backup/snapshot copy) run inside AllowInternalTableDml, upstream's + // is_nested_stmt exemption. + private static void RejectInternalTableDml(string name) + { + RejectInternalTypeTableMutation(name); + if (s_internalTableDmlScopes.Value > 0) + return; + if (ManagedSchemaName.TrySplit(name, out _, out var localName)) + name = localName; + if (name.StartsWith(TursoInternalReservedPrefix, StringComparison.OrdinalIgnoreCase)) + throw new EmbeddedSqlException($"table {name} may not be modified"); + } + private static SourceData GetNamedTableRows( NamedTableSource source, QueryContext context, @@ -59777,9 +59847,19 @@ or AttachDatabaseStatement tempTriggerSession?.Commit(); // PRAGMA count_changes: each INSERT/UPDATE/DELETE returns one row with - // the number of rows it changed (SQLite's deprecated count_changes). - if (_countChanges && result.Changed && statement is InsertStatement or UpdateStatement or DeleteStatement) - return new ExecutionResult(["changes"], [[SqlValue.Integer(result.RowsAffected)]], 0, result.Changed); + // the number of rows it changed (SQLite's deprecated count_changes), even + // when that is zero. A RETURNING statement returns its own rows instead. + if (GetCountChangesColumn(statement) is { } countChangesColumn) + { + return new ExecutionResult( + [countChangesColumn], + [[SqlValue.Integer(result.CountChangesRows ?? result.RowsAffected)]], + result.RowsAffected, + result.Changed) + { + LastInsertRowId = result.LastInsertRowId, + }; + } return result; } @@ -65114,16 +65194,21 @@ private ExecutionResult ExecutePragmaBusyTimeout(PragmaBusyTimeoutStatement stat private ExecutionResult ExecutePragmaSynchronous(PragmaSynchronousStatement statement) { var database = ResolvePragmaDatabase(statement.Schema); + // The temp database is never synced: SQLite reports its safety level as OFF and its + // pragma.c skips the assignment for iDb 1, so a write leaves it OFF. + var isTemp = ReferenceEquals(database, _tempDatabase); if (statement.Value is null) { return new ExecutionResult( ["synchronous"], - [[SqlValue.Integer((int)GetSynchronousMode(database))]], + [[SqlValue.Integer((int)(isTemp ? SqliteSynchronousMode.Off : GetSynchronousMode(database)))]], 0); } if (HasActiveTransaction) throw new EmbeddedSqlException("Safety level may not be changed inside a transaction"); + if (isTemp) + return ExecutionResult.Empty; var current = GetSynchronousMode(database); _synchronousModes[database] = statement.Value.ToUpperInvariant() switch @@ -65426,6 +65511,22 @@ private ExecutionResult ExecuteVacuum(VacuumStatement statement, SqlValue[] para /// public long LastInsertRowId => _lastInsertRowId; + // SQLite's count_changes result column for a top-level INSERT/UPDATE/DELETE, or null when the + // pragma is off or the statement has RETURNING (which returns its own rows instead). + private string? GetCountChangesColumn(ParsedStatement statement) + { + if (!_countChanges || EmbeddedDatabase.TryGetReturning(statement, out _, out _)) + return null; + + return (statement is WithDmlStatement with ? with.Dml : statement) switch + { + InsertStatement => "rows inserted", + UpdateStatement => "rows updated", + DeleteStatement => "rows deleted", + _ => null, + }; + } + internal string[] DescribeColumns(ParsedStatement statement) { ThrowIfRecursiveTriggerCallbackReentry(); @@ -65463,6 +65564,8 @@ internal string[] DescribeColumns(ParsedStatement statement) return ["query_only"]; if (statement is PragmaCountChangesStatement { Enabled: null }) return ["count_changes"]; + if (GetCountChangesColumn(statement) is { } countChangesColumn) + return [countChangesColumn]; if (statement is PragmaForeignKeysStatement { Enabled: null }) return ["foreign_keys"]; if (statement is PragmaDeferForeignKeysStatement { Enabled: null }) @@ -69664,6 +69767,10 @@ internal sealed record ExecutionResult( // the owning connection can answer last_insert_rowid(). Null for statements that did // not insert a row. public long? LastInsertRowId { get; init; } + + // The count PRAGMA count_changes reports when it differs from RowsAffected: an UPSERT + // counts only the rows it inserted, not the ones DO UPDATE changed. Null means RowsAffected. + public int? CountChangesRows { get; init; } } internal sealed class StreamingProjectionRows( diff --git a/src/Ahtola.Core/ManagedLocalAdapters.cs b/src/Ahtola.Core/ManagedLocalAdapters.cs index 12685267..8c699445 100644 --- a/src/Ahtola.Core/ManagedLocalAdapters.cs +++ b/src/Ahtola.Core/ManagedLocalAdapters.cs @@ -1028,6 +1028,9 @@ public static void Copy( IManagedConnectionAdapter destination, bool sourceTransactionActive = false) { + // The copy replays sequence backing tables through ordinary DML, which user statements + // may not do (RejectInternalTableDml). + using var internalTableWrites = EmbeddedDatabase.AllowInternalTableDml(); var sourceTransactionStarted = false; var destinationTransactionStarted = false; var destinationForeignKeysDisabled = false; diff --git a/src/Ahtola.Core/Parsing/SqlLexer.cs b/src/Ahtola.Core/Parsing/SqlLexer.cs index d8f2d28b..207ca016 100644 --- a/src/Ahtola.Core/Parsing/SqlLexer.cs +++ b/src/Ahtola.Core/Parsing/SqlLexer.cs @@ -339,34 +339,70 @@ private SqlToken ReadBlob(int start) private SqlToken ReadParameter(int start) { - ConsumeParameterIdentifier(); - if (_sql[start] == '$') + if (_sql[start] == '?') { - while (_offset + 1 < _sql.Length && _sql[_offset] == ':' && _sql[_offset + 1] == ':') - { - _offset += 2; - ConsumeParameterIdentifier(); - } - - if (_offset < _sql.Length && _sql[_offset] == '(') - { + // CC_QUESTION: `?` takes only the digits of a `?NNN` index. + while (_offset < _sql.Length && char.IsAsciiDigit(_sql[_offset])) _offset++; - ConsumeParameterIdentifier(); - if (_offset < _sql.Length && _sql[_offset] == ')') - _offset++; - } + } + else + { + _offset = ScanNamedParameter(_sql, start, out var valid); + if (!valid) + throw new EmbeddedSqlException($"unrecognized token: \"{_sql[start.._offset]}\""); } return new SqlToken(TokenKind.Parameter, _sql[start.._offset], start); } - private void ConsumeParameterIdentifier() + /// + /// Scans a $/@/: named parameter starting at with + /// SQLite's tokenizer rule (sqlite3GetToken, CC_VARALPHA): the name is identifier characters, a + /// :: pair anywhere in it belongs to the name (TCL $::var, $ns::var), and once + /// at least one identifier character was read a (...) suffix without whitespace ends it (TCL + /// $arr(elem)). A marker with no identifier character, or with an unclosed suffix, is + /// SQLite's TK_ILLEGAL, which it reports as an unrecognized token (Turso's "report bad parameter + /// names as unrecognized tokens" and "accept namespace-qualified and array-element parameter + /// names"). + /// + /// The offset just past the scanned token. + internal static int ScanNamedParameter(string sql, int start, out bool valid) { - while (_offset < _sql.Length - && (char.IsAsciiLetterOrDigit(_sql[_offset]) || _sql[_offset] is '_' or '$')) + var offset = start + 1; + var identifierCharacters = 0; + valid = true; + while (offset < sql.Length) { - _offset++; + var current = sql[offset]; + if (IsIdentifierContinue(current)) + { + identifierCharacters++; + offset++; + } + else if (current == '(' && identifierCharacters > 0) + { + offset++; + while (offset < sql.Length && sql[offset] is not (')' or ' ' or '\t' or '\n' or '\v' or '\f' or '\r')) + offset++; + if (offset < sql.Length && sql[offset] == ')') + offset++; + else + valid = false; + break; + } + else if (current == ':' && offset + 1 < sql.Length && sql[offset + 1] == ':') + { + offset += 2; + } + else + { + break; + } } + + if (identifierCharacters == 0) + valid = false; + return offset; } private SqlToken ReadNumber(int start, bool startsWithDecimalPoint = false) diff --git a/src/Ahtola.Core/Parsing/SqlParameterMap.cs b/src/Ahtola.Core/Parsing/SqlParameterMap.cs index 1bfcdbf8..162479f1 100644 --- a/src/Ahtola.Core/Parsing/SqlParameterMap.cs +++ b/src/Ahtola.Core/Parsing/SqlParameterMap.cs @@ -127,9 +127,10 @@ private static int AddNamedParameter( List referenced, Dictionary indices) { - var end = ScanNamedParameterEnd(sql, cursor); + // The lexer rejects an invalid marker as an unrecognized token; the map just skips it. + var end = SqlLexer.ScanNamedParameter(sql, cursor, out var valid); - if (end == cursor + 1) + if (!valid) return end; var name = sql[cursor..end]; @@ -145,34 +146,6 @@ private static int AddNamedParameter( return end; } - private static int ScanNamedParameterEnd(string sql, int cursor) - { - var end = cursor + 1; - while (end < sql.Length && IsParameterIdentifierCharacter(sql[end])) - end++; - - if (sql[cursor] != '$') - return end; - - while (end + 1 < sql.Length && sql[end] == ':' && sql[end + 1] == ':') - { - end += 2; - while (end < sql.Length && IsParameterIdentifierCharacter(sql[end])) - end++; - } - - if (end < sql.Length && sql[end] == '(') - { - end++; - while (end < sql.Length && IsParameterIdentifierCharacter(sql[end])) - end++; - if (end < sql.Length && sql[end] == ')') - end++; - } - - return end; - } - private static void EnsureSlot(List names, List referenced, int index) { while (names.Count <= index) @@ -191,9 +164,6 @@ private static void EnsureParameterLimit(int count) } } - private static bool IsParameterIdentifierCharacter(char value) - => char.IsAsciiLetterOrDigit(value) || value is '_' or '$'; - private static int SkipQuoted(string sql, int cursor, char delimiter, char escapedDelimiter) { cursor++; diff --git a/src/Ahtola.Core/Parsing/SqlParser.cs b/src/Ahtola.Core/Parsing/SqlParser.cs index cebb4d19..57c0961d 100644 --- a/src/Ahtola.Core/Parsing/SqlParser.cs +++ b/src/Ahtola.Core/Parsing/SqlParser.cs @@ -7,6 +7,9 @@ namespace Ahtola.Core.Parsing; internal sealed class SqlParser { + /// SQLite's default SQLITE_MAX_COLUMN (Turso's MAX_COLUMN). + internal const int MaxColumns = 2000; + private readonly SqlLexer _lexer; private readonly string _sql; private readonly Dictionary _namedParameterIndices = new(StringComparer.Ordinal); @@ -1169,6 +1172,14 @@ private ParsedStatement ParseCreateTable(bool temporary) { extentColumn = ParseColumnDefinition(); columns.Add(extentColumn); + // sqlite3AddColumn rejects the column past SQLITE_MAX_COLUMN while parsing. + if (columns.Count > MaxColumns) + { + var tableName = ManagedSchemaName.TrySplit(name, out _, out var localTableName) + ? localTableName + : name; + throw new EmbeddedSqlException($"too many columns on {tableName}"); + } } if (_lexer.Current.Kind != TokenKind.Comma) @@ -4238,6 +4249,7 @@ private EmbeddedColumn ParseColumnDefinition() int? primaryKeyDeclarationOrder = null; int? uniqueDeclarationOrder = null; string? pendingConstraintName = null; + var hasDefault = false; while (_lexer.Current.Kind == TokenKind.Identifier) { if (ConsumeKeyword("CONSTRAINT")) @@ -4305,6 +4317,10 @@ private EmbeddedColumn ParseColumnDefinition() } if (ConsumeKeyword("DEFAULT")) { + // sqlite3AddDefaultValue rejects a DEFAULT that follows a generation clause. + if (generationExpression is not null) + throw new EmbeddedSqlException("cannot use DEFAULT on a generated column"); + hasDefault = true; var startOffset = _lexer.Current.Offset; var parenthesized = _lexer.Current.Kind == TokenKind.LeftParen; var expression = parenthesized @@ -4337,9 +4353,10 @@ private EmbeddedColumn ParseColumnDefinition() // round-trips through schema regeneration. if (ConsumeKeyword("GENERATED")) { + ThrowIfGenerationClauseIsInvalid(name, generationExpression is not null || hasDefault); ExpectKeyword("ALWAYS"); ExpectKeyword("AS"); - (generationExpression, generationSql, generatedStored, generationVirtualSpelled) = ParseGenerationClause(); + (generationExpression, generationSql, generatedStored, generationVirtualSpelled) = ParseGenerationClause(name); generationAlways = true; generationConstraintName = pendingConstraintName; pendingConstraintName = null; @@ -4347,7 +4364,8 @@ private EmbeddedColumn ParseColumnDefinition() } if (ConsumeKeyword("AS")) { - (generationExpression, generationSql, generatedStored, generationVirtualSpelled) = ParseGenerationClause(); + ThrowIfGenerationClauseIsInvalid(name, generationExpression is not null || hasDefault); + (generationExpression, generationSql, generatedStored, generationVirtualSpelled) = ParseGenerationClause(name); generationConstraintName = pendingConstraintName; pendingConstraintName = null; continue; @@ -4488,7 +4506,7 @@ private EmbeddedColumn ParseColumnDefinition() // generated column can be regenerated verbatim; VIRTUAL is the SQLite default. Also // reports whether VIRTUAL was spelled out, because SQLite preserves the original // spelling and schema regeneration must not add a VIRTUAL keyword that was not written. - private (Expression Expression, string Sql, bool Stored, bool VirtualSpelled) ParseGenerationClause() + private (Expression Expression, string Sql, bool Stored, bool VirtualSpelled) ParseGenerationClause(string columnName) { Expect(TokenKind.LeftParen); var startOffset = _lexer.Current.Offset; @@ -4504,9 +4522,29 @@ private EmbeddedColumn ParseColumnDefinition() else virtualSpelled = ConsumeKeyword("VIRTUAL"); + // SQLite's grammar reads any identifier after the parenthesized expression as the + // generated-column type, and sqlite3AddGenerated rejects everything except STORED and + // VIRTUAL (Turso's "reject invalid generated-column clauses"). A following column + // constraint keyword still begins the next constraint. + if (!stored + && !virtualSpelled + && _lexer.Current.Kind == TokenKind.Identifier + && (_lexer.Current.IsQuoted || !IsColumnConstraintKeyword(_lexer.Current.Text))) + { + ThrowIfGenerationClauseIsInvalid(columnName, invalid: true); + } + return (expression, rawSql, stored, virtualSpelled); } + // sqlite3AddGenerated's generated_error: a second generation clause, a generation clause + // after DEFAULT, or an unknown generated-column type all report this one message. + private static void ThrowIfGenerationClauseIsInvalid(string columnName, bool invalid) + { + if (invalid) + throw new EmbeddedSqlException($"error in generated column \"{columnName}\""); + } + private bool IsTableConstraintStart() { // A quoted identifier is never a keyword, so `CREATE TABLE t("NULL", "CHECK")` declares two diff --git a/src/Ahtola.Tests/AlterTableMigrationReviewRegressionTests.cs b/src/Ahtola.Tests/AlterTableMigrationReviewRegressionTests.cs index 9d53e114..860acf07 100644 --- a/src/Ahtola.Tests/AlterTableMigrationReviewRegressionTests.cs +++ b/src/Ahtola.Tests/AlterTableMigrationReviewRegressionTests.cs @@ -277,7 +277,7 @@ public void RenamingATableOntoAViewIsRefusedWhilePlanning() Execute(connection, "CREATE VIEW v AS SELECT 1 AS x;"); ShouldThrow(connection, "ALTER TABLE t RENAME TO v;") - .Message.Should().Be("there is already a view named v"); + .Message.Should().Be("there is already another table or index with this name: v"); ReadRows(connection, SchemaQuery).Should().Equal("table|t|t", "view|v|v"); } @@ -329,7 +329,7 @@ public void ARefusedRenameNeverSurfacesTheRuntimeSchemaException() } } - [TestCase("v", "there is already a view named v")] + [TestCase("v", "there is already another table or index with this name: v")] [TestCase("tr", "there is already a trigger named tr")] [TestCase("occupied", "table occupied already exists")] [TestCase("ix", "there is already an index named ix")] diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index 3d47aa06..c4927248 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -167,16 +167,6 @@ agg-functions/json-group-object-null-label.sqltest::json_group_object_null_label agg-functions/json-group-object-null-label.sqltest::json_group_object_skips_null_label | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"a":1,"b":3}' but was '{"a":1,"":2,"b":3}' agg-functions/json-group-object-null-label.sqltest::jsonb_group_object_skips_null_label | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"a":1,"b":3}' but was '{"a":1,"":2,"b":3}' alter_table.sqltest::alter-table-add-column-preserve-schema-collate-locale | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected 'CREATE TABLE tbl (c TEXT COLLATE "fr-FR", d)' but was 'CREATE TABLE tbl (c TEXT COLLATE 'fr-FR', d)' -alter_table.sqltest::fail-alter-table-rename-to-existing-view | database variant 1/1 (:memory:): error 'there is already a view named y' does not match pattern 'there is already another table or index with this name: y' -alter_table.sqltest::fail-alter-table-rename-to-existing-view-different-case | database variant 1/1 (:memory:): error 'there is already a view named Y' does not match pattern 'there is already another table or index with this name: Y' -gencol.sqltest::gencol_issue6161_strict_compute_after_replace_default | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '42/42/integer' but was '42//null' -gencol.sqltest::gencol_issue7058_alter_add_default_on_gencol | database variant 1/1 (:memory:): error 'cannot use DEFAULT on a generated column' does not match pattern 'error in generated column "b"' -gencol.sqltest::gencol_issue7059_unknown_type_token_rejected | database variant 1/1 (:memory:): error 'Unsupported column constraint 'WAT'. At SQL offset 27.' does not match pattern 'error in generated column "b"' -gencol.sqltest::gencol_issue7060_multiple_generated_clauses_rejected | database variant 1/1 (:memory:): expected an error but the statement succeeded -gencol.sqltest::gencol_issue7068_check_sees_recomputed_value | database variant 1/1 (:memory:): expected success but got error: CHECK constraint failed: b IS 6 -gencol.sqltest::gencol_issue7068_multiple_defaults_before_generated_notnull | database variant 1/1 (:memory:): expected success but got error: NOT NULL constraint failed: t.b -gencol.sqltest::gencol_issue7068_returning_recomputed_after_default | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '5/6' but was '5/' -gencol.sqltest::gencol_issue7068_unique_index_uses_recomputed_value | database variant 1/1 (:memory:): expected 2 row(s), got 2 row(s); row 0 expected '1/5/6' but was '1/5/' generate_series.sqltest::generate-series-null-arguments-return-no-rows | database variant 1/1 (:memory:): expected success but got error: generate_series() requires integer arguments integrity_check/memory.sqltest::integrity-check-qualified-corrupt-attached-database | database variant 1/1 (:memory:): expected success but got error: Managed ATTACH requires a file-backed managed primary database so attachments share its file system. join/hash.sqltest::hash-join-result-matrix-full-join-uses-a-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN matrix_right AS r' @@ -219,7 +209,6 @@ partial_idx.sqltest::partial-index-or-right-term-with-range-search | database va partial_idx.sqltest::partial-index-or-term-with-covering-scan | database variant 1/1 (:memory:): expected 3 row(s), got 3 row(s); row 0 expected '1/0/0/SCAN t USING COVERING INDEX idx' but was '1/0/0/SCAN t' partial_idx.sqltest::partial-index-or-term-with-equality-search | database variant 1/1 (:memory:): expected 2 row(s), got 2 row(s); row 0 expected '1/0/0/SEARCH t USING INDEX idx (a=?)' but was '1/0/0/SCAN t' partial_idx.sqltest::partial-index-or-term-with-order | database variant 1/1 (:memory:): expected 3 row(s), got 4 row(s); row 0 expected '1/0/0/SCAN t USING INDEX idx' but was '1/0/0/SCAN t' -pragma/memory.sqltest::pragma-temp-synchronous-stays-off | database variant 1/1 (:memory:): expected 3 row(s), got 3 row(s); row 0 expected '0' but was '2' scalar-functions-datetime.sqltest::date-blob-modifier | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '2024-06-16' but was '' scalar-functions-datetime.sqltest::date-blob-time-value | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '2024-01-01' but was '' scalar-functions-datetime.sqltest::date-unixepoch-at-1-millisecond-before-epochtime | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '1969-12-31' but was '1970-01-01' @@ -253,13 +242,6 @@ turso/explain-query-plan-json.sqltest::not-exists-is-reported-as-an-anti-join | turso/explain-query-plan-json.sqltest::quotes-in-the-statement-are-escaped-in-the-echo | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.name = 'a \"quote turso/explain-query-plan-json.sqltest::search-nodes-name-the-index-the-constraints-and-the-join | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, o.amount\n FROM users u\n LEFT JOIN order turso/explain-query-plan-json.sqltest::the-json-tree-matches-the-text-rows | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 4 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.age IN (SELECT ag -turso/sequence.sqltest::sequence-backing-table-rejects-mixed-case-insert | database variant 1/1 (:memory:): expected an error but the statement succeeded -turso/sequence.sqltest::sequence-backing-table-rejects-mixed-case-update | database variant 1/1 (:memory:): expected an error but the statement succeeded -turso/sequence.sqltest::sequence-backing-table-rejects-uppercase-insert | database variant 1/1 (:memory:): expected an error but the statement succeeded -turso/sequence.sqltest::sequence-backing-table-rejects-uppercase-update | database variant 1/1 (:memory:): expected an error but the statement succeeded -turso/upsert-target-alias.sqltest::upsert-base-name-is-out-of-scope-when-aliased | database variant 1/1 (:memory:): expected an error but the statement succeeded -turso/upsert-target-alias.sqltest::upsert-target-alias-is-only-target-name | database variant 1/1 (:memory:): expected success but got error: no such column: x.n -turso/upsert-target-alias.sqltest::upsert-target-alias-shadows-excluded | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '10' but was '99' turso/vacuum.sqltest::vacuum-rejects-query-only | database variant 1/1 (:temp:): error 'attempt to write a readonly database' does not match pattern 'Cannot execute VACUUM in query_only mode' unnest-correlated.sqltest::correlated-comparison-after-full-join[exists][s-amount-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 6 row(s) unnest-correlated.sqltest::correlated-comparison-after-full-join[exists][s-amount-is-not-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 7 row(s) @@ -275,4 +257,3 @@ unnest-correlated.sqltest::correlated-in-error-expressions-stay-subqueries | dat unnest-correlated.sqltest::correlated-in-inequality-uses-indexed-semi-join | database variant 1/1 (:memory:): output does not match pattern '(?s)SCAN outer_rows AS o.*SEARCH i USING COVERING INDEX inner_key_amount' unnest-correlated.sqltest::correlated-in-with-only-inequality-uses-indexed-semi-join | database variant 1/1 (:memory:): output does not match pattern '(?s)SCAN outer_rows AS o.*SEARCH i USING COVERING INDEX inner_key_amount \(key1<\?\)' unnest-correlated.sqltest::not-exists-can-use-hash-anti-join | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN hash_anti_inner AS i.*SCAN hash_anti_outer AS o' -upsert.sqltest::upsert-before-update-trigger-changes-non-set-column | database variant 1/1 (:memory:): expected 3 row(s), got 3 row(s); row 0 expected '1/6/11' but was '1/6/1' diff --git a/src/Ahtola.Tests/SchemaDmlPragmaParityTests.cs b/src/Ahtola.Tests/SchemaDmlPragmaParityTests.cs new file mode 100644 index 00000000..0225e03c --- /dev/null +++ b/src/Ahtola.Tests/SchemaDmlPragmaParityTests.cs @@ -0,0 +1,465 @@ +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; +using AwesomeAssertions; +using Ahtola.Core; +using MsData = Microsoft.Data.Sqlite; + +namespace Ahtola.Tests; + +/// +/// Differential regressions (managed engine vs bundled SQLite) for the schema, DML and PRAGMA +/// parity fixes ported from Turso v0.8.1: REPLACE defaults feeding generated columns, invalid +/// generated-column clauses, UPSERT target aliases, BEFORE UPDATE triggers fired by UPSERT, +/// the 2,000-column limit, parameter tokenization, PRAGMA count_changes, temp.synchronous and +/// RENAME onto a view. Sequence backing-table protection is Turso-only and is checked directly. +/// +public class SchemaDmlPragmaParityTests +{ + [TestCase(""" + CREATE TABLE t(a NOT NULL DEFAULT 5, b AS (a+1) UNIQUE); + REPLACE INTO t(a) VALUES(NULL) RETURNING a, b; + """)] + [TestCase(""" + CREATE TABLE t(a NOT NULL DEFAULT 5, b AS (a+1) UNIQUE); + REPLACE INTO t(a) VALUES(NULL); + SELECT rowid, a, b FROM t; + SELECT b FROM t INDEXED BY sqlite_autoindex_t_1 WHERE b = 6; + PRAGMA integrity_check; + """)] + [TestCase(""" + CREATE TABLE t(a NOT NULL DEFAULT 5, b AS (a+1) CHECK(b IS 6)); + REPLACE INTO t(a) VALUES(NULL); + SELECT a, b FROM t; + """)] + [TestCase(""" + CREATE TABLE t(b AS(a+c) NOT NULL UNIQUE CHECK(b=12), a NOT NULL DEFAULT 5, c NOT NULL DEFAULT 7); + INSERT OR REPLACE INTO t(a,c) VALUES(NULL,NULL) RETURNING a,c,b; + PRAGMA integrity_check; + """)] + [TestCase(""" + CREATE TABLE t(a TEXT NOT NULL ON CONFLICT REPLACE DEFAULT '42', b INT AS(a)) STRICT; + INSERT INTO t VALUES(NULL); + SELECT a, b, typeof(b) FROM t; + """)] + [TestCase(""" + CREATE TABLE t(a NOT NULL DEFAULT 5, b AS (a*2)); + CREATE TABLE log(v); + CREATE TRIGGER tr BEFORE INSERT ON t BEGIN INSERT INTO log VALUES (quote(new.a) || '/' || quote(new.b)); END; + REPLACE INTO t(a) VALUES(NULL); + SELECT a, b FROM t; + SELECT v FROM log; + """)] + [TestCase(""" + CREATE TABLE t(id INTEGER PRIMARY KEY, a NOT NULL DEFAULT 5, b AS (a+1)); + INSERT INTO t VALUES(1, 1); + UPDATE OR REPLACE t SET a = NULL WHERE id = 1; + SELECT a, b FROM t; + """)] + public void ReplaceDefaultsAreAppliedBeforeGeneratedColumnsAreComputed(string sql) + => AssertSameOutcome(sql); + + [TestCase("CREATE TABLE t(a); ALTER TABLE t ADD COLUMN b DEFAULT 5 AS (a+1);")] + [TestCase("CREATE TABLE t(a); ALTER TABLE t ADD COLUMN b AS (a+1) DEFAULT 5;")] + [TestCase("CREATE TABLE t(a, b DEFAULT 5 AS (a+1));")] + [TestCase("CREATE TABLE t(a, b AS (a+1) DEFAULT 5);")] + [TestCase("CREATE TABLE t(a, b DEFAULT 5 GENERATED ALWAYS AS (a+1));")] + [TestCase("CREATE TABLE t(a, b AS (a) WAT);")] + [TestCase("CREATE TABLE t(a, b AS (a) REINDEX);")] + [TestCase("CREATE TABLE t(a, \"b\" AS (a) WAT);")] + [TestCase("CREATE TABLE t(a, b, c AS (a+1) AS (a+2));")] + + [TestCase("CREATE TABLE t(a, b AS (a) VIRTUAL AS (a+1));")] + [TestCase("CREATE TABLE t(a, b AS (a) CONSTRAINT another AS (a+1));")] + [TestCase("CREATE TABLE t(a); ALTER TABLE t ADD COLUMN b AS (a) WAT;")] + [TestCase(""" + CREATE TABLE t(a, b AS (a) stored NOT NULL, c GENERATED ALWAYS AS (a) ViRtUaL, + d AS (a) CONSTRAINT positive CHECK (d > 0), e AS (a) COLLATE nocase, f DEFAULT 5); + INSERT INTO t(a) VALUES (3); + SELECT a, b, c, d, e, f FROM t; + """)] + public void InvalidGeneratedColumnClausesAreRejectedLikeSqlite(string sql) + => AssertSameOutcome(sql); + + // SQLite's grammar reads GENERATED after a generation clause as the column type and then + // fails with `near "ALWAYS": syntax error`; like Turso's parser the managed one reports the + // second generation clause itself. + [TestCase("CREATE TABLE t(a, b AS (a+1) GENERATED ALWAYS AS (a+2));")] + [TestCase("CREATE TABLE t(a); ALTER TABLE t ADD COLUMN b GENERATED ALWAYS AS (a+1) GENERATED ALWAYS AS (a+2);")] + public void ASecondGenerationClauseIsRejectedLikeTurso(string sql) + { + RunSqlite(sql).Error.Should().NotBeNull(); + RunManaged(sql).Error.Should().Be("error in generated column \"b\""); + } + + [TestCase(""" + CREATE TABLE t(k INT PRIMARY KEY, n INT); + INSERT INTO t VALUES(1,10); + INSERT INTO t AS excluded VALUES(1,99) ON CONFLICT(k) DO UPDATE SET n = excluded.n; + SELECT n FROM t; + """)] + [TestCase(""" + CREATE TABLE t(k INT PRIMARY KEY, n INT); + INSERT INTO t VALUES(1,10); + INSERT INTO t AS x VALUES(1,7) ON CONFLICT(k) DO UPDATE SET n = x.n + excluded.n WHERE x.n > 0; + SELECT n FROM t; + """)] + [TestCase(""" + CREATE TABLE t(k INT PRIMARY KEY, n INT); + INSERT INTO t VALUES(1,10); + INSERT INTO t AS x VALUES(1,7) ON CONFLICT(k) DO UPDATE SET n = t.n + 1; + """)] + [TestCase(""" + CREATE TABLE t(k INT PRIMARY KEY, n INT); + CREATE TABLE log(v); + CREATE TRIGGER tr BEFORE UPDATE ON t BEGIN INSERT INTO log VALUES (old.n); END; + INSERT INTO t VALUES(1,10); + INSERT INTO t AS x VALUES(1,7) ON CONFLICT(k) DO UPDATE SET n = x.n * 2; + SELECT n FROM t; + SELECT v FROM log; + """)] + public void UpsertHonorsTheInsertTargetAlias(string sql) + => AssertSameOutcome(sql); + + [TestCase(""" + CREATE TABLE t(a INTEGER PRIMARY KEY, b INTEGER, c INTEGER); + CREATE INDEX i ON t(c); + INSERT INTO t VALUES(1,1,1); + CREATE TRIGGER tg BEFORE UPDATE ON t BEGIN UPDATE t SET c=c+10 WHERE a=NEW.a; END; + INSERT INTO t VALUES(1,9,9) ON CONFLICT(a) DO UPDATE SET b=b+5; + SELECT a, b, c FROM t; + SELECT c FROM t WHERE c > 0 ORDER BY c; + PRAGMA integrity_check; + """)] + [TestCase(""" + CREATE TABLE t(a INTEGER PRIMARY KEY, c INTEGER, g AS (c*2)); + CREATE INDEX i ON t(c); + CREATE INDEX ig ON t(g); + INSERT INTO t VALUES(1,1); + CREATE TRIGGER tg BEFORE UPDATE ON t BEGIN UPDATE t SET c=c+1 WHERE a=NEW.a; END; + INSERT INTO t VALUES(1,9) ON CONFLICT(a) DO UPDATE SET c=c+5; + SELECT a, c, g FROM t; + SELECT g FROM t WHERE g > 0; + DELETE FROM t; + SELECT count(*) FROM t WHERE c > 0; + PRAGMA integrity_check; + """)] + public void UpsertKeepsRowChangesMadeByABeforeUpdateTrigger(string sql) + => AssertSameOutcome(sql); + + [TestCase(""" + CREATE TABLE x(a); + CREATE VIEW y AS SELECT 1; + ALTER TABLE x RENAME TO y; + """)] + [TestCase(""" + CREATE TABLE x(a); + CREATE VIEW y AS SELECT 1; + ALTER TABLE x RENAME TO Y; + """)] + public void RenamingOntoAViewReportsSqlitesMessage(string sql) + => AssertSameOutcome(sql); + + [TestCase(""" + PRAGMA temp.synchronous; + PRAGMA temp.synchronous=FULL; + PRAGMA temp.synchronous; + PRAGMA temp.synchronous=NORMAL; + PRAGMA temp.synchronous; + PRAGMA main.synchronous; + PRAGMA synchronous=OFF; + PRAGMA main.synchronous; + """)] + [TestCase("BEGIN; PRAGMA temp.synchronous=OFF;")] + public void TempSynchronousStaysOff(string sql) + => AssertSameOutcome(sql); + + [Test] + public void CreateTableRejectsMoreThanTwoThousandColumns() + { + AssertSameOutcome($"CREATE TABLE t({Columns(2000)}); SELECT count(*) FROM pragma_table_info('t');"); + AssertSameOutcome($"CREATE TABLE t({Columns(2001)});"); + AssertSameOutcome($"CREATE TABLE \"My T\"({Columns(2001)});"); + AssertSameOutcome($"CREATE TABLE main.t({Columns(2001)});"); + AssertSameOutcome($"CREATE TEMP TABLE t({Columns(2001)});"); + } + + [Test] + public void AlterTableAddColumnRejectsTheTwoThousandFirstColumn() + { + AssertSameOutcome($"CREATE TABLE t({Columns(1999)}); ALTER TABLE t ADD COLUMN last; SELECT count(*) FROM pragma_table_info('t');"); + AssertSameOutcome($"CREATE TABLE t({Columns(2000)}); ALTER TABLE t ADD COLUMN extra;"); + AssertSameOutcome($"CREATE TABLE t({Columns(2000)}); ALTER TABLE t ADD COLUMN c1;"); + } + + [TestCase("SELECT $;")] + [TestCase("SELECT :;")] + [TestCase("SELECT @;")] + [TestCase("SELECT 1 WHERE $ = 1;")] + [TestCase("SELECT $::;")] + [TestCase("SELECT @::;")] + [TestCase("SELECT $a(b;")] + [TestCase("SELECT $a(b c);")] + public void BadParameterNamesAreUnrecognizedTokens(string sql) + { + var managed = RunManaged(sql); + var sqlite = RunSqlite(sql); + + sqlite.Error.Should().StartWith("unrecognized token: "); + managed.Error.Should().Be(sqlite.Error); + } + + [Test] + public void TclStyleParameterNamesTokenizeLikeSqlite() + { + string[] names = ["$::ii", "$ns::var", "$arr(elem)", "@a::b::c", ":a::b", ":::g", "$a::", "$a(x,y)", "$a::::b", "$é"]; + var sql = $"SELECT {string.Join(", ", names)};"; + + using var managedDatabase = new EmbeddedDatabase(); + using var managedConnection = managedDatabase.Connect(); + using var statement = managedConnection.Prepare(sql); + statement.ParameterCount.Should().Be(names.Length); + for (var index = 0; index < names.Length; index++) + { + statement.GetParameterName(index + 1).Should().Be(names[index]); + statement.Bind(statement.GetParameterIndex(names[index]), SqlValue.Integer(index + 1)); + } + + statement.Step().Should().Be(StatementStepResult.Row); + var managedRow = Enumerable.Range(0, statement.ColumnCount) + .Select(column => statement.GetValue(column).AsInteger()) + .ToArray(); + + using var sqlite = new MsData.SqliteConnection("Data Source=:memory:;Pooling=False"); + sqlite.Open(); + using var command = sqlite.CreateCommand(); + command.CommandText = sql; + for (var index = 0; index < names.Length; index++) + command.Parameters.AddWithValue(names[index], index + 1); + using var reader = command.ExecuteReader(); + reader.Read().Should().BeTrue(); + var sqliteRow = Enumerable.Range(0, reader.FieldCount).Select(reader.GetInt64).ToArray(); + + managedRow.Should().Equal(sqliteRow); + managedRow.Should().Equal(Enumerable.Range(1, names.Length).Select(static value => (long)value)); + } + + [TestCase("INSERT INTO t1 VALUES (5), (6);")] + [TestCase("INSERT INTO t1 SELECT x + 100 FROM t1;")] + [TestCase("INSERT OR IGNORE INTO t1 VALUES (2), (7);")] + [TestCase("INSERT INTO t1 VALUES (3), (8) ON CONFLICT(x) DO UPDATE SET x = x + 10;")] + [TestCase("INSERT INTO t1 VALUES (5), (1) ON CONFLICT DO NOTHING;")] + [TestCase("REPLACE INTO t1 VALUES (1), (2);")] + [TestCase("WITH c(v) AS (SELECT 7) INSERT INTO t1 SELECT v FROM c;")] + [TestCase("UPDATE t1 SET x = x + 1 WHERE x >= 2;")] + [TestCase("UPDATE t1 SET x = x WHERE 0;")] + [TestCase("DELETE FROM t1 WHERE 0;")] + [TestCase("DELETE FROM t1;")] + [TestCase("INSERT INTO t1 VALUES (9) RETURNING x;")] + [TestCase("UPDATE t1 SET x = x + 20 WHERE x = 1 RETURNING x;")] + [TestCase("DELETE FROM t1 WHERE x = 1 RETURNING x;")] + [TestCase("INSERT INTO t1 VALUES (5); SELECT changes(), total_changes();")] + [TestCase("UPDATE t1 SET x = x + 1; SELECT changes();")] + [TestCase("PRAGMA count_changes = off; INSERT INTO t1 VALUES (5); PRAGMA count_changes;")] + public void CountChangesReturnsTheChangedRowCount(string sql) + { + const string setup = """ + CREATE TABLE t1(x UNIQUE); + CREATE TABLE log(v); + CREATE TRIGGER tr AFTER INSERT ON t1 BEGIN INSERT INTO log VALUES (new.x); UPDATE log SET v = v; END; + INSERT INTO t1 VALUES (1), (2), (3); + PRAGMA count_changes = 1; + """; + AssertSameOutcome(setup + sql, compareColumns: true); + } + + [Test] + public void CountChangesDescribesItsResultColumnAtPrepareTime() + { + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + Execute(connection, "CREATE TABLE t(x); PRAGMA count_changes = 1;"); + foreach (var (sql, column) in new[] + { + ("INSERT INTO t VALUES (1)", "rows inserted"), + ("UPDATE t SET x = 2", "rows updated"), + ("DELETE FROM t", "rows deleted"), + }) + { + using var statement = connection.Prepare(sql); + statement.GetColumnCount().Should().Be(1, sql); + statement.GetColumnName(0).Should().Be(column, sql); + } + + using var returning = connection.Prepare("INSERT INTO t VALUES (1) RETURNING x"); + returning.GetColumnCount().Should().Be(1); + returning.GetColumnName(0).Should().Be("x"); + } + + [TestCase("INSERT INTO __TURSO_INTERNAL_SEQ_S DEFAULT VALUES;", "table __TURSO_INTERNAL_SEQ_S may not be modified")] + [TestCase("INSERT INTO __TuRsO_iNtErNaL_sEq_S DEFAULT VALUES;", "table __TuRsO_iNtErNaL_sEq_S may not be modified")] + [TestCase("INSERT INTO __turso_internal_seq_s DEFAULT VALUES;", "table __turso_internal_seq_s may not be modified")] + [TestCase("UPDATE __TURSO_INTERNAL_SEQ_S SET value = 2;", "table __TURSO_INTERNAL_SEQ_S may not be modified")] + [TestCase("DELETE FROM main.__Turso_Internal_Seq_S;", "table __Turso_Internal_Seq_S may not be modified")] + public void SequenceBackingTablesRejectUserDmlRegardlessOfCase(string sql, string message) + { + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + Execute(connection, "CREATE SEQUENCE s START WITH 5;"); + + Action write = () => Execute(connection, sql); + + write.Should().Throw().Which.Message.Should().Be(message); + ReadRows(connection, "SELECT nextval('s'); SELECT nextval('s');").Should().Equal("5", "6"); + } + + private static string Columns(int count) + => string.Join(", ", Enumerable.Range(1, count).Select(static index => $"c{index}")); + + private static void AssertSameOutcome(string sql, bool compareColumns = false) + { + var managed = RunManaged(sql); + var sqlite = RunSqlite(sql); + + managed.Error.Should().Be(sqlite.Error, sql); + managed.Rows.Should().Equal(sqlite.Rows, sql); + if (compareColumns) + managed.Columns.Should().Equal(sqlite.Columns, sql); + } + + private sealed record Outcome(List Columns, List Rows, string? Error); + + private static Outcome RunManaged(string sql) + { + var columns = new List(); + var rows = new List(); + try + { + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + foreach (var statement in connection.PrepareScript(sql)) + { + using (statement) + { + while (statement.Step() == StatementStepResult.Row) + { + var values = new string[statement.ColumnCount]; + for (var column = 0; column < values.Length; column++) + { + columns.Add(statement.GetColumnName(column)); + values[column] = FormatManaged(statement.GetValue(column)); + } + + rows.Add(string.Join('|', values)); + } + } + } + + return new Outcome(columns, rows, null); + } + catch (EmbeddedSqlException exception) + { + return new Outcome(columns, rows, exception.Message); + } + } + + private static Outcome RunSqlite(string sql) + { + var columns = new List(); + var rows = new List(); + try + { + using var connection = new MsData.SqliteConnection("Data Source=:memory:;Pooling=False"); + connection.Open(); + using var command = connection.CreateCommand(); + command.CommandText = sql; + using var reader = command.ExecuteReader(); + do + { + while (reader.Read()) + { + var values = new string[reader.FieldCount]; + for (var column = 0; column < values.Length; column++) + { + columns.Add(reader.GetName(column)); + values[column] = FormatSqlite(reader, column); + } + + rows.Add(string.Join('|', values)); + } + } + while (reader.NextResult()); + + return new Outcome(columns, rows, null); + } + catch (MsData.SqliteException exception) + { + // Microsoft.Data.Sqlite decorates sqlite3_errmsg as "SQLite Error N: 'message'.". + var match = Regex.Match(exception.Message, "^SQLite Error \\d+: '(?.*)'\\.$", RegexOptions.Singleline); + return new Outcome(columns, rows, match.Success ? match.Groups["message"].Value : exception.Message); + } + } + + private static string FormatManaged(SqlValue value) => value.Kind switch + { + SqlValueKind.Null => "NULL", + SqlValueKind.Integer => value.AsInteger().ToString(CultureInfo.InvariantCulture), + SqlValueKind.Real => value.AsReal().ToString("R", CultureInfo.InvariantCulture), + SqlValueKind.Text => value.AsText(), + SqlValueKind.Blob => "x'" + Convert.ToHexString(value.AsBlob().Span) + "'", + _ => throw new InvalidOperationException($"Unknown SQL value kind {value.Kind}."), + }; + + private static string FormatSqlite(MsData.SqliteDataReader reader, int ordinal) + { + if (reader.IsDBNull(ordinal)) + return "NULL"; + + return reader.GetValue(ordinal) switch + { + long integer => integer.ToString(CultureInfo.InvariantCulture), + double real => real.ToString("R", CultureInfo.InvariantCulture), + byte[] bytes => "x'" + Convert.ToHexString(bytes) + "'", + string text => text, + var other => Convert.ToString(other, CultureInfo.InvariantCulture) ?? string.Empty, + }; + } + + private static void Execute(EmbeddedConnection connection, string sql) + { + foreach (var statement in connection.PrepareScript(sql)) + { + using (statement) + { + while (statement.Step() == StatementStepResult.Row) + { + } + } + } + } + + private static List ReadRows(EmbeddedConnection connection, string sql) + { + var rows = new List(); + foreach (var statement in connection.PrepareScript(sql)) + { + using (statement) + { + while (statement.Step() == StatementStepResult.Row) + { + var builder = new StringBuilder(); + for (var column = 0; column < statement.ColumnCount; column++) + { + if (column > 0) + builder.Append('|'); + builder.Append(FormatManaged(statement.GetValue(column))); + } + + rows.Add(builder.ToString()); + } + } + } + + return rows; + } +} From f8b4297cb8ad5972e52ded7242c2830e287cc106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 16:05:05 -0400 Subject: [PATCH 08/20] Report STRICT storage-class and generated-column violations from integrity_check Turso "report STRICT type violations from integrity_check" parity: - PRAGMA integrity_check/quick_check now checks every non-rowid-alias column of a STRICT table against its declared INT/INTEGER/REAL/TEXT/BLOB storage class ("non-TEXT value in t.b"), before its NOT NULL check, using SQLite's OP_IsType masks (NULL always passes; REAL accepts the integer storage class SQLite writes for integral reals). ANY and domain/custom types are exempt. - Stored rows' VIRTUAL generated columns are recomputed on read with affinity only, like a SQLite/Turso column read, instead of enforcing NOT NULL and the STRICT storage class at load. A row that violates them stays readable and integrity_check reports it ("NULL value in t.b", "non-INT value in t.b") rather than the database failing to open. - The sqltest harness now builds the four database/integrity_strict_*.db fixtures like upstream's generate_strict_fixture (managed write path plus a same-length first-row serial-type swap), so the parity_strict_* corpus files run instead of being skipped, and all pass. Ledger: removed the two parity_gencol_not_null_violation entries, which now pass. Co-Authored-By: Claude Opus 5.5 --- src/Ahtola.Core/EmbeddedDatabase.cs | 69 ++++++++++++++-- .../managed-sqltest-expected-failures.txt | 17 ++-- .../Sqltest/SqliteFixtureBytePatcher.cs | 55 +++++++++++++ .../SqltestIntegrityFixtureGenerator.cs | 79 ++++++++++++++++++- .../SqltestPhysicalFixtureTests.cs | 23 +++++- 5 files changed, 224 insertions(+), 19 deletions(-) diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index 08552917..53546e2c 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -6375,7 +6375,8 @@ private ExecutionResult ExecutePragmaForeignKeyCheck( /// /// Reports the SQLite integrity problems the managed catalog can actually - /// prove: declared NOT NULL and CHECK constraints that stored rows violate. + /// prove: STRICT storage-class and declared NOT NULL violations (in Turso's + /// per-column order, type first) and CHECK constraints that stored rows violate. /// /// /// Index and page-structure problems are unreachable here. The managed file @@ -6450,15 +6451,28 @@ private void AppendIntegrityRowProblems( int maxErrors, List problems) { + var rowidAliasColumnIndex = table.HasRowid ? table.RowidAliasColumnIndex : -1; for (var rowIndex = 0; rowIndex < table.Rows.Count && problems.Count < maxErrors; rowIndex++) { var row = table.Rows[rowIndex]; for (var columnIndex = 0; - columnIndex < table.ColumnDefinitions.Length && problems.Count < maxErrors; + columnIndex < table.ColumnDefinitions.Length && columnIndex < row.Length && problems.Count < maxErrors; columnIndex++) { + // Turso's integrity_check skips the INTEGER PRIMARY KEY: the rowid is its value. + if (columnIndex == rowidAliasColumnIndex) + continue; + var column = table.ColumnDefinitions[columnIndex]; - if (column.NotNull && columnIndex < row.Length && row[columnIndex].Kind == SqlValueKind.Null) + var value = row[columnIndex]; + if (table.Strict && StrictIntegrityTypeName(column) is { } typeName && !SatisfiesStrictIntegrityType(typeName, value)) + { + problems.Add($"non-{typeName} value in {tableName}.{column.Name}"); + if (problems.Count >= maxErrors) + break; + } + + if (column.NotNull && value.Kind == SqlValueKind.Null) problems.Add($"NULL value in {tableName}.{column.Name}"); } @@ -6471,6 +6485,34 @@ private void AppendIntegrityRowProblems( } } + /// + /// Turso's Column::strict_value_type: the STRICT declared types integrity_check can + /// test (ANY and custom/domain types have no storage-class constraint). + /// + private static string? StrictIntegrityTypeName(EmbeddedColumn column) + { + if (column.Domain is not null || column.IdentityType is not null || column.DeclaredType is null) + return null; + + var declared = column.DeclaredType.Trim().ToUpperInvariant(); + return declared is "INT" or "INTEGER" or "REAL" or "TEXT" or "BLOB" ? declared : null; + } + + /// + /// SQLite's OP_IsType masks for STRICT integrity checking: NULL always passes, and a + /// REAL column accepts the integer storage class SQLite uses for integral reals. + /// + private static bool SatisfiesStrictIntegrityType(string typeName, SqlValue value) + => value.Kind == SqlValueKind.Null + || typeName switch + { + "INT" or "INTEGER" => value.Kind == SqlValueKind.Integer, + "REAL" => value.Kind is SqlValueKind.Real or SqlValueKind.Integer, + "TEXT" => value.Kind == SqlValueKind.Text, + "BLOB" => value.Kind == SqlValueKind.Blob, + _ => true, + }; + private bool SatisfiesCheckConstraints( string tableName, EmbeddedTable table, @@ -13770,7 +13812,8 @@ private void ComputeGeneratedColumns( SqlValue[] parameters, QueryContext context, bool virtualOnly = false, - bool enforceNotNull = true) + bool enforceNotNull = true, + bool validateStrict = true) { if (!table.HasGeneratedColumns) return; @@ -13785,9 +13828,10 @@ private void ComputeGeneratedColumns( if (virtualOnly && column.GeneratedStored) continue; - var value = table.ApplyColumnAffinity( - column, - Evaluate(column.GenerationExpression!, parameters, source, context)); + var computed = Evaluate(column.GenerationExpression!, parameters, source, context); + var value = validateStrict + ? table.ApplyColumnAffinity(column, computed) + : table.CoerceColumnAffinity(column, computed); row[columnIndex] = value; if (enforceNotNull && column.NotNull && value.Kind == SqlValueKind.Null) { @@ -13822,6 +13866,13 @@ private static void EnforceGeneratedNotNullConstraints( } } + /// + /// Recomputes a stored row's VIRTUAL generated columns as it is read. Like SQLite's and + /// Turso's column read, this applies the declared affinity but enforces neither NOT NULL nor + /// the STRICT storage class: those are write-time constraints, and a stored row that no + /// longer satisfies them must stay readable so PRAGMA integrity_check can report it + /// ("NULL value in t.b", "non-INT value in t.b") instead of the database failing to load. + /// internal static void RecomputeVirtualGeneratedColumns( EmbeddedTable table, string tableName, @@ -13842,7 +13893,9 @@ internal static void RecomputeVirtualGeneratedColumns( [tableName] = table, }, new Dictionary(StringComparer.OrdinalIgnoreCase)), - virtualOnly: true); + virtualOnly: true, + enforceNotNull: false, + validateStrict: false); } // Materializes generated columns for a pre-existing row right after ALTER TABLE ADD COLUMN diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index 3125b828..0e1bb972 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -78,8 +78,8 @@ turso/without_rowid.sqltest::without-rowid-foreign-keys-rejected | intentional e # through the managed harness -- SqltestManagedRunner.RunVariant converts the resulting # open-time EmbeddedSqlException/InvalidDataException into a plain SqltestOutcome instead of # letting it escape as an unhandled test exception, so each case is compared and recorded like -# any other. All 18 cases below are real observed engine behavior: -# EmbeddedFileStore.Load() validates b-tree/index/allocation-map/generated-column structural +# any other. All 16 cases below are real observed engine behavior: +# EmbeddedFileStore.Load() validates b-tree/index/allocation-map structural # integrity eagerly at file-open time, where Turso/SQLite open the file unconditionally and # defer to PRAGMA integrity_check/quick_check to report the same defect as row output (the # same class of pre-existing divergence already exercised by @@ -87,11 +87,12 @@ turso/without_rowid.sqltest::without-rowid-foreign-keys-rejected | intentional e # The separate legacy-view loader gap is closed: malformed stored views no longer prevent # unrelated operations, and resolving a broken view for queries or trigger DML fails closed. # The other -# 3 files this generator builds (parity_check_constraint, parity_quick_check_constraint, -# parity_not_null_violation) are NOT listed here: ordinary column CHECK/NOT NULL values are not -# re-validated on load, only generated-column values are recomputed (and their NOT NULL -# rechecked), so those 3 open successfully and PRAGMA integrity_check/quick_check see the -# corruption exactly as the pinned corpus expects. +# 8 files this generator builds (parity_check_constraint, parity_quick_check_constraint, +# parity_not_null_violation, parity_gencol_not_null_violation and the four parity_strict_* +# files) are NOT listed here: stored values are not re-validated on load (VIRTUAL generated +# columns are recomputed with affinity only, like a SQLite column read, without NOT NULL or +# STRICT enforcement), so those files open successfully and PRAGMA integrity_check/quick_check +# report the NULL, STRICT storage-class and CHECK violations exactly as the pinned corpus expects. integrity_check/parity_corrupt_index.sqltest::integrity-check-corrupt-missing-index-entry | database variant 1/1 (database/integrity_missing_index_entry.db): database failed to open: Stored index 'idx_b' is not a valid supported SQLite index b-tree. integrity_check/parity_corrupt_index.sqltest::quick-check-corrupt-missing-index-entry | database variant 1/1 (database/integrity_missing_index_entry.db): database failed to open: Stored index 'idx_b' is not a valid supported SQLite index b-tree. integrity_check/parity_corrupt_expression_index.sqltest::integrity-check-corrupt-missing-expression-index-entry | database variant 1/1 (database/integrity_missing_expression_index_entry.db): database failed to open: Stored index 'idx_expr' is not a valid supported SQLite index b-tree. @@ -102,8 +103,6 @@ integrity_check/parity_freelist_count_mismatch.sqltest::integrity-check-freelist integrity_check/parity_freelist_count_mismatch.sqltest::quick-check-freelist-count-mismatch | open-time validation: SQLite freelist header declares 202 pages but its trunks contain 201. integrity_check/parity_freelist_trunk_corrupt.sqltest::integrity-check-freelist-trunk-corrupt | open-time validation: SQLite freelist trunk page 2 declares 1023 leaves, exceeding capacity 1022. integrity_check/parity_freelist_trunk_corrupt.sqltest::quick-check-freelist-trunk-corrupt | open-time validation: SQLite freelist trunk page 2 declares 1023 leaves, exceeding capacity 1022. -integrity_check/parity_gencol_not_null_violation.sqltest::integrity-check-gencol-not-null-violation | database variant 1/1 (database/integrity_gencol_not_null_violation.db): database failed to open: NOT NULL constraint failed: t.b -integrity_check/parity_gencol_not_null_violation.sqltest::quick-check-gencol-not-null-violation | database variant 1/1 (database/integrity_gencol_not_null_violation.db): database failed to open: NOT NULL constraint failed: t.b integrity_check/parity_missing_unique_index.sqltest::integrity-check-missing-unique-index-entry | database variant 1/1 (database/integrity_missing_unique_index_entry.db): database failed to open: Stored index 'idx_u_missing' does not match table 't' at record 1. integrity_check/parity_missing_unique_index.sqltest::quick-check-missing-unique-index-entry | database variant 1/1 (database/integrity_missing_unique_index_entry.db): database failed to open: Stored index 'idx_u_missing' does not match table 't' at record 1. integrity_check/parity_non_unique_index.sqltest::integrity-check-non-unique-index-entry | database variant 1/1 (database/integrity_non_unique_index_entry.db): database failed to open: Stored unique index 'idx_u' on table 't' contains duplicate non-NULL keys. diff --git a/src/Ahtola.Tests/Sqltest/SqliteFixtureBytePatcher.cs b/src/Ahtola.Tests/Sqltest/SqliteFixtureBytePatcher.cs index 6ea98217..0ce0785b 100644 --- a/src/Ahtola.Tests/Sqltest/SqliteFixtureBytePatcher.cs +++ b/src/Ahtola.Tests/Sqltest/SqliteFixtureBytePatcher.cs @@ -283,6 +283,61 @@ public static void SetSecondTableColumnI8InFirstRow(byte[] bytes, int pageSize, /// an empty TEXT value (serial type 13, zero body bytes) so swapping in NULL (serial type /// 0, also zero body bytes) does not disturb the record's payload layout. /// + /// + /// Replaces the serial type of one column of the table's first row with another that stores + /// the same number of body bytes (mirrors patch_first_row_serial_type), so the rest of + /// the record keeps its place while the value's storage class changes. + /// + public static void PatchFirstRowSerialType( + byte[] bytes, + int pageSize, + int rootPage, + int columnIndex, + byte expectedSerialType, + byte newSerialType) + { + if (SerialTypePayloadLength(expectedSerialType) != SerialTypePayloadLength(newSerialType)) + { + throw new InvalidOperationException( + $"serial types {expectedSerialType} and {newSerialType} do not store the same number of bytes"); + } + + var pageStart = (rootPage - 1) * pageSize; + ValidatePageBounds(bytes, pageSize, rootPage, pageStart, 8); + if (ReadPageHeaderCellCount(bytes, pageStart) < 1) + throw new InvalidOperationException($"cannot patch table row on page {rootPage}: no cells"); + + var cellStart = pageStart + ReadCellPointer(bytes, pageStart + 8); + if (cellStart >= bytes.Length) + throw new InvalidOperationException($"cell pointer out of bounds on page {rootPage}"); + + var (_, payloadVarintLength) = ParseVarint(bytes, cellStart); + var (_, rowidVarintLength) = ParseVarint(bytes, cellStart + payloadVarintLength); + var payloadStart = cellStart + payloadVarintLength + rowidVarintLength; + var (headerSize, headerSizeVarintLength) = ParseVarint(bytes, payloadStart); + var headerEnd = payloadStart + (int)headerSize; + if (headerEnd > bytes.Length) + throw new InvalidOperationException($"record header out of bounds on page {rootPage}"); + + var serialOffset = payloadStart + headerSizeVarintLength; + for (var column = 0; column < columnIndex; column++) + { + if (serialOffset >= headerEnd) + throw new InvalidOperationException($"record does not contain column {columnIndex} on page {rootPage}"); + serialOffset += ParseVarint(bytes, serialOffset).Length; + } + + if (serialOffset >= headerEnd) + throw new InvalidOperationException($"record does not contain column {columnIndex} on page {rootPage}"); + if (bytes[serialOffset] != expectedSerialType) + { + throw new InvalidOperationException( + $"unexpected serial type {bytes[serialOffset]} for fixture row column {columnIndex}; expected {expectedSerialType}"); + } + + bytes[serialOffset] = newSerialType; + } + public static void SetSecondTableColumnToNullInFirstRow(byte[] bytes, int pageSize, int rootPage) { var pageStart = (rootPage - 1) * pageSize; diff --git a/src/Ahtola.Tests/Sqltest/SqltestIntegrityFixtureGenerator.cs b/src/Ahtola.Tests/Sqltest/SqltestIntegrityFixtureGenerator.cs index eceec7d5..df661bb1 100644 --- a/src/Ahtola.Tests/Sqltest/SqltestIntegrityFixtureGenerator.cs +++ b/src/Ahtola.Tests/Sqltest/SqltestIntegrityFixtureGenerator.cs @@ -3,7 +3,7 @@ namespace Ahtola.Tests.Sqltest; /// -/// Builds the 12 integrity_check/parity_* corruption fixtures the pinned corpus +/// Builds the 16 integrity_check/parity_* corruption fixtures the pinned corpus /// expects at database/integrity_*.db. Each fixture is produced by running ordinary /// setup SQL through the managed engine's real write path (so the physical page layout is /// whatever Ahtola actually produces, not a synthetic mock), then patching a handful of @@ -65,6 +65,83 @@ public static void RegisterInto(IDictionary> generators) generators["database/integrity_freelist_trunk_corrupt.db"] = GenerateFreelistTrunkCorrupt; generators["database/integrity_overflow_list_length_mismatch.db"] = GenerateOverflowListLengthMismatch; generators["database/integrity_gencol_not_null_violation.db"] = GenerateGencolNotNullViolation; + + // Mirrors generate_strict_*_fixture: build a STRICT table through the real write path, + // then swap one first-row serial type for another of the same body length. + generators["database/integrity_strict_type_violation.db"] = path => GenerateStrictFixture( + path, + """ + PRAGMA page_size=4096; + CREATE TABLE t(a ANY, b TEXT) STRICT; + INSERT INTO t VALUES('x', 'A'); + """, + columnIndex: 1, + expectedSerialType: SerialTypeTextOneByte, + newSerialType: SerialTypeInt8); + generators["database/integrity_strict_gencol_type_violation.db"] = path => GenerateStrictFixture( + path, + """ + PRAGMA page_size=4096; + CREATE TABLE t(a ANY, b INT AS (a) VIRTUAL) STRICT; + INSERT INTO t(a) VALUES(0); + """, + columnIndex: 0, + expectedSerialType: SerialTypeIntZero, + newSerialType: SerialTypeTextEmpty); + generators["database/integrity_strict_not_null_violation.db"] = path => GenerateStrictFixture( + path, + """ + PRAGMA page_size=4096; + CREATE TABLE t(a ANY, b TEXT NOT NULL) STRICT; + INSERT INTO t VALUES('x', ''); + """, + columnIndex: 1, + expectedSerialType: SerialTypeTextEmpty, + newSerialType: SerialTypeNull); + generators["database/integrity_strict_real_integer_serial.db"] = path => GenerateStrictFixture( + path, + """ + PRAGMA page_size=4096; + CREATE TABLE t(a ANY, r REAL) STRICT; + INSERT INTO t VALUES('x', 1.5); + """, + columnIndex: 1, + expectedSerialType: SerialTypeFloat64, + newSerialType: SerialTypeInt64); + } + + private const byte SerialTypeNull = 0; + private const byte SerialTypeInt8 = 1; + private const byte SerialTypeInt64 = 6; + private const byte SerialTypeFloat64 = 7; + private const byte SerialTypeIntZero = 8; + private const byte SerialTypeTextEmpty = 13; + private const byte SerialTypeTextOneByte = 15; + + /// + /// Mirrors generate_strict_fixture: Turso (and SQLite) never write these storage + /// classes into a STRICT column through SQL, so the fixture changes the file directly. + /// + private static void GenerateStrictFixture( + string path, + string setupSql, + int columnIndex, + byte expectedSerialType, + byte newSerialType) + { + Build(path, connection => ExecuteScript(connection, setupSql)); + + var bytes = File.ReadAllBytes(path); + var pageSize = SqliteFixtureBytePatcher.ReadDbHeaderPageSize(bytes); + var rootPage = SqliteFixtureBytePatcher.FindObjectRootPage(bytes, pageSize, "t"); + SqliteFixtureBytePatcher.PatchFirstRowSerialType( + bytes, + pageSize, + rootPage, + columnIndex, + expectedSerialType, + newSerialType); + File.WriteAllBytes(path, bytes); } /// diff --git a/src/Ahtola.Tests/SqltestPhysicalFixtureTests.cs b/src/Ahtola.Tests/SqltestPhysicalFixtureTests.cs index 5e80f145..14fd58a2 100644 --- a/src/Ahtola.Tests/SqltestPhysicalFixtureTests.cs +++ b/src/Ahtola.Tests/SqltestPhysicalFixtureTests.cs @@ -14,7 +14,6 @@ public class SqltestPhysicalFixtureTests [TestCase("integrity_check/parity_freelist_count_mismatch.sqltest", "freelist header declares")] [TestCase("integrity_check/parity_freelist_trunk_corrupt.sqltest", "exceeding capacity")] [TestCase("integrity_check/parity_overflow_list_length_mismatch.sqltest", "overflow chain ends")] - [TestCase("integrity_check/parity_gencol_not_null_violation.sqltest", "NOT NULL constraint failed", false)] public void FixtureConstructionSucceedsBeforeExpectedEngineRejection( string relativePath, string diagnostic, bool duringOpen = true) { @@ -35,4 +34,26 @@ public void FixtureConstructionSucceedsBeforeExpectedEngineRejection( ? "database failed to open:" : "expected success but got error:"); } + + // Turso "report STRICT type violations from integrity_check": each fixture is a real + // managed-written STRICT (or generated-column) table with one first-row serial type + // swapped, and PRAGMA integrity_check/quick_check must report exactly upstream's rows. + [TestCase("integrity_check/parity_strict_type_violation.sqltest", "non-TEXT value in t.b")] + [TestCase("integrity_check/parity_strict_gencol_type_violation.sqltest", "non-INT value in t.b")] + [TestCase("integrity_check/parity_strict_not_null_violation.sqltest", "NULL value in t.b")] + [TestCase("integrity_check/parity_strict_real_integer_serial.sqltest", "ok")] + [TestCase("integrity_check/parity_gencol_not_null_violation.sqltest", "NULL value in t.b")] + public void StoredValueViolationsAreReportedByIntegrityCheck(string relativePath, string expected) + { + var discovered = SqltestCorpus.Cases.Where(candidate => candidate.RelativePath == relativePath).ToList(); + discovered.Should().NotBeEmpty(); + discovered.Should().OnlyContain(candidate => candidate.Status == SqltestCaseStatus.Runnable); + var file = SqltestCorpus.LoadFile(relativePath, discovered[0].FullPath); + foreach (var candidate in discovered) + { + var test = file.Tests.Single(entry => entry.Name == candidate.TestName); + var outcome = SqltestManagedRunner.Run(file, test); + outcome.Matched.Should().BeTrue($"{candidate.Id} expects '{expected}': {outcome.Detail}"); + } + } } From 76e716716d7acde18e10d4a0c81a4877f0701df9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 16:09:51 -0400 Subject: [PATCH 09/20] Let WAL readers pin read-marks 1..4 while a checkpoint holds read-mark 0 Turso 74eb80c83 / SQLite walTryBeginRead: a reader whose WAL is fully backfilled normally takes read-mark 0 and ignores the WAL, but a checkpoint holds read-mark 0 exclusively for as long as it runs. Instead of retrying until the checkpoint finishes, SqliteWalReadSnapshotCoordinator now pins the committed boundary with an existing mark already at mxFrame or an idle mark it advances to mxFrame. Every frame is already in the database file, so the mark only keeps a later checkpoint from restarting the WAL underneath the reader; the usual confirmation (mark value, boundary, commit frame) still applies. An empty WAL has no commit frame to pin and still waits. Co-Authored-By: Claude Opus 5.5 --- docs/wal-interoperability-contract.md | 5 +- .../SqliteWalReadSnapshotCoordinator.cs | 53 +++++++++++++++++-- .../SqliteWalReadSnapshotCoordinatorTests.cs | 47 ++++++++++++++++ 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/docs/wal-interoperability-contract.md b/docs/wal-interoperability-contract.md index b9223b44..fdbb175b 100644 --- a/docs/wal-interoperability-contract.md +++ b/docs/wal-interoperability-contract.md @@ -504,7 +504,10 @@ readers coexist on the same mark. validated WAL-index accessor, and shared byte-range lease primitive into a detached read-snapshot coordinator. It opens only existing `-wal` and `-shm` artifacts, validates a stable header against an independent WAL checksum scan, -uses `WAL_READ_LOCK(0)` for an already-backfilled database-only view, otherwise +uses `WAL_READ_LOCK(0)` for an already-backfilled database-only view (while a +checkpoint holds read-mark 0 it instead pins that backfilled boundary with a +current or newly advanced mark 1..4, as SQLite's `walTryBeginRead` and Turso +`74eb80c83` do, rather than waiting; an empty WAL still waits), otherwise shares an existing current mark, advances any exclusively acquired idle mark to the current committed boundary, or falls back to the greatest usable existing mark. An advanced mark is exclusively held while changed and then reacquired diff --git a/src/Ahtola.Core/Storage/SqliteWalReadSnapshotCoordinator.cs b/src/Ahtola.Core/Storage/SqliteWalReadSnapshotCoordinator.cs index be7515b7..e1389cd4 100644 --- a/src/Ahtola.Core/Storage/SqliteWalReadSnapshotCoordinator.cs +++ b/src/Ahtola.Core/Storage/SqliteWalReadSnapshotCoordinator.cs @@ -259,6 +259,18 @@ private SqliteWalReadSnapshot BeginReadCore(TimeSpan timeout, CancellationToken readMarkLease?.Dispose(); } + + // SQLite's walTryBeginRead and Turso 74eb80c83: a checkpoint holds read-mark 0 + // exclusively for as long as it runs, and a reader must not wait for it. Every + // frame this reader could want is already in the database file, so it pins the + // committed boundary with one of read-marks 1..4 instead; that mark only keeps + // a later checkpoint from restarting the WAL underneath it. An empty WAL has no + // commit frame to pin, so it still waits for read-mark 0. + if (region.Header.MaximumFrame != 0 + && TryPinBackfilledBoundary(region, cancellationToken, out var backfilled)) + { + return backfilled; + } } else { @@ -299,11 +311,40 @@ private SqliteWalReadSnapshot BeginReadCore(TimeSpan timeout, CancellationToken } } + /// + /// Pins a fully backfilled boundary with read-marks 1..4 while read-mark 0 is busy: share a + /// mark already at mxFrame, or advance an idle one to it. + /// + private bool TryPinBackfilledBoundary( + SqliteWalIndexHeaderRegion region, + CancellationToken cancellationToken, + out SqliteWalReadSnapshot snapshot) + { + foreach (var candidate in SelectExistingReadMarks(region)) + { + if (candidate.Frame != region.Header.MaximumFrame) + break; + + if (TryAcquireExistingReadMark( + candidate.Index, + candidate.Frame, + cancellationToken, + out snapshot, + allowBackfilled: true)) + { + return true; + } + } + + return TryAdvanceReadMark(cancellationToken, out snapshot, allowBackfilled: true); + } + private bool TryAcquireExistingReadMark( int readMarkIndex, uint maximumFrame, CancellationToken cancellationToken, - out SqliteWalReadSnapshot snapshot) + out SqliteWalReadSnapshot snapshot, + bool allowBackfilled = false) { snapshot = null!; if (!_locks.TryAcquireShared(GetReadMarkLockOffset(readMarkIndex), length: 1, out var readMarkLease)) @@ -313,7 +354,7 @@ private bool TryAcquireExistingReadMark( { cancellationToken.ThrowIfCancellationRequested(); var confirmation = _index.ReadValidatedHeader(_wal); - if (IsDatabaseOnlySnapshot(confirmation) + if ((!allowBackfilled && IsDatabaseOnlySnapshot(confirmation)) || confirmation.Header.MaximumFrame < maximumFrame || confirmation.CheckpointInfo.GetReadMark(readMarkIndex) != maximumFrame) { @@ -338,7 +379,8 @@ private bool TryAcquireExistingReadMark( private bool TryAdvanceReadMark( CancellationToken cancellationToken, - out SqliteWalReadSnapshot snapshot) + out SqliteWalReadSnapshot snapshot, + bool allowBackfilled = false) { snapshot = null!; for (var readMarkIndex = 1; readMarkIndex < ReadMarkCount; readMarkIndex++) @@ -350,7 +392,7 @@ private bool TryAdvanceReadMark( { cancellationToken.ThrowIfCancellationRequested(); var region = _index.ReadValidatedHeader(_wal); - if (IsDatabaseOnlySnapshot(region)) + if (region.Header.MaximumFrame == 0 || (!allowBackfilled && IsDatabaseOnlySnapshot(region))) return false; _index.PublishReadMark(readMarkIndex, region.Header.MaximumFrame); @@ -365,7 +407,8 @@ private bool TryAdvanceReadMark( readMarkIndex, maximumFrame, cancellationToken, - out snapshot)) + out snapshot, + allowBackfilled)) { return true; } diff --git a/src/Ahtola.Tests/SqliteWalReadSnapshotCoordinatorTests.cs b/src/Ahtola.Tests/SqliteWalReadSnapshotCoordinatorTests.cs index 90136ad6..9aaccd47 100644 --- a/src/Ahtola.Tests/SqliteWalReadSnapshotCoordinatorTests.cs +++ b/src/Ahtola.Tests/SqliteWalReadSnapshotCoordinatorTests.cs @@ -86,6 +86,53 @@ public void SnapshotRepurposesAnUnlockedStaleReadMarkAtTheCurrentBoundary() snapshot.MaximumFrame.Should().Be(region.Header.MaximumFrame); } + // Turso 74eb80c83 / SQLite walTryBeginRead: a checkpoint holds read-mark 0 exclusively + // while it runs, and a reader of a fully backfilled WAL must not wait for it. It pins the + // committed boundary with read-marks 1..4 instead, which the checkpoint does not touch. + [Test] + [NonParallelizable] + public void BackfilledReaderPinsAnotherReadMarkWhileACheckpointHoldsReadMarkZero() + { + RequireSnapshotSupport(); + using var artifact = SqliteWalArtifact.Create(); + using (var checkpointer = new SqliteConnection($"Data Source={artifact.DatabasePath};Mode=ReadWrite;Pooling=False")) + { + checkpointer.Open(); + Execute(checkpointer, "PRAGMA wal_checkpoint(PASSIVE);"); + } + + using var coordinator = SqliteWalReadSnapshotCoordinator.Open(artifact.DatabasePath); + uint backfilledFrame; + using (var wal = OpenWalCopy(artifact.DatabasePath)) + { + backfilledFrame = checked((uint)wal.ScanRecovery().LastCommittedFrameNumber); + backfilledFrame.Should().BeGreaterThan(0, "a PASSIVE checkpoint keeps the backfilled frames"); + } + + using (var checkpointHoldsReadMarkZero = new CrossProcessReadMarkProbe( + artifact.WorkDirectory, + artifact.DatabasePath + "-shm", + readMarkIndex: 0)) + { + checkpointHoldsReadMarkZero.Result.Should().Be("acquired"); + + using var pinned = coordinator.BeginRead(TimeSpan.Zero); + pinned.ReadMarkIndex.Should().BeInRange(1, SqliteWalIndexCheckpointInfo.ReadMarkCount - 1); + pinned.MaximumFrame.Should().Be(backfilledFrame); + pinned.ReadFrame(pinned.MaximumFrame).Header.IsCommit.Should().BeTrue(); + + // A second backfilled reader shares the mark the first one published. + using var shared = coordinator.BeginRead(TimeSpan.Zero); + shared.ReadMarkIndex.Should().Be(pinned.ReadMarkIndex); + shared.MaximumFrame.Should().Be(backfilledFrame); + } + + // Once the checkpoint lets read-mark 0 go, a backfilled reader ignores the WAL again. + using var databaseOnly = coordinator.BeginRead(TimeSpan.Zero); + databaseOnly.ReadMarkIndex.Should().Be(0); + databaseOnly.MaximumFrame.Should().Be(0); + } + [Test] [NonParallelizable] public void SnapshotIgnoresFramesAppendedByASeparateSqliteWriterProcess() From 2baa7a65e8d4983163f8ef84ac45238bd396386b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 16:37:16 -0400 Subject: [PATCH 10/20] Close v0.8.1 join/unnest correctness gaps and partial-index/EQP planner cases Correctness: - Parser: ON/USING join constraints on comma joins and CROSS JOIN (`a, b ON ...`, `a CROSS JOIN b USING (...)`, NATURAL CROSS JOIN), matching SQLite's joinop/on_using grammar. JoinTableSource carries a Cross flag; CROSS JOIN is a join-order barrier (Turso JoinInfo::no_reorder) and never flips its hash-build orientation, so its left operand stays the outer loop. - FULL OUTER JOIN with correlated subqueries referencing the joined tables is no longer rejected: Turso v0.8.1 unnests these after the FULL JOIN, and the managed evaluator already evaluates them per joined row, null-padded rows included. - The transient equality hash probe no longer prunes rows a SQLite scan would still evaluate: an equality is only probeable while no conjunct before it can raise. A positive single-table EXISTS keeps pruning, matching SQLite's existsToJoin automatic-index semantics. Fixes NOT EXISTS hiding 'malformed JSON' errors. Planner/EQP (from the executed plan): - Partial-index implication accepts either disjunct of an OR predicate (Turso query_term_implies_predicate), and a covering partial index can be scanned without a key constraint. - A correlated IN the semi-join rewrite declines is described as its real per-row CORRELATED LIST SUBQUERY scan. - FORMAT=JSON reports the ported Turso scan estimate for an unfiltered table scan (sqlite_stat1 row count or the 1,000,000-row default). Ledger: 17 v0.8.1 entries closed; 5 now-parsing join/memory entries record their new plan-pattern differences. Co-Authored-By: Claude Opus 5.5 --- ...eddedDatabase.ExplainQueryPlanDescriber.cs | 110 +++++++++- .../EmbeddedDatabase.JoinOrderRewrites.cs | 5 +- src/Ahtola.Core/EmbeddedDatabase.cs | 191 +++++++++--------- src/Ahtola.Core/IndexSemantics.cs | 13 +- src/Ahtola.Core/Parsing/SqlAst.cs | 5 +- src/Ahtola.Core/Parsing/SqlParser.cs | 57 +++--- .../managed-sqltest-expected-failures.txt | 27 +-- .../ExplainQueryPlanFormatJsonTests.cs | 41 ++++ src/Ahtola.Tests/ExplainQueryPlanTests.cs | 62 ++++++ src/Ahtola.Tests/JoinOrderOptimizerTests.cs | 55 +++++ src/Ahtola.Tests/SubqueryRewriteTests.cs | 82 ++++++++ 11 files changed, 501 insertions(+), 147 deletions(-) diff --git a/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs b/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs index 0bdb38e7..bcbeeb96 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs @@ -35,6 +35,14 @@ namespace Ahtola.Core; /// public sealed partial class EmbeddedDatabase { + /// A FROM name bound to an ordinary base table rather than a CTE, view or module. + private static bool IsPlainEqpBaseTable(NamedTableSource source, QueryContext context) + => !IsSchemaTable(source.Name) + && context.Tables.ContainsKey(source.Name) + && !context.CommonTableExpressions.ContainsKey(source.Name) + && context.Views?.ContainsKey(source.Name) != true + && context.VirtualTables?.ContainsKey(source.Name) != true; + private bool TryDescribeCorrelatedAggregateSubqueryPlan( SelectStatement statement, SqlValue[] parameters, @@ -281,6 +289,69 @@ private bool TryDescribeCorrelatedAggregateSubqueryPlan( return true; } + // A correlated IN the semi-join rewrite declined - an operand or inner filter that can + // raise, or a link that is not a plain equality - stays in WHERE, so the evaluator scans + // the outer table and re-runs the list for every outer row (unnest.rs keeps the same + // failing shapes as correlated subqueries). Only the plain-scan shape is described: the + // outer table has no usable index plan, and the inner WHERE has no index plan and no + // equality the transient hash probe could narrow the per-row scan with. + if (statement is + { + Source: NamedTableSource listScanOuter, + Where: InSubqueryExpression + { + Query: SelectStatement + { + Source: NamedTableSource listScanInner, + GroupBy.Count: 0, + Having: null, + Limit: null, + } listScanQuery, + } listScanIn, + GroupBy.Count: 0, + Having: null, + OrderBy.Count: 0, + Limit: null, + } + && rewritten is { Source: NamedTableSource, Where: InSubqueryExpression } + && listScanOuter.IndexDirective is null + && listScanInner.IndexDirective is null + && IsPlainEqpBaseTable(listScanOuter, context) + && IsPlainEqpBaseTable(listScanInner, context) + && !statement.Projections.Any(projection => + ContainsAggregate(projection.Expression) || ContainsSubqueryExpression(projection.Expression)) + && !ContainsSubqueryExpression(listScanIn.Value) + && !listScanQuery.Projections.Any(projection => ContainsSubqueryExpression(projection.Expression)) + && !ContainsSubqueryExpression(listScanQuery.Where) + && QueryReferencesFromNames( + listScanQuery, + new HashSet(StringComparer.OrdinalIgnoreCase) + { + listScanOuter.Alias ?? listScanOuter.Name, + }) + && (listScanQuery.Where is null + || !IndexExpressionSemantics.SplitConjuncts(listScanQuery.Where).Any(static conjunct => + conjunct is BinaryExpression { Operator: BinaryOperator.Equal or BinaryOperator.Is })) + && TryPlanManagedIndexScan(statement, context) is null + && TryPlanManagedIndexScan(listScanQuery, context) is null) + { + result = new ExecutionResult( + ExplainQueryPlanColumns(), + [ + PlanRow(1, 0, $"SCAN {listScanOuter.Name}" + (listScanOuter.Alias is null ? string.Empty : $" AS {listScanOuter.Alias}")), + PlanRow(6, 0, "CORRELATED LIST SUBQUERY 1"), + PlanRow(8, 6, $"SCAN {listScanInner.Name}" + (listScanInner.Alias is null ? string.Empty : $" AS {listScanInner.Alias}")), + ], + 0); + ops = + [ + new EqpJsonScanOp(listScanOuter.Name, listScanOuter.Alias, IndexName: null, Covering: false), + new EqpJsonListSubqueryOp(1, Correlated: true), + new EqpJsonScanOp(listScanInner.Name, listScanInner.Alias, IndexName: null, Covering: false), + ]; + return true; + } + // Group-first aggregate decorrelation materializes the aggregate once per correlation // key, then left-joins that grouped result to the outer scan. The evaluator executes // this derived source as a grouped table, so expose its real source and grouping work @@ -758,7 +829,8 @@ internal sealed record EqpJsonScanOp( string? IndexName, bool Covering, string? Join = null, - bool Ephemeral = false) : EqpJsonOp + bool Ephemeral = false, + EqpJsonEstimate? Estimate = null) : EqpJsonOp { public override string ToJson() { @@ -766,10 +838,46 @@ public override string ToJson() AppendTableFields(json, Table, Alias, Join); json.Append(",\"source\":\"table\""); AppendIndexField(json, IndexName, Covering, Ephemeral); + Estimate?.AppendTo(json); return json.Append('}').ToString(); } } +/// +/// The cost-model estimate Turso attaches to a table access node +/// (core/translate/eqp.rs estimate): rows entering the loop, rows produced per +/// input row, their product, the access method's own cost and the running plan cost. +/// +internal sealed record EqpJsonEstimate( + double InputRows, + double RowsPerInput, + double OutputRows, + double AccessCost, + double TotalCost) +{ + public void AppendTo(System.Text.StringBuilder json) + => json.Append(",\"estimate\":{\"input_rows\":").Append(Format(InputRows)) + .Append(",\"rows_per_input\":").Append(Format(RowsPerInput)) + .Append(",\"output_rows\":").Append(Format(OutputRows)) + .Append(",\"access_cost\":").Append(Format(AccessCost)) + .Append(",\"total_cost\":").Append(Format(TotalCost)) + .Append('}'); + + // Rust's f64 Display, which Turso's writer uses: the shortest round-trip digits, integral + // values without a fraction, and never an exponent. + private static string Format(double value) + { + var culture = System.Globalization.CultureInfo.InvariantCulture; + if (value == Math.Floor(value) && Math.Abs(value) < 9e18) + return ((long)value).ToString(culture); + + var text = value.ToString("R", culture); + return text.Contains('E', StringComparison.Ordinal) && Math.Abs(value) < 7.9e28 + ? ((decimal)value).ToString(culture) + : text; + } +} + /// A scan through the selected virtual-table module, rather than a base b-tree. internal sealed record EqpJsonVirtualTableScanOp(string Table, string? Alias) : EqpJsonOp { diff --git a/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs b/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs index eca90616..3f00d99d 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs @@ -186,10 +186,11 @@ private static bool JoinOrderContainsUnsupportedKind(TableSource? source) /// A join node whose members may be freely permuted: plain INNER, no coalesced columns. /// NATURAL and USING are excluded because BuildJoinPairs/JoinPairMatches resolve /// their columns from the specific left/right pair, so changing which tables are adjacent - /// changes what the join means. + /// changes what the join means. CROSS JOIN is excluded because SQLite and Turso keep its + /// operands in FROM order (Turso JoinInfo::is_ordering_constrained, plan.rs). /// private static bool IsReorderableInnerJoin(JoinTableSource join) - => join.Kind == JoinKind.Inner && join.UsingColumns is null && !join.Natural; + => join.Kind == JoinKind.Inner && join.UsingColumns is null && !join.Natural && !join.Cross; private bool TryRewriteJoinOrderNode( TableSource node, diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index b1e16831..5c32be30 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -896,7 +896,8 @@ internal sealed record QueryContext( ManagedSchemaRowSet? StagedSchemaRows = null, ManagedSequenceSession? SequenceSession = null, // Connection-scoped: several connections may share the same EmbeddedDatabase. - Func? DescribeJournalMode = null) + Func? DescribeJournalMode = null, + SelectStatement? ExistsJoinBody = null) { /// /// Per-statement cache of opened managed index-method scan state. Derived contexts created @@ -22470,8 +22471,10 @@ SourceRow CombinedRow(VdbeJoinRow row) => CreateCompiledJoinSourceRow( { hashBuildRight = chosen; } - else + else if (!join.Cross) { + // CROSS JOIN pins its left operand as the outer loop (SQLite JT_CROSS), so it + // keeps the probe-left orientation whose output follows the left rows. var leftEstimate = EstimateJoinNodeRows(left.Plan, context); var rightEstimate = EstimateJoinNodeRows(right.Plan, context); // Only flip when both sides have real sqlite_stat1 estimates (not the unknown sentinel). @@ -29069,10 +29072,31 @@ when context.Tables.ContainsKey(source.Name) && context.Views?.ContainsKey(source.Name) != true && context.VirtualTables?.ContainsKey(source.Name) != true => ( $"SCAN {source.Name}" + (source.Alias is null ? string.Empty : $" AS {source.Alias}"), - new EqpJsonScanOp(source.Name, source.Alias, IndexName: null, Covering: false)), + new EqpJsonScanOp( + source.Name, + source.Alias, + IndexName: null, + Covering: false, + Estimate: select.Where is null ? EstimateUnfilteredTableScan(source.Name, context) : null)), _ => null, }; + /// + /// The estimate of a lone, unfiltered full table scan: one input row, every table row out, + /// priced by the ported Turso scan formula (optimizer/cost.rs estimate_scan_cost). The row + /// count is the planner's current sqlite_stat1 figure, or Turso's 1,000,000-row + /// default for an unanalyzed table. A filtered scan needs Turso's per-constraint selectivity + /// model, which is not ported, so it reports no estimate rather than an invented one. + /// + private static EqpJsonEstimate EstimateUnfilteredTableScan(string tableName, QueryContext context) + { + var rows = TryGetSqliteStat1TableRowCount(context, tableName, out var analyzed) + ? analyzed + : JoinCostParams.RowsPerTableFallback; + var cost = JoinCostModel.EstimateFullScanCost(rows, scanCount: 1.0); + return new EqpJsonEstimate(1, rows, rows, cost, cost); + } + internal static string JsonEscape(string value) { var escaped = new System.Text.StringBuilder(value.Length + 2); @@ -30927,7 +30951,14 @@ private static int CountTableSourceLeaves(TableSource? source) // Plain indexes also qualify when ORDER BY matches the index key order or the // WHERE probes the leading term (SEARCH). var aggregateOrGroupOrdered = AggregateOrGroupUsesIndex(statement, table, index); - if (!search && !ordered && !scansOnlyPartialPredicate && !aggregateOrGroupOrdered) + // A covering partial index whose predicate the WHERE implies holds every result row + // in fewer entries than the table, so scanning it without a key constraint and + // re-checking the WHERE beats a table scan (Turso discounts a partial index to at + // most half the table rows, optimizer/access_method.rs choose_best_btree_candidate). + var coveringPartialScan = index.IsPartial + && statement.OrderBy.Count == 0 + && IndexCoversSelect(statement, table, index); + if (!search && !ordered && !scansOnlyPartialPredicate && !aggregateOrGroupOrdered && !coveringPartialScan) continue; var plan = new ManagedIndexScanPlan(source, table, index, search, searchConstraint, reverse); @@ -32606,7 +32637,8 @@ private ExecutionResult ExecuteSelect( statement.Where, parameters, context, - outerRow) + outerRow, + preserveErrors: !Equals(context.ExistsJoinBody, statement)) ?? (statement.Source is JoinTableSource joinSource && statement.Where is not null ? GetJoinRowsWithPredicatePushdown( joinSource, @@ -36937,7 +36969,8 @@ private SourceData GetSourceRowsWithTransientIndex( Expression? predicate, SqlValue[] parameters, QueryContext context, - SourceRow? outerRow) + SourceRow? outerRow, + bool preserveErrors = true) { if (source is not NamedTableSource named || predicate is null @@ -36946,7 +36979,14 @@ private SourceData GetSourceRowsWithTransientIndex( || IsCommonTableExpression(named, context) || context.Views?.ContainsKey(named.Name) == true || !context.Tables.TryGetValue(named.Name, out var table) - || !TryCreateTransientEqualityLookup(named, table, predicate, context, outerRow, out var lookup)) + || !TryCreateTransientEqualityLookup( + named, + table, + predicate, + context, + outerRow, + out var lookup, + preserveErrors)) { return null; } @@ -37196,13 +37236,22 @@ private sealed record TransientEqualityLookup( // comparison whose affinity conversions are known on both sides. A conjunct that fails any // of those checks is skipped rather than fatal: another conjunct may still be probeable, // and a predicate with none simply scans. + // + // With the probe must also keep the statement's errors: + // unlike a real index seek it prunes rows a SQLite full scan still visits, and the WHERE + // evaluates its conjuncts left to right, so every conjunct ahead of the chosen equality + // runs on a pruned row there. The equality is therefore only probeable while no conjunct + // before it (nor its own value side, evaluated once per probe) can raise on its input; + // `json_extract(i.j, '$') < 0 AND i.k = o.k` must still report the malformed row whose key + // does not match (unnest-correlated.sqltest correlated-exists-inequality-expression-errors). private bool TryCreateTransientEqualityLookup( NamedTableSource source, EmbeddedTable table, Expression predicate, QueryContext context, SourceRow? outerRow, - [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out TransientEqualityLookup? lookup) + [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out TransientEqualityLookup? lookup, + bool preserveErrors = false) { var outputColumns = GetOutputColumns(source, context); var pending = new Stack(); @@ -37212,12 +37261,14 @@ private bool TryCreateTransientEqualityLookup( var conjunct = pending.Pop(); if (conjunct is BinaryExpression { Operator: BinaryOperator.And } and) { - pending.Push(and.Left); + // Right first so conjuncts pop in left-to-right evaluation order. pending.Push(and.Right); + pending.Push(and.Left); continue; } if (conjunct is BinaryExpression { Operator: BinaryOperator.Equal } equal + && (!preserveErrors || !ExpressionCanFailOnInput(equal)) && (TryMatchTransientEquality( equal.Left, equal.Right, @@ -37239,6 +37290,10 @@ private bool TryCreateTransientEqualityLookup( { return true; } + + // This conjunct runs on every row a scan visits; a later equality may not prune. + if (preserveErrors && ExpressionCanFail(conjunct)) + break; } lookup = null; @@ -42194,14 +42249,36 @@ private SqlValue EvaluateExists( // (exists-drops-order-by-distinct.sqltest). LIMIT and OFFSET stay: OFFSET // reduces visible rows (it decides whether a row comes out at all) and LIMIT 0 // means no rows. + var body = DropExistsSuperfluities(expression.Query); + + // SQLite (select.c existsToJoin) runs a positive EXISTS over one plain table with no + // aggregate or LIMIT as a join of the outer query, so an equality on that table probes + // an automatic index and never visits the rows the key rejects - including rows whose + // other WHERE terms would raise. NOT EXISTS stays a correlated subquery that scans every + // row. The flag lets the transient equality probe keep pruning for the join shape only. + var subqueryContext = !expression.Negated && IsSqliteExistsToJoinBody(body) + ? context with { ExistsJoinBody = (SelectStatement)body } + : context.ExistsJoinBody is null ? context : context with { ExistsJoinBody = null }; var exists = ExecuteSubquery( - DropExistsSuperfluities(expression.Query), + body, parameters, row, - context).Rows.Count > 0; + subqueryContext).Rows.Count > 0; return SqlValue.Integer(exists == expression.Negated ? 0 : 1); } + private bool IsSqliteExistsToJoinBody(QueryStatement body) + => body is SelectStatement + { + Source: NamedTableSource, + Limit: null, + Offset: null, + GroupBy.Count: 0, + Having: null, + } select + && !select.Projections.Any(projection => + ContainsAggregate(projection.Expression) || ContainsWindowFunction(projection.Expression)); + /// Removes ORDER BY and DISTINCT from an EXISTS subquery. private static QueryStatement DropExistsSuperfluities(QueryStatement query) => query switch @@ -43413,12 +43490,12 @@ private void ValidateJoinStructure(SelectStatement statement) throw new EmbeddedSqlException("FULL OUTER JOIN chaining is not yet supported"); } - if (ContainsCorrelatedSubqueryReferencingFullJoinNullSide(statement, fullOuterJoins)) - { - throw new EmbeddedSqlException( - "FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables"); - } - + // A correlated subquery over a FULL JOIN is not rejected: since v0.8.1 Turso plans it + // as a semi/anti join placed after the FULL JOIN (optimizer/unnest.rs keeps FULL joins + // out of its may-be-NULL link guard), which equals SQLite's per-row evaluation over + // the null-extended rows. RewriteCorrelatedSubqueriesAsJoins declines whenever an + // outer join is present, so the managed engine evaluates the subquery against each + // joined row, null-padded ones included. foreach (var fullOuterJoin in fullOuterJoins) { // Turso lowers NATURAL joins to INNER joins before the outer-join checks apply. @@ -43485,86 +43562,6 @@ private static bool ContainsOuterJoin(TableSource source) || ContainsOuterJoin(join.Right)); } - // A correlated subquery confined to a FULL JOIN's always-present side is evaluated as - // an ordinary per-row WHERE filter after the FULL JOIN materializes its null-padded - // rows: RewriteCorrelatedSubqueriesAsJoins already declines the semi/anti-join rewrite - // whenever any outer join is present (SourceContainsOuterJoin), so the correlated - // subquery keeps its normal three-valued evaluation against whatever row the FULL JOIN - // produced, including a null-padded one - the same answer a plain SQL WHERE clause - // gives. The chaining guard above (ContainsOuterJoin(fullOuterJoin.Left)) already - // proves that side holds no independent outer join of its own, so nothing there can be - // null-padded except by this same FULL JOIN, as a unit. Only a reference into a FULL - // JOIN's own right side - the side whose match-or-not decides the null padding, and - // whose planning Turso's join-order search cannot always complete - // (core/translate/optimizer/join.rs:1340-1365) - stays rejected. - private static bool ContainsCorrelatedSubqueryReferencingFullJoinNullSide( - SelectStatement statement, - IReadOnlyList fullOuterJoins) - { - var unsafeNames = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (var fullOuterJoin in fullOuterJoins) - CollectFromSourceNames(fullOuterJoin.Right, unsafeNames); - if (unsafeNames.Count == 0) - return false; - - return ContainsCorrelatedSubqueryReferencingFromSources(statement, unsafeNames); - } - - private static bool ContainsCorrelatedSubqueryReferencingFromSources( - SelectStatement statement, - HashSet fromNames) - { - if (fromNames.Count == 0) - return false; - - foreach (var projection in statement.Projections) - { - if (ExpressionContainsCorrelatedSubquery(projection.Expression, fromNames)) - return true; - } - - if (ExpressionContainsCorrelatedSubquery(statement.Where, fromNames)) - return true; - - foreach (var expression in statement.GroupBy) - { - if (ExpressionContainsCorrelatedSubquery(expression, fromNames)) - return true; - } - - if (ExpressionContainsCorrelatedSubquery(statement.Having, fromNames)) - return true; - - foreach (var term in statement.OrderBy) - { - if (ExpressionContainsCorrelatedSubquery(term.Expression, fromNames)) - return true; - } - - if (ExpressionContainsCorrelatedSubquery(statement.Limit, fromNames) - || ExpressionContainsCorrelatedSubquery(statement.Offset, fromNames)) - return true; - - return TableSourceConditionsContainCorrelatedSubquery(statement.Source, fromNames); - } - - private static bool TableSourceConditionsContainCorrelatedSubquery( - TableSource? source, - HashSet fromNames) - { - switch (source) - { - case JoinTableSource join: - return ExpressionContainsCorrelatedSubquery(join.Condition, fromNames) - || TableSourceConditionsContainCorrelatedSubquery(join.Left, fromNames) - || TableSourceConditionsContainCorrelatedSubquery(join.Right, fromNames); - case TableValuedFunctionSource function: - return function.Arguments.Any(argument => ExpressionContainsCorrelatedSubquery(argument, fromNames)); - default: - return false; - } - } - private static void CollectFromSourceNames(TableSource? source, HashSet names) { switch (source) diff --git a/src/Ahtola.Core/IndexSemantics.cs b/src/Ahtola.Core/IndexSemantics.cs index 38b31b7f..158d439c 100644 --- a/src/Ahtola.Core/IndexSemantics.cs +++ b/src/Ahtola.Core/IndexSemantics.cs @@ -478,13 +478,24 @@ public static bool PredicateImplies( { if (!queryConjuncts.Any(candidate => UsesOnlySourceColumns(candidate, tableName, alias) - && PredicateTermsEqual(candidate, required))) + && QueryTermImpliesPredicate(candidate, required))) return false; } return true; } + /// + /// A query conjunct proves a partial-index conjunct it equals, or any disjunct of an OR + /// conjunct: a row satisfying b < 5 satisfies b < 5 OR a > 5 + /// (Turso query_term_implies_predicate, optimizer/constraints.rs). + /// + private static bool QueryTermImpliesPredicate(Expression candidate, Expression required) + => PredicateTermsEqual(candidate, required) + || required is BinaryExpression { Operator: BinaryOperator.Or } disjunction + && (QueryTermImpliesPredicate(candidate, disjunction.Left) + || QueryTermImpliesPredicate(candidate, disjunction.Right)); + private static bool UsesOnlySourceColumns( Expression expression, string tableName, diff --git a/src/Ahtola.Core/Parsing/SqlAst.cs b/src/Ahtola.Core/Parsing/SqlAst.cs index e1bcbebd..fe50bf13 100644 --- a/src/Ahtola.Core/Parsing/SqlAst.cs +++ b/src/Ahtola.Core/Parsing/SqlAst.cs @@ -627,7 +627,10 @@ internal sealed record JoinTableSource( Expression? Condition, JoinKind Kind, IReadOnlyList? UsingColumns = null, - bool Natural = false) : TableSource; + bool Natural = false, + // Spelled CROSS JOIN: an INNER join whose operands keep their FROM order (SQLite + // JT_CROSS; Turso JoinInfo::no_reorder). + bool Cross = false) : TableSource; internal enum JoinKind { diff --git a/src/Ahtola.Core/Parsing/SqlParser.cs b/src/Ahtola.Core/Parsing/SqlParser.cs index cebb4d19..7b0566d3 100644 --- a/src/Ahtola.Core/Parsing/SqlParser.cs +++ b/src/Ahtola.Core/Parsing/SqlParser.cs @@ -3166,23 +3166,31 @@ private TableSource ParseTableSource() var source = ParseSimpleTableSource(); while (true) { + // SQLite's join-constraint grammar attaches an optional ON/USING clause to every join + // operator, the comma and CROSS JOIN included (parse.y joinop/on_using), so + // `a, b ON ...` and `a CROSS JOIN b USING (...)` are ordinary inner joins. + Expression? condition = null; + IReadOnlyList? usingColumns = null; if (Consume(TokenKind.Comma)) { - source = new JoinTableSource(source, ParseSimpleTableSource(), null, JoinKind.Inner); - continue; - } - - if (ConsumeKeyword("CROSS")) - { - ExpectKeyword("JOIN"); - source = new JoinTableSource(source, ParseSimpleTableSource(), null, JoinKind.Inner); + var commaRight = ParseSimpleTableSource(); + ParseJoinConstraint(ref condition, ref usingColumns); + source = new JoinTableSource(source, commaRight, condition, JoinKind.Inner, usingColumns); continue; } var natural = ConsumeKeyword("NATURAL"); JoinKind kind; - if (ConsumeKeyword("LEFT")) + var cross = false; + if (ConsumeKeyword("CROSS")) + { + // CROSS JOIN is an INNER join the planner must not reorder (SQLite JT_CROSS; + // Turso JoinInfo::no_reorder, planner.rs). + kind = JoinKind.Inner; + cross = true; + } + else if (ConsumeKeyword("LEFT")) { ConsumeKeyword("OUTER"); kind = JoinKind.Left; @@ -3205,30 +3213,33 @@ private TableSource ParseTableSource() if (!ConsumeKeyword("JOIN")) { - if (natural || kind != JoinKind.Inner) + if (natural || cross || kind != JoinKind.Inner) throw Error("Expected JOIN."); return source; } var right = ParseSimpleTableSource(); - Expression? condition = null; - IReadOnlyList? usingColumns = null; - if (ConsumeKeyword("ON")) - { - condition = ParseExpression(); - } - else if (ConsumeKeyword("USING")) - { - Expect(TokenKind.LeftParen); - usingColumns = ParseIdentifierList(); - Expect(TokenKind.RightParen); - } + ParseJoinConstraint(ref condition, ref usingColumns); if (natural && (condition is not null || usingColumns is not null)) throw Error("a NATURAL join may not have an ON or USING clause"); - source = new JoinTableSource(source, right, condition, kind, usingColumns, natural); + source = new JoinTableSource(source, right, condition, kind, usingColumns, natural, cross); + } + } + + private void ParseJoinConstraint(ref Expression? condition, ref IReadOnlyList? usingColumns) + { + if (ConsumeKeyword("ON")) + { + condition = ParseExpression(); + } + else if (ConsumeKeyword("USING")) + { + Expect(TokenKind.LeftParen); + usingColumns = ParseIdentifierList(); + Expect(TokenKind.RightParen); } } diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index 3d47aa06..f4bcd0e4 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -188,19 +188,18 @@ join/hash_join_order_by.sqltest::hash-join-order-by-build-table-uses-sorter | da join/memory.sqltest::equality-prefix-keeps-index-search | database variant 1/1 (:memory:): output does not match pattern 'SEARCH child USING COVERING INDEX sqlite_autoindex_prefix_child_1 \(group_id=\? AND parent_id=\?\)' join/memory.sqltest::expression-index-after-earlier-join-reads-source-table | database variant 1/1 (:memory:): output does not match pattern '(?s)MATERIALIZE hash build input for expression_build_orders AS orders.*SCAN expression_build_customers AS customers.*SEARCH orders USIN join/memory.sqltest::expression-index-hash-build-reads-source-table | database variant 1/1 (:memory:): output does not match pattern '(?s)MATERIALIZE hash build input for expression_build_orders AS orders.*HASH JOIN expression_build_inventory AS inventory.*SEARCH orders -join/memory.sqltest::future-table-cannot-make-index-usable | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 121. -join/memory.sqltest::future-table-index-does-not-change-join-results | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 133. +join/memory.sqltest::future-table-cannot-make-index-usable | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN future_index_child AS child.*SEARCH group_table USING INTEGER PRIMARY KEY.*SCAN future_index_parent AS parent' join/memory.sqltest::group-sort-cost-keeps-cheaper-correlated-seeks | database variant 1/1 (:memory:): output does not match pattern '(?s)CORRELATED SCALAR SUBQUERY.*SEARCH child USING INDEX sqlite_autoindex_sort_cost_child_1 \(parent_id=\?\)' -join/memory.sqltest::hash-build-row-limit-counts-prefix-rows | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 155. +join/memory.sqltest::hash-build-row-limit-counts-prefix-rows | database variant 1/1 (:memory:): output does not match pattern '"detail":"SEARCH probe_rows USING COVERING INDEX ephemeral_prefix_row_limit_probe_t3' join/memory.sqltest::hash-join-cost-includes-rows-visited | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN hash_cost_right".*"access_cost":300110000' join/memory.sqltest::index-suffix-does-not-prevent-hash-join | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN suffix_hash_child AS child.*SCAN suffix_hash_parent AS parent' join/memory.sqltest::indexed-build-hashes-an-unfiltered-large-probe | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN indexed_hash_child AS child.*SCAN indexed_hash_parent AS parent' join/memory.sqltest::materialized-hash-build-keeps-index-search | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"MATERIALIZE hash build input for selected_hash_build AS build_rows".*"detail":"SEARCH build_rows USING INDEX selected_hash -join/memory.sqltest::no-stats-hash-join-keeps-unindexed-probe-fanout | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 79. -join/memory.sqltest::partial-stats-hash-join-uses-known-unique-key-count | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 79. +join/memory.sqltest::no-stats-hash-join-keeps-unindexed-probe-fanout | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN c".*"rows_per_input":100000.*"detail":"SEARCH d USING INTEGER PRIMARY KEY' +join/memory.sqltest::partial-stats-hash-join-uses-known-unique-key-count | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN c".*"rows_per_input":5.*"detail":"SEARCH d USING INTEGER PRIMARY KEY' join/memory.sqltest::range-prefix-still-allows-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN prefix_child AS child' join/memory.sqltest::unique-composite-hash-key-uses-one-key-estimate | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN composite_hash_child AS child".*"rows_per_input":5' -join/memory.sqltest::unique-hash-key-does-not-apply-to-a-different-collation | database variant 1/1 (:memory:): expected success but got error: Expected End. At SQL offset 127. +join/memory.sqltest::unique-hash-key-does-not-apply-to-a-different-collation | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN collation_unique_probe".*"rows_per_input":0.1' json/default.sqltest::jsonb_null_is_sql_null | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected 'null/1' but was 'blob/0' json/json_tree_columns.sqltest::json-tree-rowid-into-table | database variant 1/1 (:memory:): expected success but got error: no such column: rt.rowid json/json_tree_columns.sqltest::json-tree-rowid-is-selectable | database variant 1/1 (:memory:): expected success but got error: no such column: rt.rowid @@ -213,12 +212,6 @@ multi_index_or_compound.sqltest::implied-or-filter-keeps-compound-index-searches multi_index_or_compound.sqltest::implied-or-filter-uses-only-indexed-column | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' multi_index_or_compound.sqltest::nested-and-branches-use-one-index-search | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '1/0/0/SEARCH t USING INDEX tx (x=?)' but was '0/0/0/MANAGED EVALUATOR FALLBACK' multi_index_or_compound.sqltest::nested-or-branches-use-one-index-search | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '1/0/0/SEARCH t USING COVERING INDEX tx (x=?)' but was '0/0/0/MANAGED EVALUATOR FALLBACK' -mvcc-analyze-refreshes-planner-stats.sqltest::mvcc-analyze-next-plan-uses-new-row-estimate | database variant 1/1 (:temp:): output does not match pattern '"rows_per_input":100,"output_rows":100,' -mvcc-analyze-refreshes-planner-stats.sqltest::wal-analyze-next-plan-uses-new-row-estimate | database variant 1/1 (:temp:): output does not match pattern '"rows_per_input":100,"output_rows":100,' -partial_idx.sqltest::partial-index-or-right-term-with-range-search | database variant 1/1 (:memory:): expected 2 row(s), got 2 row(s); row 0 expected '1/0/0/SEARCH t USING COVERING INDEX idx (a>?)' but was '1/0/0/SCAN t' -partial_idx.sqltest::partial-index-or-term-with-covering-scan | database variant 1/1 (:memory:): expected 3 row(s), got 3 row(s); row 0 expected '1/0/0/SCAN t USING COVERING INDEX idx' but was '1/0/0/SCAN t' -partial_idx.sqltest::partial-index-or-term-with-equality-search | database variant 1/1 (:memory:): expected 2 row(s), got 2 row(s); row 0 expected '1/0/0/SEARCH t USING INDEX idx (a=?)' but was '1/0/0/SCAN t' -partial_idx.sqltest::partial-index-or-term-with-order | database variant 1/1 (:memory:): expected 3 row(s), got 4 row(s); row 0 expected '1/0/0/SCAN t USING INDEX idx' but was '1/0/0/SCAN t' pragma/memory.sqltest::pragma-temp-synchronous-stays-off | database variant 1/1 (:memory:): expected 3 row(s), got 3 row(s); row 0 expected '0' but was '2' scalar-functions-datetime.sqltest::date-blob-modifier | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '2024-06-16' but was '' scalar-functions-datetime.sqltest::date-blob-time-value | database variant 1/2 (:default:): expected 1 row(s), got 1 row(s); row 0 expected '2024-01-01' but was '' @@ -261,17 +254,7 @@ turso/upsert-target-alias.sqltest::upsert-base-name-is-out-of-scope-when-aliased turso/upsert-target-alias.sqltest::upsert-target-alias-is-only-target-name | database variant 1/1 (:memory:): expected success but got error: no such column: x.n turso/upsert-target-alias.sqltest::upsert-target-alias-shadows-excluded | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '10' but was '99' turso/vacuum.sqltest::vacuum-rejects-query-only | database variant 1/1 (:temp:): error 'attempt to write a readonly database' does not match pattern 'Cannot execute VACUUM in query_only mode' -unnest-correlated.sqltest::correlated-comparison-after-full-join[exists][s-amount-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 6 row(s) -unnest-correlated.sqltest::correlated-comparison-after-full-join[exists][s-amount-is-not-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 7 row(s) -unnest-correlated.sqltest::correlated-comparison-after-full-join[not-exists][s-amount-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 1 row(s) -unnest-correlated.sqltest::correlated-comparison-after-full-join[not-exists][s-amount-is-not-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 0 row(s) unnest-correlated.sqltest::correlated-comparison-uses-joins | database variant 1/1 (:memory:): expected 4 row(s), got 2 row(s); row 0 expected '1/0/0/SCAN outer_rows AS o' but was '0/0/0/MANAGED EVALUATOR FALLBACK' -unnest-correlated.sqltest::correlated-exists-inequality-expression-errors[not-exists][json-extract-i-json-value-o-limit-value][and-i-value-o-wanted] | database variant 1/1 (:memory:): SQLite errored ('SQLite Error 1: 'malformed JSON'.') but managed returned 1 row(s) -unnest-correlated.sqltest::correlated-exists-inequality-expression-errors[not-exists][o-limit-value-json-extract-i-json-value][and-i-value-o-wanted] | database variant 1/1 (:memory:): SQLite errored ('SQLite Error 1: 'malformed JSON'.') but managed returned 1 row(s) -unnest-correlated.sqltest::correlated-in-after-full-join[i-amount-s-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 0 row(s) -unnest-correlated.sqltest::correlated-in-after-full-join[s-amount-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 0 row(s) -unnest-correlated.sqltest::correlated-in-after-full-join[s-amount-is-not-i-amount] | database variant 1/1 (:memory:): managed errored ('FULL OUTER JOIN is not supported with correlated subqueries that reference the joined tables') but SQLite returned 0 row(s) -unnest-correlated.sqltest::correlated-in-error-expressions-stay-subqueries | database variant 1/1 (:memory:): output does not match pattern '(?s)CORRELATED LIST SUBQUERY.*CORRELATED LIST SUBQUERY.*CORRELATED LIST SUBQUERY' unnest-correlated.sqltest::correlated-in-inequality-uses-indexed-semi-join | database variant 1/1 (:memory:): output does not match pattern '(?s)SCAN outer_rows AS o.*SEARCH i USING COVERING INDEX inner_key_amount' unnest-correlated.sqltest::correlated-in-with-only-inequality-uses-indexed-semi-join | database variant 1/1 (:memory:): output does not match pattern '(?s)SCAN outer_rows AS o.*SEARCH i USING COVERING INDEX inner_key_amount \(key1<\?\)' unnest-correlated.sqltest::not-exists-can-use-hash-anti-join | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN hash_anti_inner AS i.*SCAN hash_anti_outer AS o' diff --git a/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs b/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs index 587b5ea7..8a0c2b80 100644 --- a/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs +++ b/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs @@ -326,6 +326,47 @@ public void PlainTableFallbackStillDescribesItsActualBaseScan() node.GetProperty("op").GetProperty("alias").GetString().Should().Be("base"); } + [Test] + public void UnfilteredTableScanReportsThePlannersCurrentRowEstimate() + { + using var embedded = new EmbeddedDatabase(); + using var connection = embedded.Connect(); + Execute(connection, "CREATE TABLE t(id INTEGER PRIMARY KEY, value INTEGER);"); + + JsonElement Estimate(string query) + { + using var document = JsonDocument.Parse( + ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON " + query).Single()); + return document.RootElement.GetProperty("nodes")[0].GetProperty("op").GetProperty("estimate").Clone(); + } + + // No statistics: Turso's 1,000,000-row default, priced by estimate_scan_cost + // (1,000,000 / 50 pages + 1,000,000 * 0.003 CPU). + var unanalyzed = Estimate("SELECT * FROM t;"); + unanalyzed.GetProperty("input_rows").GetDouble().Should().Be(1); + unanalyzed.GetProperty("rows_per_input").GetDouble().Should().Be(1_000_000); + unanalyzed.GetProperty("output_rows").GetDouble().Should().Be(1_000_000); + unanalyzed.GetProperty("access_cost").GetDouble().Should().Be(23_000); + unanalyzed.GetProperty("total_cost").GetDouble().Should().Be(23_000); + + // The next plan after ANALYZE uses the fresh sqlite_stat1 count + // (mvcc-analyze-refreshes-planner-stats.sqltest). + Execute(connection, "WITH RECURSIVE seq(n) AS (VALUES(1) UNION ALL SELECT n + 1 FROM seq WHERE n < 100) INSERT INTO t SELECT n, n FROM seq;"); + Execute(connection, "ANALYZE;"); + var analyzed = Estimate("SELECT * FROM t;"); + analyzed.GetProperty("rows_per_input").GetDouble().Should().Be(100); + analyzed.GetProperty("output_rows").GetDouble().Should().Be(100); + analyzed.GetProperty("access_cost").GetDouble().Should().BeApproximately(2.3, 1e-12); + ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON SELECT * FROM t;").Single() + .Should().Contain("\"rows_per_input\":100,\"output_rows\":100,"); + + // A filtered scan needs the unported selectivity model, so it reports no estimate. + using var filtered = JsonDocument.Parse( + ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON SELECT * FROM t WHERE value > 3;").Single()); + foreach (var node in filtered.RootElement.GetProperty("nodes").EnumerateArray()) + node.GetProperty("op").TryGetProperty("estimate", out _).Should().BeFalse(); + } + [Test] public void DerivedSourceNestsItsIndexScanUnderTheCoroutineNode() { diff --git a/src/Ahtola.Tests/ExplainQueryPlanTests.cs b/src/Ahtola.Tests/ExplainQueryPlanTests.cs index e15be59e..045b4418 100644 --- a/src/Ahtola.Tests/ExplainQueryPlanTests.cs +++ b/src/Ahtola.Tests/ExplainQueryPlanTests.cs @@ -505,6 +505,68 @@ public void RendersTursoStylePartialIndexSearchAndScanPlans() .Rows[0][3].Should().Be(SqlValue.Text("SCAN products")); } + [Test] + public void PartialIndexWithAnOrPredicateIsProvenByEitherDisjunct() + { + // Turso v0.8.1 query_term_implies_predicate: a query conjunct implies an OR conjunct of + // the index predicate when it implies either side (partial_idx.sqltest + // partial-index-or-term-*). Every result must still match the table scan's answer. + using var connection = new EmbeddedDatabase().Connect(); + Execute(connection, "CREATE TABLE t(a, b);"); + Execute(connection, "CREATE INDEX idx ON t(a) WHERE b < 5 OR a > 5;"); + Execute(connection, "CREATE TABLE t2(a, b);"); + Execute(connection, "CREATE INDEX idx2 ON t2(a, b) WHERE b < 5 OR a > 5;"); + Execute(connection, "INSERT INTO t VALUES (-1, 1), (-3, 3), (-5, 5), (-8, 8), (10, 8);"); + Execute(connection, "INSERT INTO t2 SELECT * FROM t;"); + + string Detail(string sql) => ReadPlan(connection, "EXPLAIN QUERY PLAN " + sql).Rows.Single()[3].AsText(); + List Values(string sql) => ReadPlan(connection, sql).Rows.Select(static row => row[0].AsInteger()).ToList(); + + Detail("SELECT a FROM t WHERE b < 5 AND a = -3;").Should().Be("SEARCH t USING INDEX idx (a=?)"); + Values("SELECT a FROM t WHERE b < 5 AND a = -3;").Should().Equal(-3); + Detail("SELECT a FROM t WHERE b < 5 ORDER BY a;").Should().Be("SCAN t USING INDEX idx"); + Values("SELECT a FROM t WHERE b < 5 ORDER BY a;").Should().Equal(-3, -1); + Detail("SELECT a FROM t WHERE a > 5;").Should().Be("SEARCH t USING COVERING INDEX idx (a>?)"); + Values("SELECT a FROM t WHERE a > 5;").Should().Equal(10); + Detail("SELECT a FROM t2 WHERE b < 5;").Should().Be("SCAN t2 USING COVERING INDEX idx2"); + Values("SELECT a FROM t2 WHERE b < 5 ORDER BY a;").Should().Equal(-3, -1); + + // A conjunct that implies neither disjunct still cannot use the index. + Detail("SELECT a FROM t WHERE b <= 5 AND a = -5;").Should().NotContain("idx"); + Values("SELECT a FROM t WHERE b <= 5 AND a = -5;").Should().Equal(-5); + } + + [Test] + public void UnrewrittenCorrelatedInDescribesItsPerRowListScan() + { + // A correlated IN the semi-join rewrite declines (a failing operand or filter, or a + // non-equality link) stays a list subquery the evaluator re-runs for each outer row + // (unnest-correlated.sqltest correlated-in-error-expressions-stay-subqueries). + using var connection = new EmbeddedDatabase().Connect(); + Execute(connection, "CREATE TABLE error_outer(id INTEGER, wanted INTEGER, limit_value INTEGER, text_value TEXT);"); + Execute(connection, "CREATE TABLE error_inner(value INTEGER, json_value TEXT);"); + + foreach (var sql in new[] + { + "SELECT o.id FROM error_outer AS o WHERE o.wanted IN (SELECT i.value FROM error_inner AS i WHERE json_extract(i.json_value, '$') < o.limit_value);", + "SELECT o.id FROM error_outer AS o WHERE o.wanted IN (SELECT json_extract(i.json_value, '$') FROM error_inner AS i WHERE i.value < o.limit_value);", + "SELECT o.id FROM error_outer AS o WHERE json_extract(o.text_value, '$') IN (SELECT i.value FROM error_inner AS i WHERE i.value < o.limit_value);", + }) + { + ReadPlan(connection, "EXPLAIN QUERY PLAN " + sql).Rows.Select(static row => row[3].AsText()) + .Should().Equal( + new[] { "SCAN error_outer AS o", "CORRELATED LIST SUBQUERY 1", "SCAN error_inner AS i" }, + sql); + } + + // An equality link is still unnested, so it is not described as a per-row list. + ReadPlan( + connection, + "EXPLAIN QUERY PLAN SELECT o.id FROM error_outer AS o WHERE o.wanted IN (SELECT i.value FROM error_inner AS i WHERE i.value = o.limit_value);") + .Rows.Select(static row => row[3].AsText()) + .Should().NotContain("CORRELATED LIST SUBQUERY 1"); + } + [Test] public void ReverseIndexScanReversesExplicitNullPlacement() { diff --git a/src/Ahtola.Tests/JoinOrderOptimizerTests.cs b/src/Ahtola.Tests/JoinOrderOptimizerTests.cs index ed64b962..313559bb 100644 --- a/src/Ahtola.Tests/JoinOrderOptimizerTests.cs +++ b/src/Ahtola.Tests/JoinOrderOptimizerTests.cs @@ -353,6 +353,61 @@ public void CommaCrossJoinSegmentReordersAndKeepsTheSameRowMultiset() database.JoinOrderDiagnostics.SegmentsReordered.Should().BeGreaterThan(0); } + [Test] + public void ExplicitCrossJoinKeepsTheFromOrder() + { + var setup = + $""" + CREATE TABLE c1(a INTEGER); + CREATE TABLE c2(b INTEGER); + CREATE TABLE c3(c INTEGER); + {Singles("c1", 20)} + {Singles("c2", 4)} + {Singles("c3", 2)} + ANALYZE; + """; + // The comma spelling of this query is reordered (see above); CROSS JOIN is SQLite's + // manual join-order control, so the same three tables stay in FROM order, and the + // unordered result follows that nested-loop order exactly like SQLite. + const string query = "SELECT a, b, c FROM c1 CROSS JOIN c2 CROSS JOIN c3;"; + + AssertMatchesSqlite(setup, query); + + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + Execute(connection, setup); + Rows(connection, query).Should().HaveCount(160); + ScanOrder(JoinDescription(connection, "EXPLAIN " + query)).Should().Equal("c1", "c2", "c3"); + database.JoinOrderDiagnostics.SegmentsReordered.Should().Be(0); + } + + [TestCase("SELECT p.id, c.k FROM p CROSS JOIN c ON c.pid = p.id ORDER BY p.id, c.k;")] + [TestCase("SELECT p.id, c.k FROM p, c ON c.pid = p.id ORDER BY p.id, c.k;")] + [TestCase("SELECT p.id, c.k FROM p, c ON c.pid = p.id WHERE c.k > 1 ORDER BY p.id, c.k;")] + [TestCase("SELECT * FROM p CROSS JOIN q USING (id) ORDER BY id;")] + [TestCase("SELECT * FROM p, q USING (id) ORDER BY id;")] + [TestCase("SELECT * FROM p NATURAL CROSS JOIN q ORDER BY id;")] + [TestCase("SELECT p.id, c.k, q.w FROM p CROSS JOIN c ON c.pid = p.id CROSS JOIN q ON q.id = c.k;")] + [TestCase("SELECT count(*) FROM p CROSS JOIN c ON c.pid = p.id CROSS JOIN q ON q.id = c.k WHERE p.v = 'x';")] + [TestCase("SELECT p.id, c.k FROM p CROSS JOIN c ON c.pid = p.id LEFT JOIN q ON q.id = c.k ORDER BY 1, 2;")] + // Unordered: CROSS JOIN keeps q as the outer loop even though ANALYZE shows it is the + // smaller input a plain equijoin would hash-build, so rows follow q's order like SQLite. + [TestCase("SELECT q.id, c.pid FROM q CROSS JOIN c ON c.k = q.id;")] + public void JoinConstraintsOnCommaAndCrossJoinsMatchSqlite(string query) + { + var setup = + """ + CREATE TABLE p(id INTEGER PRIMARY KEY, v TEXT); + CREATE TABLE c(pid INTEGER, k INTEGER); + CREATE TABLE q(id INTEGER PRIMARY KEY, w TEXT); + INSERT INTO p VALUES (1, 'x'), (2, 'y'), (3, 'x'); + INSERT INTO c VALUES (1, 1), (1, 2), (2, 3), (NULL, 1), (3, 9); + INSERT INTO q VALUES (1, 'q1'), (3, 'q3'), (9, 'q9'); + ANALYZE; + """; + AssertMatchesSqlite(setup, query); + } + [Test] public void SelfJoinAliasesStayDistinctMembers() { diff --git a/src/Ahtola.Tests/SubqueryRewriteTests.cs b/src/Ahtola.Tests/SubqueryRewriteTests.cs index 2a0bf6b1..47d36cf4 100644 --- a/src/Ahtola.Tests/SubqueryRewriteTests.cs +++ b/src/Ahtola.Tests/SubqueryRewriteTests.cs @@ -910,6 +910,88 @@ public void DoesNotRewriteInWhoseInnerWhereCanFailOnALaterRow() AssertRewrites(wellFormed, query, semiJoins: 0); } + private const string ErrorRowsSetup = + """ + CREATE TABLE error_outer(id INTEGER, wanted INTEGER, limit_value INTEGER, text_value TEXT); + CREATE TABLE error_inner(value INTEGER, json_value TEXT); + INSERT INTO error_outer VALUES (1, 1, 10, '1'), (2, 1, 0, 'bad JSON'); + INSERT INTO error_inner VALUES (1, '1'), (2, 'bad JSON'); + """; + + [TestCase("json_extract(i.json_value, '$') < o.limit_value AND i.value = o.wanted")] + [TestCase("o.limit_value > json_extract(i.json_value, '$') AND i.value = o.wanted")] + public void NotExistsKeepsTheErrorOfARowTheEqualityWouldPrune(string innerWhere) + { + // NOT EXISTS stays a correlated subquery that scans every inner row, evaluating its + // WHERE left to right. For outer row 2 the malformed row's key does not match, but its + // json_extract runs first and raises. The transient equality probe must not prune that + // row away (unnest-correlated.sqltest correlated-exists-inequality-expression-errors). + var query = + $"SELECT o.id FROM error_outer AS o WHERE NOT EXISTS (SELECT 1 FROM error_inner AS i WHERE {innerWhere}) ORDER BY o.id;"; + AssertFailsLikeSqlite(ErrorRowsSetup, query, "malformed JSON"); + } + + [TestCase("json_extract(i.json_value, '$') < o.limit_value AND i.value = o.wanted")] + [TestCase("o.limit_value > json_extract(i.json_value, '$') AND i.value = o.wanted")] + public void PositiveExistsProbesTheEqualityLikeSqlitesExistsJoin(string innerWhere) + { + // SQLite runs a positive EXISTS over one plain table as a join of the outer query + // (select.c existsToJoin) whose automatic index on i.value never visits the malformed + // row, so it answers instead of raising; the managed probe keeps doing the same. + AssertMatchesSqlite( + ErrorRowsSetup, + $"SELECT o.id FROM error_outer AS o WHERE EXISTS (SELECT 1 FROM error_inner AS i WHERE {innerWhere}) ORDER BY o.id;"); + } + + [Test] + public void EqualityProbeStillPrunesWhenNothingBeforeItCanFail() + { + // The equality comes first, so a SQLite scan rejects the malformed row before its + // json_extract runs: pruning it through the probe changes nothing. + AssertMatchesSqlite( + ErrorRowsSetup, + "SELECT o.id FROM error_outer AS o WHERE NOT EXISTS (SELECT 1 FROM error_inner AS i WHERE i.value = o.wanted AND json_extract(i.json_value, '$') < o.limit_value) ORDER BY o.id;"); + AssertFailsLikeSqlite( + ErrorRowsSetup, + "SELECT 1 FROM error_inner AS i WHERE json_extract(i.json_value, '$') < 0 AND i.value = 1;", + "malformed JSON"); + } + + private const string FullJoinSetup = + """ + CREATE TABLE outer_rows(id INTEGER PRIMARY KEY, key1 INTEGER, key2 INTEGER, amount INTEGER); + CREATE TABLE inner_rows(key1 INTEGER, key2 INTEGER, amount INTEGER); + CREATE TABLE outer_key2(id INTEGER PRIMARY KEY, key2 INTEGER); + INSERT INTO outer_rows VALUES (1, 1, 10, 5), (2, 1, 10, 15), (3, 2, 20, 7), (4, 3, 30, 9), (5, NULL, 40, 4), (6, 2, NULL, NULL); + INSERT INTO inner_rows VALUES (1, 10, 10), (1, 10, 20), (1, 99, 100), (2, 20, 6), (2, 20, NULL), (2, NULL, 7), (NULL, 40, 99); + INSERT INTO outer_key2 VALUES (1, 10), (3, 20), (5, 40); + """; + + [TestCase("i.amount IN (SELECT s.amount FROM outer_rows AS s WHERE s.amount < i.amount)")] + [TestCase("i.amount IN (SELECT s.amount FROM outer_rows AS s WHERE s.amount IS NOT i.amount)")] + [TestCase("EXISTS (SELECT 1 FROM outer_rows AS s WHERE s.amount < i.amount)")] + [TestCase("EXISTS (SELECT 1 FROM outer_rows AS s WHERE s.amount IS NOT i.amount)")] + [TestCase("NOT EXISTS (SELECT 1 FROM outer_rows AS s WHERE s.amount < i.amount)")] + [TestCase("NOT EXISTS (SELECT 1 FROM outer_rows AS s WHERE s.amount IS NOT i.amount)")] + [TestCase("EXISTS (SELECT 1 FROM outer_rows AS s WHERE s.id = k.id)")] + [TestCase("(SELECT count(*) FROM outer_rows AS s WHERE s.amount < i.amount) > 1")] + public void CorrelatedSubqueryOverAFullJoinSeesTheNullExtendedRows(string predicate) + { + // The subquery references the FULL JOIN's null-supplying side, so it must run against + // every joined row, NULL-padded ones included, exactly like SQLite (and Turso v0.8.1, + // which now plans these as semi/anti joins placed after the FULL JOIN). + AssertMatchesSqlite( + FullJoinSetup, + $""" + SELECT o.id, k.id, i.key1, i.amount + FROM outer_rows AS o + JOIN outer_key2 AS k ON o.id = k.id + FULL JOIN inner_rows AS i ON k.key2 = i.key2 + WHERE {predicate} + ORDER BY o.id, k.id, i.key1, i.amount; + """); + } + [Test] public void SemiJoinProbePreservesComparisonAffinity() { From 6fcb7860cd6b3aebb7ae7c848bcbc716806c3f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 17:21:06 -0400 Subject: [PATCH 11/20] Unnest correlated comparisons and plan semi/anti inner access with the Turso cost model Port Turso v0.8.1 f7aac1fc5 ("optimizer: allow unnesting correlated subqueries with non-equality comparisons"): a correlated EXISTS, NOT EXISTS or positive IN may now move any direct inner/outer comparison (=, <>, <, <=, >, >=, IS, IS NOT) into the semi/anti join condition (unnest.rs is_inner_outer_comparison). The fallible/non-deterministic inner-WHERE gates are unchanged. The inner access of a semi/anti join is now chosen by a port of the v0.8.1 join planner (choose_best_btree_candidate, the temporary-index alternative of find_best_access_method_for_btree, and the LeftAnti hash join of join_lhs_and_rhs / try_hash_join_access_method with its should_not_use_hash_join and probe-index gates), costed with exact ports of cost.rs / estimate_hash_join_cost (TursoCostModel). Execution and EXPLAIN QUERY PLAN both consume the same SemiAntiInnerAccess: - declared index searches (equality prefix plus one range), - automatic (ephemeral) index searches, - a new left anti hash join that hashes the outer rows, probes the inner table once and emits unmatched build rows in order. The compiled automatic-index semi-join lowering declines when the cost model prefers another access, so EQP and execution stay one plan. Closes unnest-correlated.sqltest correlated-comparison-uses-joins, correlated-in-inequality-uses-indexed-semi-join, correlated-in-with-only-inequality-uses-indexed-semi-join and not-exists-can-use-hash-anti-join. Co-Authored-By: Claude Opus 5.5 --- .../JoinOrdering/TursoCostModel.cs | 216 ++++ ...eddedDatabase.ExplainQueryPlanDescriber.cs | 134 ++ .../EmbeddedDatabase.SemiAntiJoinAccess.cs | 1100 +++++++++++++++++ .../EmbeddedDatabase.SubqueryRewrites.cs | 35 +- src/Ahtola.Core/EmbeddedDatabase.cs | 186 ++- .../managed-sqltest-expected-failures.txt | 4 - src/Ahtola.Tests/SubqueryRewriteTests.cs | 31 +- 7 files changed, 1682 insertions(+), 24 deletions(-) create mode 100644 src/Ahtola.Core/Compilation/JoinOrdering/TursoCostModel.cs create mode 100644 src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs diff --git a/src/Ahtola.Core/Compilation/JoinOrdering/TursoCostModel.cs b/src/Ahtola.Core/Compilation/JoinOrdering/TursoCostModel.cs new file mode 100644 index 00000000..d231691a --- /dev/null +++ b/src/Ahtola.Core/Compilation/JoinOrdering/TursoCostModel.cs @@ -0,0 +1,216 @@ +namespace Ahtola.Core.Compilation.JoinOrdering; + +/// +/// Cost-model constants of Turso v0.8.1 core/translate/optimizer/cost_params.rs +/// (CostModelParams::new) that does not already carry. +/// +internal static class TursoCostParams +{ + /// cost_params.rs: sel_is_null. + public const double SelectivityIsNull = 0.1; + + /// cost_params.rs: sel_is_not_null. + public const double SelectivityIsNotNull = 0.9; + + /// cost_params.rs: sel_like. + public const double SelectivityLike = 0.2; + + /// cost_params.rs: sel_not_like. + public const double SelectivityNotLike = 0.2; + + /// cost_params.rs: in_subquery_rows. + public const double InSubqueryRows = 25.0; + + /// cost_params.rs: cpu_cost_per_where_step. + public const double CpuCostPerWhereStep = 0.003; + + /// cost_params.rs: hash_bytes_per_row. + public const double HashBytesPerRow = 100.0; + + /// cost_params.rs: closed_range_selectivity_factor. + public const double ClosedRangeSelectivityFactor = 0.2; + + /// + /// core/vdbe/hash_table.rs DEFAULT_MEM_BUDGET for the debug profile the + /// upstream sqltest corpus is generated with (32 KiB; release builds use 64 MiB). The + /// value only decides whether charges the + /// grace-hash spill term. + /// + public const double HashJoinMemoryBudgetBytes = 32 * 1024; +} + +/// +/// Exact ports of the Turso v0.8.1 cost formulas in core/translate/optimizer/cost.rs +/// and access_method.rs (estimate_hash_join_cost). Unlike +/// (a narrowed, older subset tuned to the managed join operator), +/// these reproduce upstream's arithmetic term for term so a planner decision or an +/// EXPLAIN QUERY PLAN FORMAT=JSON estimate computed from them is upstream's own number. +/// +internal static class TursoCostModel +{ + /// cost.rs:120-139 estimate_scan_cost. + public static double EstimateScanCost(double baseRowCount, double scanCount) + { + var tablePages = Math.Max(baseRowCount / JoinCostParams.RowsPerTablePage, 1.0); + var ioCost = scanCount <= 1.0 + ? tablePages + : tablePages + (scanCount - 1.0) * tablePages * JoinCostParams.CacheReuseFactor; + var cpuCost = scanCount * baseRowCount * JoinCostParams.CpuCostPerRow; + return ioCost + cpuCost; + } + + /// cost.rs:145-151 estimate_ephemeral_index_build_cost. + public static double EstimateEphemeralIndexBuildCost(double rowCount) + => rowCount * Math.Log2(Math.Max(rowCount, 2.0)) * JoinCostParams.CpuCostPerSeek; + + /// cost.rs:153-155 estimate_sort_cpu_cost. + public static double EstimateSortCpuCost(double rowCount) + => rowCount * Math.Log2(Math.Max(rowCount, 1.0)) * JoinCostParams.SortCpuPerRow; + + /// cost.rs:158-164 estimate_btree_depth. + public static double EstimateBtreeDepth(double rowCount, double rowsPerPage) + => rowCount <= 1.0 + ? 1.0 + : Math.Max(1.0, Math.Ceiling(Math.Log(rowCount) / Math.Log(rowsPerPage))); + + /// cost.rs:83-94 index_leaf_rows_per_page. + public static double IndexLeafRowsPerPage( + int indexColumnCount, + int tableColumnCount, + bool hasRowIdAlias) + { + var tableWidth = tableColumnCount + (hasRowIdAlias ? 0.0 : 1.0); + var indexWidth = indexColumnCount + 1.0; + return Math.Max(JoinCostParams.RowsPerTablePage * tableWidth / indexWidth, 1.0); + } + + /// cost.rs:179-250 estimate_index_cost. + public static double EstimateIndexCost( + double baseRowCount, + double treeDepth, + TursoIndexInfo indexInfo, + double inputCardinality, + double rowsPerSeek) + { + var isFullScan = Math.Abs(rowsPerSeek - baseRowCount) < 1.0; + double seekCost; + if (isFullScan) + { + seekCost = inputCardinality <= 1.0 + ? treeDepth + : treeDepth + (inputCardinality - 1.0) * treeDepth * JoinCostParams.CacheReuseFactor; + } + else if (inputCardinality <= 1.0) + { + seekCost = inputCardinality * treeDepth; + } + else + { + var cachedRootPageCost = JoinCostParams.CacheReuseFactor; + var leafPageCostedBelow = rowsPerSeek > 1.0 ? 1.0 : 0.0; + var repeatedTreeSearchCost = Math.Max( + treeDepth - 1.0 + cachedRootPageCost - leafPageCostedBelow, + cachedRootPageCost); + seekCost = treeDepth + (inputCardinality - 1.0) * repeatedTreeSearchCost; + } + + var indexLeafPagesCount = Math.Max(rowsPerSeek / indexInfo.RowsPerLeafPage, 1.0); + double leafScanCost; + if (isFullScan) + { + leafScanCost = inputCardinality <= 1.0 + ? indexLeafPagesCount + : indexLeafPagesCount + + (inputCardinality - 1.0) * indexLeafPagesCount * JoinCostParams.CacheReuseFactor; + } + else if (rowsPerSeek <= 1.0) + { + leafScanCost = 0.0; + } + else if (inputCardinality <= 1.0) + { + leafScanCost = indexLeafPagesCount; + } + else + { + leafScanCost = indexLeafPagesCount + + (inputCardinality - 1.0) * indexLeafPagesCount * JoinCostParams.CacheReuseFactor; + } + + double tableLookupCost; + if (indexInfo.Covering) + { + tableLookupCost = 0.0; + } + else + { + var tablePagesCount = Math.Max(baseRowCount / JoinCostParams.RowsPerTablePage, 1.0); + var selectivity = rowsPerSeek / Math.Max(baseRowCount, 1.0); + tableLookupCost = inputCardinality * selectivity * tablePagesCount; + } + + var ioCost = seekCost + leafScanCost + tableLookupCost; + var totalRows = inputCardinality * rowsPerSeek; + var cpuCost = inputCardinality * JoinCostParams.CpuCostPerSeek + + totalRows * JoinCostParams.CpuCostPerRow; + return Math.Max(ioCost + cpuCost - JoinCostParams.IndexBonus, 0.001); + } + + /// + /// access_method.rs:1242-1290 estimate_hash_join_cost, including the grace-hash + /// spill term against . + /// + public static double EstimateHashJoinCost( + double buildCardinality, + double probeCardinality, + double rowsVisited, + bool keepsUnmatchedBuildRows, + double probeMultiplier) + { + // `build_cardinality as usize` saturates (and truncates) before the multiply. + var estimatedHashTableSize = Math.Floor(Math.Max(buildCardinality, 0.0)) * TursoCostParams.HashBytesPerRow; + var willSpill = estimatedHashTableSize > TursoCostParams.HashJoinMemoryBudgetBytes; + var buildCost = buildCardinality * (JoinCostParams.HashCpuCost + JoinCostParams.HashInsertCost); + var probeScanCost = EstimateScanCost(probeCardinality, probeMultiplier); + var probeHashCost = probeCardinality + * (JoinCostParams.HashCpuCost + JoinCostParams.HashLookupCost) + * probeMultiplier; + var visitCost = rowsVisited * JoinCostParams.CpuCostPerRow; + var unmatchedBuildCost = keepsUnmatchedBuildRows + ? buildCardinality * JoinCostParams.CpuCostPerRow + : 0.0; + var spillCost = 0.0; + if (willSpill) + { + var buildPages = Math.Ceiling(buildCardinality / JoinCostParams.RowsPerTablePage); + var probePages = Math.Ceiling(probeCardinality / JoinCostParams.RowsPerTablePage); + spillCost = (buildPages + probePages) * 2.0 * probeMultiplier; + } + + return buildCost + probeScanCost + probeHashCost + visitCost + unmatchedBuildCost + spillCost; + } + + /// + /// access_method.rs:687-707 cost_with_where_work: the CPU spent evaluating the + /// WHERE terms that become ready at one loop. Terms the access path consumes run + /// once per input or output row (whichever is larger); the rest run once per output row. + /// + public static double EstimateWhereWork( + double inputCardinality, + double rowsPerOuterRow, + int consumedSteps, + int remainingSteps) + { + var outputRows = inputCardinality * rowsPerOuterRow; + var usedRows = Math.Max(inputCardinality, outputRows); + var work = usedRows * consumedSteps + outputRows * remainingSteps; + return work * TursoCostParams.CpuCostPerWhereStep; + } +} + +/// cost.rs:68-79 IndexInfo. +internal readonly record struct TursoIndexInfo( + bool Unique, + int ColumnCount, + bool Covering, + double RowsPerLeafPage); diff --git a/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs b/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs index bcbeeb96..99956a9f 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs @@ -462,6 +462,15 @@ private bool TryDescribeCorrelatedAggregateSubqueryPlan( return true; } + // Semi/anti joins whose inner access the ported cost model planned: describe exactly the + // access GetSemiOrAntiJoinRows runs (EmbeddedDatabase.SemiAntiJoinAccess.cs). + if (TryDescribePlannedSemiAntiJoins(rewritten, context, out var plannedRows, out var plannedOps)) + { + result = new ExecutionResult(ExplainQueryPlanColumns(), plannedRows, 0); + ops = plannedOps; + return true; + } + // EXISTS/NOT EXISTS unnested into an internal semi/anti join (unnest.rs's // try_rewrite_exists): the inner table's columns are never visible past a semi/anti join // (JoinKind.ProducesLeftShapeOnly), so it is used only for the correlation test itself. @@ -724,6 +733,131 @@ private static bool IsColumnQualifiedBy(Expression expression, string qualifier) && column.Qualifier is not null && string.Equals(column.Qualifier, qualifier, StringComparison.OrdinalIgnoreCase); + /// + /// Describes a chain of semi/anti joins over one outer base table when every inner access + /// was planned by . The outer read, the inner + /// accesses and their order are the ones GetSemiOrAntiJoinRows executes: the outer + /// table first (its pushed WHERE may select an index), then each inner search, except that + /// a hash anti join reports its probe as HASH JOIN ahead of the build scan, the way + /// Turso's EQP lists a hash join. + /// + private bool TryDescribePlannedSemiAntiJoins( + SelectStatement rewritten, + QueryContext context, + out List rows, + out List ops) + { + rows = []; + ops = []; + if (rewritten.Source is not JoinTableSource { Kind: JoinKind.Semi or JoinKind.Anti } top) + return false; + + var chain = new List(); + TableSource current = top; + while (current is JoinTableSource { Kind: JoinKind.Semi or JoinKind.Anti } link) + { + chain.Add(link); + current = link.Left; + } + + if (current is not NamedTableSource outer || !IsPlainEqpBaseTable(outer, context)) + return false; + chain.Reverse(); + + var leftPredicate = rewritten.Where is null + ? null + : SplitJoinSidePredicates(rewritten.Where, top, context, IsPushableMethodFunction).Left; + var accesses = new List(chain.Count); + foreach (var join in chain) + { + if (PlanSemiOrAntiInnerAccess(join, leftPredicate, context) is not { } access) + return false; + accesses.Add(access); + } + + var outerSelect = new SelectStatement( + Distinct: false, + Projections: [], + Source: outer, + Where: leftPredicate, + GroupBy: [], + Having: null, + NamedWindows: [], + OrderBy: [], + Limit: null, + Offset: null); + var outerPlan = leftPredicate is null ? null : TryPlanManagedIndexScan(outerSelect, context); + var outerDetail = outerPlan is null + ? $"SCAN {outer.Name}" + (outer.Alias is null ? string.Empty : $" AS {outer.Alias}") + : FormatManagedIndexExplainDetail(outerPlan, outerSelect); + var outerOp = outerPlan is null + ? (EqpJsonOp)new EqpJsonScanOp(outer.Name, outer.Alias, IndexName: null, Covering: false) + : BuildIndexScanOp(outerPlan, outerSelect); + + var planRows = new List(); + var planOps = new List(); + void Add(string detail, EqpJsonOp op) + { + planRows.Add(PlanRow(planRows.Count + 1, 0, detail)); + planOps.Add(op); + } + + var startIndex = 0; + if (accesses[0].Kind == SemiAntiInnerAccessKind.HashAnti) + { + var probe = accesses[0].Inner; + Add( + $"HASH JOIN {probe.Name}" + (probe.Alias is null ? string.Empty : $" AS {probe.Alias}"), + new EqpJsonHashJoinOp(probe.Name, probe.Alias, "anti")); + startIndex = 1; + } + + Add(outerDetail, outerOp); + for (var index = startIndex; index < accesses.Count; index++) + { + var access = accesses[index]; + var inner = access.Inner; + var alias = inner.Alias ?? inner.Name; + var joinMarker = chain[index].Kind == JoinKind.Semi ? "semi" : "anti"; + var constraints = access.ConstraintTexts; + switch (access.Kind) + { + case SemiAntiInnerAccessKind.DeclaredIndexSearch: + Add( + $"SEARCH {alias} USING {(access.Covering ? "COVERING " : string.Empty)}INDEX {access.Index!.Name} ({string.Join(" AND ", constraints)})", + new EqpJsonSearchOp(inner.Name, inner.Alias, access.Index.Name, access.Covering, constraints, Join: joinMarker)); + break; + case SemiAntiInnerAccessKind.DeclaredIndexScan: + Add( + $"SCAN {inner.Name}" + (inner.Alias is null ? string.Empty : $" AS {inner.Alias}") + + $" USING {(access.Covering ? "COVERING " : string.Empty)}INDEX {access.Index!.Name}", + new EqpJsonScanOp(inner.Name, inner.Alias, access.Index.Name, access.Covering, joinMarker)); + break; + case SemiAntiInnerAccessKind.EphemeralIndex: + { + var indexName = $"ephemeral_{inner.Name}_t{index * 2 + 3}"; + Add( + $"SEARCH {alias} USING COVERING INDEX {indexName} ({string.Join(" AND ", constraints)})", + new EqpJsonSearchOp(inner.Name, inner.Alias, indexName, Covering: true, constraints, Join: joinMarker, Ephemeral: true)); + break; + } + + case SemiAntiInnerAccessKind.TableScan: + Add( + $"SCAN {inner.Name}" + (inner.Alias is null ? string.Empty : $" AS {inner.Alias}"), + new EqpJsonScanOp(inner.Name, inner.Alias, IndexName: null, Covering: false, joinMarker)); + break; + default: + // Only the first join of a chain can hash its (plain base table) outer input. + return false; + } + } + + rows = planRows; + ops = planOps; + return true; + } + private static bool TryDescribeSemiAntiJoinChain( TableSource? source, QueryContext context, diff --git a/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs b/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs new file mode 100644 index 00000000..374ff5ed --- /dev/null +++ b/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs @@ -0,0 +1,1100 @@ +using Ahtola.Core.Compilation.JoinOrdering; +using Ahtola.Core.Parsing; + +namespace Ahtola.Core; + +/// +/// Cost-based access selection for the inner table of the internal semi/anti joins the +/// correlated-subquery rewrite produces (EmbeddedDatabase.SubqueryRewrites.cs). +/// +/// This ports the part of Turso v0.8.1's join planner that decides how the inner loop of a +/// two-table semi or anti join reads its table: choose_best_btree_candidate and the +/// temporary-index alternative of find_best_access_method_for_btree +/// (optimizer/access_method.rs:195-457, 809-987), the hash-join alternative of +/// join_lhs_and_rhs (optimizer/join.rs:586-1000) with the try_hash_join_access_method +/// / should_not_use_hash_join gates (access_method.rs:1294-1537), and the constraint +/// selectivities of estimate_selectivity (optimizer/constraints.rs:367-437). Every cost +/// is computed with , so the choice is upstream's own. +/// +/// +/// The selected is consumed by both +/// GetSemiOrAntiJoinRows (which runs it) and the EXPLAIN QUERY PLAN describer +/// (which prints it), so the plan text always names the strategy that executes. A shape the +/// planner does not model returns and keeps the legacy access path and +/// its legacy description. +/// +/// +public sealed partial class EmbeddedDatabase +{ + internal enum SemiAntiInnerAccessKind + { + /// A declared index searched by an equality prefix and/or one range. + DeclaredIndexSearch, + + /// A full scan of a declared index (cheaper than the table itself). + DeclaredIndexScan, + + /// An automatic (ephemeral) index built over the inner table once. + EphemeralIndex, + + /// A full scan of the inner table for every outer row. + TableScan, + + /// + /// A left anti hash join: the outer rows are hashed on the join key, the inner table + /// is scanned once as the probe, and outer rows no probe row matched are emitted. + /// + HashAnti, + } + + /// A comparison operator as seen from the constrained inner column. + internal enum TursoConstraintOperator + { + Equal, + Is, + NotEqual, + IsNot, + Less, + LessOrEqual, + Greater, + GreaterOrEqual, + } + + /// + /// One condition conjunct that compares an inner column with an expression that reads no + /// inner column (Turso's Constraint, constraints.rs:43-104). + /// + private sealed record SemiAntiConstraint( + int ConjunctIndex, + Expression Conjunct, + int ColumnOrdinal, + TursoConstraintOperator Operator, + Expression ColumnSide, + Expression Value, + bool DependsOnOuter, + bool NullMatching, + double Selectivity, + string ComparisonCollation, + bool IndexAffinityOk, + bool UsesCustomCollation); + + /// One index column of a seek key (Turso's RangeConstraintRef). + private sealed record SemiAntiSeekTerm( + int ColumnOrdinal, + string ColumnName, + SemiAntiConstraint? Equality, + SemiAntiConstraint? Lower, + SemiAntiConstraint? Upper); + + /// The access the inner loop of a semi/anti join uses, with its estimates. + private sealed record SemiAntiInnerAccess( + SemiAntiInnerAccessKind Kind, + NamedTableSource Inner, + EmbeddedTable Table, + EmbeddedIndex? Index, + IReadOnlyList SeekTerms, + IReadOnlyList HashKeys, + bool Covering, + double InputRows, + double RowsPerOuterRow, + double Cost) + { + public IReadOnlyList ConstraintTexts { get; } = BuildConstraintTexts(SeekTerms); + + private static IReadOnlyList BuildConstraintTexts(IReadOnlyList terms) + { + var texts = new List(terms.Count); + foreach (var term in terms) + { + if (term.Equality is not null) + { + texts.Add($"{term.ColumnName}=?"); + continue; + } + + if (term.Lower is { } lower) + texts.Add($"{term.ColumnName}{(lower.Operator == TursoConstraintOperator.Greater ? ">" : ">=")}?"); + if (term.Upper is { } upper) + texts.Add($"{term.ColumnName}{(upper.Operator == TursoConstraintOperator.Less ? "<" : "<=")}?"); + } + + return texts; + } + } + + /// The outer base table a semi/anti chain filters (its left-most leaf). + private static NamedTableSource? GetSemiAntiOuterLeaf(TableSource source) + => source switch + { + NamedTableSource named => named, + JoinTableSource { Kind: JoinKind.Semi or JoinKind.Anti } join => GetSemiAntiOuterLeaf(join.Left), + _ => null, + }; + + /// + /// Plans the inner access of one semi/anti join. is the + /// part of the outer WHERE pushed onto the outer table (the same predicate + /// GetSemiOrAntiJoinRows receives); it only feeds the outer row estimate. + /// + private SemiAntiInnerAccess? PlanSemiOrAntiInnerAccess( + JoinTableSource join, + Expression? leftPredicate, + QueryContext context) + { + if (join.Kind is not (JoinKind.Semi or JoinKind.Anti) + || join.Condition is null + || context.ConcurrentMvStore is not null + || !TryGetDecorrelationBaseTable(join.Right, context, out var inner, out var innerQualifier) + || GetSemiAntiOuterLeaf(join.Left) is not { } outerLeaf + || !TryGetDecorrelationBaseTable(outerLeaf, context, out _, out var outerQualifier) + || string.Equals(innerQualifier, outerQualifier, StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + var table = context.Tables[inner.Name]; + var outerTable = context.Tables[outerLeaf.Name]; + var innerColumns = new HashSet(table.Columns, StringComparer.OrdinalIgnoreCase); + var conjuncts = IndexExpressionSemantics.SplitConjuncts(join.Condition); + var constraints = new List(); + var usedColumns = new HashSet(); + var steps = new int[conjuncts.Count]; + for (var index = 0; index < conjuncts.Count; index++) + { + var conjunct = conjuncts[index]; + steps[index] = CountTursoWhereSteps(conjunct); + if (!ForEachColumnReference(conjunct, column => + { + if (!IsInnerColumnReference(column, innerQualifier, innerColumns)) + return true; + var name = column.UnqualifiedName ?? column.Name; + if (IsRowIdAlias(name) && !table.TryGetColumnIndex(name, out _)) + return false; + if (table.TryGetColumnIndex(name, out var ordinal)) + usedColumns.Add(ordinal); + return true; + })) + { + // A rowid reference (or a node the walker does not model) is outside what the + // ported candidate model describes; keep the legacy access path. + return null; + } + + if (TryDescribeSemiAntiConstraint( + index, + conjunct, + table, + innerQualifier, + innerColumns, + outerLeaf, + outerTable, + context, + out var constraint)) + { + constraints.Add(constraint); + } + } + + // Turso's rowid candidate would serve an INTEGER PRIMARY KEY equality with a rowid + // seek, which this planner does not model; leave that shape to the legacy path. + if (table.HasRowidAlias + && constraints.Any(constraint => constraint.ColumnOrdinal == table.RowidAliasColumnIndex)) + { + return null; + } + + // Stable partition: equalities first (constraints.rs:1165-1171). + constraints = [.. constraints.Where(static c => IsTursoEqualityOperator(c.Operator)), + .. constraints.Where(static c => !IsTursoEqualityOperator(c.Operator))]; + + var inputRows = EstimateTursoTableRows(outerLeaf.Name, context) + * EstimateTursoLocalSelectivity(leftPredicate, outerLeaf, outerTable, context); + var innerRows = EstimateTursoTableRows(inner.Name, context); + var covering = usedColumns.Count > 0; + + // choose_best_btree_candidate (access_method.rs:195-457). + var forcedIndex = inner.IndexDirective is IndexedByDirective indexedBy ? indexedBy.IndexName : null; + var candidateIndexes = inner.IndexDirective is NotIndexedDirective + ? [] + : table.Indexes + .Where(index => !index.IsMethodIndex + && !index.IsPartial + && index.Columns.Count > 0 + && (forcedIndex is null + || string.Equals(index.Name, forcedIndex, StringComparison.OrdinalIgnoreCase))) + .ToArray(); + if (forcedIndex is not null && candidateIndexes.Length == 0) + return null; + + SemiAntiInnerAccess best; + double bestCost; + double bestAdjustedOutput = double.MaxValue; + if (forcedIndex is null) + { + bestCost = TursoCostModel.EstimateScanCost(innerRows, inputRows); + best = new SemiAntiInnerAccess( + SemiAntiInnerAccessKind.TableScan, + inner, + table, + Index: null, + SeekTerms: [], + HashKeys: [], + Covering: false, + inputRows, + RowsPerOuterRow: innerRows, + bestCost); + } + else + { + bestCost = double.MaxValue; + best = null!; + } + + foreach (var index in candidateIndexes) + { + var terms = BuildSemiAntiIndexSeekTerms(index, table, constraints); + var indexCovering = covering + && index.Columns.All(static column => !column.IsExpression) + && usedColumns.All(ordinal => index.Columns.Any(column => column.ColumnIndex == ordinal)); + var info = new TursoIndexInfo( + index.Unique, + index.Columns.Count, + indexCovering, + TursoCostModel.IndexLeafRowsPerPage(index.Columns.Count, table.Columns.Length, table.HasRowidAlias)); + var cost = EstimateTursoScanOrSeekCost(info, terms, inputRows, innerRows, table, index, context); + var adjustedOutput = constraints + .Where(constraint => !terms.Any(term => ReferenceEquals(term.Equality, constraint) + || ReferenceEquals(term.Lower, constraint) + || ReferenceEquals(term.Upper, constraint)) + && constraint.Operator is TursoConstraintOperator.Equal + or TursoConstraintOperator.Less + or TursoConstraintOperator.LessOrEqual + or TursoConstraintOperator.Greater + or TursoConstraintOperator.GreaterOrEqual) + .Aggregate(1.0, static (product, constraint) => product * constraint.Selectivity); + var costsEqual = Math.Abs(cost - bestCost) < 1e-9; + if (cost < bestCost || costsEqual && adjustedOutput < bestAdjustedOutput - 1e-12) + { + bestCost = cost; + bestAdjustedOutput = adjustedOutput; + best = new SemiAntiInnerAccess( + terms.Count == 0 ? SemiAntiInnerAccessKind.DeclaredIndexScan : SemiAntiInnerAccessKind.DeclaredIndexSearch, + inner, + table, + index, + terms, + HashKeys: [], + indexCovering, + inputRows, + RowsPerOuterRow: terms.Count == 0 + ? innerRows + : EstimateTursoRowsPerSeek(info, terms, innerRows, table, index, context), + cost); + } + } + + var bestWithWhere = best.Cost + EstimateSemiAntiWhereWork(best, steps, inputRows); + + // The temporary index is only considered over a plain full table scan + // (access_method.rs:903-987). + if (best.Kind == SemiAntiInnerAccessKind.TableScan) + { + var terms = BuildSemiAntiEphemeralSeekTerms(table, constraints); + if (terms.Count != 0) + { + var columnCount = usedColumns.Count; + var info = new TursoIndexInfo( + Unique: false, + columnCount, + Covering: true, + TursoCostModel.IndexLeafRowsPerPage(columnCount, table.Columns.Length, table.HasRowidAlias)); + var rowsPerSeek = EstimateTursoRowsPerSeek(info, terms, innerRows, table, index: null, context); + var cost = TursoCostModel.EstimateScanCost(innerRows, 1.0) + + TursoCostModel.EstimateEphemeralIndexBuildCost(innerRows) + + inputRows * JoinCostParams.CpuCostPerSeek + + inputRows * rowsPerSeek * JoinCostParams.CpuCostPerRow; + var ephemeral = new SemiAntiInnerAccess( + SemiAntiInnerAccessKind.EphemeralIndex, + inner, + table, + Index: null, + terms, + HashKeys: [], + Covering: true, + inputRows, + rowsPerSeek, + cost); + var ephemeralWithWhere = cost + EstimateSemiAntiWhereWork(ephemeral, steps, inputRows); + if (ephemeralWithWhere < bestWithWhere) + { + best = ephemeral; + bestWithWhere = ephemeralWithWhere; + } + } + } + + if (join.Kind == JoinKind.Anti + && join.Left is NamedTableSource buildSource + && TryPlanSemiAntiHashAnti( + join, + buildSource, + outerTable, + leftPredicate, + inner, + table, + constraints, + candidateIndexes, + best, + innerRows, + inputRows, + context) is { } hash) + { + var hashWithWhere = hash.Cost + EstimateSemiAntiWhereWork(hash, steps, inputRows); + if (hashWithWhere < bestWithWhere) + best = hash; + } + + return best; + } + + /// + /// The left anti hash join alternative (join.rs:586-1000 with access_method.rs:1294-1537), + /// restricted to what the managed executor runs: the build side is the plain outer table + /// read by a full scan, and every hash key compares an inner column with an outer + /// expression under a built-in collation. + /// + private SemiAntiInnerAccess? TryPlanSemiAntiHashAnti( + JoinTableSource join, + NamedTableSource buildSource, + EmbeddedTable buildTable, + Expression? leftPredicate, + NamedTableSource probeSource, + EmbeddedTable probeTable, + IReadOnlyList constraints, + IReadOnlyList probeIndexes, + SemiAntiInnerAccess bestProbeAccess, + double probeRows, + double inputRows, + QueryContext context) + { + // should_not_use_hash_join: both sides need a rowid, a self-join keeps the nested loop, + // and INDEXED BY / NOT INDEXED on either side is honoured. + if (!buildTable.HasRowid + || !probeTable.HasRowid + || ReferenceEquals(buildTable, probeTable) + || buildSource.IndexDirective is not null + || probeSource.IndexDirective is not null) + { + return null; + } + + // A selective declared-index probe seek is preferred (rhs_has_selective_seek). + if (bestProbeAccess.Kind == SemiAntiInnerAccessKind.DeclaredIndexSearch) + return null; + + // The build read must be a plain table scan; a filtered index read would need the + // materialized build input this executor does not implement. + if (leftPredicate is not null + && TryPlanManagedIndexScan( + new SelectStatement( + Distinct: false, + Projections: [], + Source: buildSource, + Where: leftPredicate, + GroupBy: [], + Having: null, + NamedWindows: [], + OrderBy: [], + Limit: null, + Offset: null), + context) is not null) + { + return null; + } + + var keys = constraints + .Where(constraint => constraint.Operator == TursoConstraintOperator.Equal + && constraint.DependsOnOuter + && !constraint.UsesCustomCollation + && IsHashableJoinKeyCollation(constraint.ComparisonCollation) + && !IsUnsafeCompiledCollation(constraint.ComparisonCollation) + && IsSemiAntiHashableOuterValue(constraint.Value)) + .ToArray(); + if (keys.Length == 0) + return null; + + // probe_index_can_seek_join_key: an index that can seek the probe join key wins. + foreach (var key in keys) + { + if (probeIndexes.Any(index => BuildSemiAntiIndexSeekTerms(index, probeTable, constraints) + .Any(term => ReferenceEquals(term.Equality, key)))) + { + return null; + } + } + + // A plain build column that an index could seek keeps the index path unless the + // probe side has no constant filter the hash would have to give up + // (can_replace_build_index_with_hash, access_method.rs:1360-1389). + var hashCanReplaceBuildIndex = !constraints.Any(static constraint => !constraint.DependsOnOuter); + if (!hashCanReplaceBuildIndex) + { + foreach (var key in keys) + { + if (key.Value is ColumnExpression { BooleanKeyword: null } buildColumn + && buildTable.TryGetColumnIndex(buildColumn.UnqualifiedName ?? buildColumn.Name, out var buildOrdinal) + && (buildTable.HasRowidAlias && buildTable.RowidAliasColumnIndex == buildOrdinal + || buildTable.Indexes.Any(index => !index.IsMethodIndex + && index.Columns.Any(column => !column.IsExpression && column.ColumnIndex == buildOrdinal)))) + { + return null; + } + } + } + + var buildRows = EstimateTursoTableRows(buildSource.Name, context) + * EstimateTursoLocalSelectivity(leftPredicate, buildSource, buildTable, context); + const double rowsPerOuterRow = 1.0; // HashJoinType::LeftAnti (access_method.rs:1416) + var cost = TursoCostModel.EstimateHashJoinCost( + buildRows, + probeRows, + rowsVisited: buildRows * rowsPerOuterRow, + keepsUnmatchedBuildRows: true, + probeMultiplier: 1.0); + return new SemiAntiInnerAccess( + SemiAntiInnerAccessKind.HashAnti, + probeSource, + probeTable, + Index: null, + SeekTerms: [], + keys, + Covering: false, + inputRows, + rowsPerOuterRow, + cost); + } + + /// + /// An outer hash-key expression the managed hash build can canonicalize the same way the + /// transient equality lookup does: a column, or an expression without CAST/COLLATE, and + /// never a function call or subquery. + /// + private static bool IsSemiAntiHashableOuterValue(Expression value) + { + var stripped = value; + while (stripped is UnaryExpression { Operator: UnaryOperator.Plus } unary) + stripped = unary.Operand; + if (stripped is ColumnExpression { BooleanKeyword: null }) + return true; + + return !ContainsCastOrCollate(stripped) && IsScanConstantExpression(stripped, []); + } + + /// cost_with_where_work (access_method.rs:687-707) for the inner loop. + private static double EstimateSemiAntiWhereWork( + SemiAntiInnerAccess access, + int[] steps, + double inputRows) + { + var consumed = new HashSet(); + foreach (var term in access.SeekTerms) + { + if (term.Equality is { } equality) + consumed.Add(equality.ConjunctIndex); + if (term.Lower is { } lower) + consumed.Add(lower.ConjunctIndex); + if (term.Upper is { } upper) + consumed.Add(upper.ConjunctIndex); + } + + foreach (var key in access.HashKeys) + consumed.Add(key.ConjunctIndex); + + var consumedSteps = 0; + var remainingSteps = 0; + for (var index = 0; index < steps.Length; index++) + { + if (consumed.Contains(index)) + consumedSteps += steps[index]; + else + remainingSteps += steps[index]; + } + + return TursoCostModel.EstimateWhereWork(inputRows, access.RowsPerOuterRow, consumedSteps, remainingSteps); + } + + /// cost.rs:19-50 where_expr_steps. + private static int CountTursoWhereSteps(Expression expression) + { + var steps = 0; + var pending = new Stack(); + pending.Push(expression); + while (pending.Count > 0) + { + switch (pending.Pop()) + { + case ColumnExpression: + case LiteralExpression: + case ParameterExpression: + break; + case CollationExpression collation: + pending.Push(collation.Expression); + break; + case BetweenExpression between: + steps += 2; + pending.Push(between.Value); + pending.Push(between.Lower); + pending.Push(between.Upper); + break; + case InExpression inList: + steps += Math.Max(inList.Values.Count, 1); + pending.Push(inList.Value); + foreach (var value in inList.Values) + pending.Push(value); + break; + case CaseExpression caseExpression: + steps += Math.Max(caseExpression.Clauses.Count, 1); + if (caseExpression.Operand is not null) + pending.Push(caseExpression.Operand); + foreach (var clause in caseExpression.Clauses) + { + pending.Push(clause.When); + pending.Push(clause.Then); + } + + if (caseExpression.Else is not null) + pending.Push(caseExpression.Else); + break; + case BinaryExpression binary: + steps++; + pending.Push(binary.Left); + pending.Push(binary.Right); + break; + case UnaryExpression unary: + steps++; + pending.Push(unary.Operand); + break; + case CastExpression cast: + steps++; + pending.Push(cast.Expression); + break; + case LikeExpression like: + steps++; + pending.Push(like.Value); + pending.Push(like.Pattern); + if (like.Escape is not null) + pending.Push(like.Escape); + break; + case FunctionExpression function: + steps++; + foreach (var argument in function.Arguments) + pending.Push(argument); + break; + default: + steps++; + break; + } + } + + return Math.Max(steps, 1); + } + + private static bool IsTursoEqualityOperator(TursoConstraintOperator op) + => op is TursoConstraintOperator.Equal or TursoConstraintOperator.Is; + + /// + /// Describes one conjunct as a constraint on an inner column + /// (constraints_from_where_clause, constraints.rs:732-993): a direct comparison with a + /// bare inner column on one side and an expression that reads no inner column on the + /// other. + /// + private bool TryDescribeSemiAntiConstraint( + int conjunctIndex, + Expression conjunct, + EmbeddedTable table, + string innerQualifier, + HashSet innerColumns, + NamedTableSource outerLeaf, + EmbeddedTable outerTable, + QueryContext context, + out SemiAntiConstraint constraint) + { + constraint = null!; + if (conjunct is not BinaryExpression + { + Operator: BinaryOperator.Equal + or BinaryOperator.Is + or BinaryOperator.NotEqual + or BinaryOperator.IsNot + or BinaryOperator.LessThan + or BinaryOperator.LessThanOrEqual + or BinaryOperator.GreaterThan + or BinaryOperator.GreaterThanOrEqual, + } binary) + { + return false; + } + + // `x IS TRUE` / `x IS FALSE` is a truth test, not a comparison (constraints.rs:743-752). + if (binary.Operator is BinaryOperator.Is or BinaryOperator.IsNot + && (binary.Right is ColumnExpression { BooleanKeyword: not null } + || binary.Left is ColumnExpression { BooleanKeyword: not null })) + { + return false; + } + + Expression columnSide; + Expression value; + bool columnOnLeft; + if (IsBareInnerColumn(binary.Left) && !ReferencesInner(binary.Right)) + { + columnSide = binary.Left; + value = binary.Right; + columnOnLeft = true; + } + else if (IsBareInnerColumn(binary.Right) && !ReferencesInner(binary.Left)) + { + columnSide = binary.Right; + value = binary.Left; + columnOnLeft = false; + } + else + { + return false; + } + + var column = (ColumnExpression)columnSide; + if (!table.TryGetColumnIndex(column.UnqualifiedName ?? column.Name, out var ordinal)) + return false; + + var op = binary.Operator switch + { + BinaryOperator.Equal => TursoConstraintOperator.Equal, + BinaryOperator.Is => TursoConstraintOperator.Is, + BinaryOperator.NotEqual => TursoConstraintOperator.NotEqual, + BinaryOperator.IsNot => TursoConstraintOperator.IsNot, + BinaryOperator.LessThan => columnOnLeft ? TursoConstraintOperator.Less : TursoConstraintOperator.Greater, + BinaryOperator.LessThanOrEqual => columnOnLeft ? TursoConstraintOperator.LessOrEqual : TursoConstraintOperator.GreaterOrEqual, + BinaryOperator.GreaterThan => columnOnLeft ? TursoConstraintOperator.Greater : TursoConstraintOperator.Less, + _ => columnOnLeft ? TursoConstraintOperator.GreaterOrEqual : TursoConstraintOperator.LessOrEqual, + }; + + var dependsOnOuter = false; + ForEachColumnReference(value, _ => + { + dependsOnOuter = true; + return true; + }); + + // An IS whose value may be NULL matches NULL keys (Constraint::null_matching). + var nullMatching = op == TursoConstraintOperator.Is + && value is not LiteralExpression { Value.Kind: not SqlValueKind.Null }; + var selectivityOperator = op == TursoConstraintOperator.Is && !nullMatching + ? TursoConstraintOperator.Equal + : op; + var selectivity = EstimateTursoColumnSelectivity(table, ordinal, selectivityOperator, context); + + var columnDefinition = table.ColumnDefinitions[ordinal]; + var outerDefinition = ResolveSemiAntiOuterColumnDefinition(value, outerLeaf, outerTable); + var leftExplicit = GetExplicitCollation(binary.Left); + var rightExplicit = GetExplicitCollation(binary.Right); + var columnDeclared = NormalizeDeclaredCollation(columnDefinition.Collation) ?? "BINARY"; + var valueDeclared = outerDefinition is null + ? null + : NormalizeDeclaredCollation(outerDefinition.Collation) ?? "BINARY"; + var collation = (leftExplicit + ?? rightExplicit + ?? (columnOnLeft ? columnDeclared : valueDeclared) + ?? (columnOnLeft ? valueDeclared : columnDeclared) + ?? "BINARY").ToUpperInvariant(); + + var columnAffinity = GetJoinKeyAffinity(columnDefinition); + var valueAffinity = value is CastExpression cast + ? EmbeddedTable.GetAffinity(cast.TypeName) + : GetJoinKeyAffinity(outerDefinition); + var comparisonAffinity = (columnAffinity, valueAffinity) switch + { + ({ } left, { } right) => IsNumericAffinity(left) || IsNumericAffinity(right) + ? ColumnAffinity.Numeric + : ColumnAffinity.Blob, + ({ } left, null) => left, + (null, { } right) => right, + _ => ColumnAffinity.Blob, + }; + var affinityOk = comparisonAffinity switch + { + ColumnAffinity.Blob => true, + ColumnAffinity.Text => columnAffinity == ColumnAffinity.Text, + _ => IsNumericAffinity(columnAffinity), + }; + + constraint = new SemiAntiConstraint( + conjunctIndex, + conjunct, + ordinal, + op, + columnSide, + value, + dependsOnOuter, + nullMatching, + selectivity, + collation, + affinityOk, + UsesCustomCollation: (leftExplicit is not null && !IsBuiltInCollation(leftExplicit)) + || (rightExplicit is not null && !IsBuiltInCollation(rightExplicit))); + return true; + + bool IsBareInnerColumn(Expression expression) + => expression is ColumnExpression { BooleanKeyword: null } bare + && IsInnerColumnReference(bare, innerQualifier, innerColumns); + + bool ReferencesInner(Expression expression) + { + var referencesInner = false; + var modelled = ForEachColumnReference(expression, reference => + { + if (IsInnerColumnReference(reference, innerQualifier, innerColumns)) + referencesInner = true; + return true; + }); + return referencesInner || !modelled; + } + + static string? GetExplicitCollation(Expression expression) + => expression is CollationExpression collated ? collated.Name.ToUpperInvariant() : null; + } + + /// The declared column an outer operand names, when it names one directly. + private static EmbeddedColumn? ResolveSemiAntiOuterColumnDefinition( + Expression value, + NamedTableSource outerLeaf, + EmbeddedTable outerTable) + { + while (value is UnaryExpression { Operator: UnaryOperator.Plus } unary) + value = unary.Operand; + while (value is CollationExpression collation) + value = collation.Expression; + if (value is not ColumnExpression { BooleanKeyword: null } column) + return null; + if (column.Qualifier is { } qualifier + && !string.Equals(qualifier, outerLeaf.Alias ?? outerLeaf.Name, StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + return outerTable.TryGetColumnIndex(column.UnqualifiedName ?? column.Name, out var ordinal) + ? outerTable.ColumnDefinitions[ordinal] + : null; + } + + /// + /// Whether a constraint may key an index column: equality or range, a collation that + /// matches the column (constraints.rs:1180-1214), and sqlite3IndexAffinityOk. + /// + private static bool CanSemiAntiConstraintSeek(SemiAntiConstraint constraint, EmbeddedTable table) + => constraint.IndexAffinityOk + && string.Equals( + constraint.ComparisonCollation, + NormalizeDeclaredCollation(table.ColumnDefinitions[constraint.ColumnOrdinal].Collation) ?? "BINARY", + StringComparison.OrdinalIgnoreCase); + + /// + /// usable_constraints_for_lhs_mask (constraints.rs:1467-1586) over a declared index: a + /// contiguous equality prefix followed by at most one range column. + /// + private static IReadOnlyList BuildSemiAntiIndexSeekTerms( + EmbeddedIndex index, + EmbeddedTable table, + IReadOnlyList constraints) + { + var terms = new List(); + foreach (var indexColumn in index.Columns) + { + if (indexColumn.IsExpression) + break; + var indexCollation = (indexColumn.Collation + ?? NormalizeDeclaredCollation(table.ColumnDefinitions[indexColumn.ColumnIndex].Collation) + ?? "BINARY").ToUpperInvariant(); + var tableCollation = (NormalizeDeclaredCollation(table.ColumnDefinitions[indexColumn.ColumnIndex].Collation) + ?? "BINARY").ToUpperInvariant(); + if (!string.Equals(indexCollation, tableCollation, StringComparison.Ordinal)) + break; + + if (!TryBuildSemiAntiSeekTerm(indexColumn.ColumnIndex, indexColumn.Name, table, constraints, out var term)) + break; + terms.Add(term); + if (term.Equality is null) + break; + } + + return terms; + } + + /// + /// The automatic-index key (automatic_index_terms / ordered_ephemeral_key_columns, + /// constraints.rs:292-323, 1608-1637): equality columns in table order, then range-only + /// columns, consumed with the same prefix rule as a declared index. + /// + private static IReadOnlyList BuildSemiAntiEphemeralSeekTerms( + EmbeddedTable table, + IReadOnlyList constraints) + { + var usable = constraints + .Where(constraint => !constraint.UsesCustomCollation && CanSemiAntiConstraintSeek(constraint, table)) + .ToArray(); + var equalityColumns = usable + .Where(static constraint => IsTursoEqualityOperator(constraint.Operator)) + .Select(static constraint => constraint.ColumnOrdinal) + .Distinct() + .Order() + .ToArray(); + var rangeColumns = usable + .Where(static constraint => constraint.Operator is TursoConstraintOperator.Less + or TursoConstraintOperator.LessOrEqual + or TursoConstraintOperator.Greater + or TursoConstraintOperator.GreaterOrEqual) + .Select(static constraint => constraint.ColumnOrdinal) + .Where(ordinal => !equalityColumns.Contains(ordinal)) + .Distinct() + .Order(); + + var terms = new List(); + foreach (var ordinal in equalityColumns.Concat(rangeColumns)) + { + if (!TryBuildSemiAntiSeekTerm(ordinal, table.Columns[ordinal], table, usable, out var term)) + break; + terms.Add(term); + if (term.Equality is null) + break; + } + + return terms; + } + + private static bool TryBuildSemiAntiSeekTerm( + int ordinal, + string columnName, + EmbeddedTable table, + IReadOnlyList constraints, + out SemiAntiSeekTerm term) + { + SemiAntiConstraint? equality = null; + SemiAntiConstraint? lower = null; + SemiAntiConstraint? upper = null; + foreach (var constraint in constraints) + { + if (constraint.ColumnOrdinal != ordinal || !CanSemiAntiConstraintSeek(constraint, table)) + continue; + + switch (constraint.Operator) + { + case TursoConstraintOperator.Equal or TursoConstraintOperator.Is: + equality ??= constraint; + break; + case TursoConstraintOperator.Greater or TursoConstraintOperator.GreaterOrEqual: + lower ??= constraint; + break; + case TursoConstraintOperator.Less or TursoConstraintOperator.LessOrEqual: + upper ??= constraint; + break; + } + } + + term = equality is not null + ? new SemiAntiSeekTerm(ordinal, columnName, equality, null, null) + : new SemiAntiSeekTerm(ordinal, columnName, null, lower, upper); + return equality is not null || lower is not null || upper is not null; + } + + /// cost.rs:461-519 estimate_cost_for_scan_or_seek for a declared index. + private static double EstimateTursoScanOrSeekCost( + TursoIndexInfo info, + IReadOnlyList terms, + double inputRows, + double baseRows, + EmbeddedTable table, + EmbeddedIndex index, + QueryContext context) + { + var treeDepth = TursoCostModel.EstimateBtreeDepth(baseRows, JoinCostParams.RowsPerTablePage); + if (IsTursoUniquePointLookup(info, terms)) + return TursoCostModel.EstimateIndexCost(baseRows, treeDepth, info, inputRows, 1.0); + + var rowsPerSeek = EstimateTursoRowsPerSeek(info, terms, baseRows, table, index, context); + var cost = TursoCostModel.EstimateIndexCost(baseRows, treeDepth, info, inputRows, rowsPerSeek); + return !info.Covering && terms.Count == 0 ? cost * 2.0 : cost; + } + + /// cost.rs:252-264 is_unique_point_lookup. + private static bool IsTursoUniquePointLookup(TursoIndexInfo info, IReadOnlyList terms) + { + var equalityCount = 0; + foreach (var term in terms) + { + if (term.Equality is not { NullMatching: false }) + break; + equalityCount++; + } + + return info.Unique && equalityCount >= info.ColumnCount; + } + + /// cost.rs:325-407 estimate_rows_per_seek. + private static double EstimateTursoRowsPerSeek( + TursoIndexInfo info, + IReadOnlyList terms, + double baseRows, + EmbeddedTable table, + EmbeddedIndex? index, + QueryContext context) + { + var equalityPrefix = terms.TakeWhile(static term => term.Equality is not null).ToArray(); + var joinProbeConstantSelectivity = equalityPrefix.Any(static term => term.Equality!.DependsOnOuter) + ? equalityPrefix + .Where(static term => !term.Equality!.DependsOnOuter) + .Aggregate(1.0, static (product, term) => product * term.Equality!.Selectivity) + : 1.0; + if (IsTursoUniquePointLookup(info, terms)) + return joinProbeConstantSelectivity; + + if (index is not null + && equalityPrefix.Length > 0 + && equalityPrefix.All(static term => !term.Equality!.NullMatching) + && TryGetSqliteStat1PrefixAverage(context, table.Name, index.Name, equalityPrefix.Length, out var prefixAverage)) + { + var rangeSelectivity = terms + .Skip(equalityPrefix.Length) + .Aggregate(1.0, static (product, term) => product + * (term.Lower?.Selectivity ?? 1.0) + * (term.Upper?.Selectivity ?? 1.0)); + return Math.Max(prefixAverage * rangeSelectivity, 1.0) * joinProbeConstantSelectivity; + } + + var multiplier = terms.Aggregate(1.0, static (product, term) => product * (term.Equality is { } equality + ? equality.Selectivity + : (term.Lower?.Selectivity ?? 1.0) * (term.Upper?.Selectivity ?? 1.0))); + return Math.Max(multiplier * baseRows, 1.0) * joinProbeConstantSelectivity; + } + + /// + /// The row estimate the planner starts from (optimizer/mod.rs:2064-2082 + /// base_row_estimate): the sqlite_stat1 row count, else Turso's + /// 1,000,000-row fallback. + /// + private static double EstimateTursoTableRows(string tableName, QueryContext context) + => TryGetSqliteStat1TableRowCount(context, tableName, out var rows) + ? rows + : JoinCostParams.RowsPerTableFallback; + + /// + /// constraints.rs:367-437 estimate_selectivity (with + /// selectivity_index_for_column, constraints.rs:470-501) for a column constraint. + /// + private static double EstimateTursoColumnSelectivity( + EmbeddedTable table, + int ordinal, + TursoConstraintOperator op, + QueryContext context) + { + switch (op) + { + case TursoConstraintOperator.Equal: + { + var hasStats = TryGetSqliteStat1TableRowCount(context, table.Name, out var rowCount); + var uniqueSelectivity = hasStats && rowCount > 0 + ? 1.0 / rowCount + : 1.0 / JoinCostParams.RowsPerTableFallback; + if (table.HasRowidAlias && table.RowidAliasColumnIndex == ordinal) + return uniqueSelectivity; + if (table.ColumnDefinitions[ordinal].PrimaryKey) + return uniqueSelectivity; + + var index = table.Indexes.FirstOrDefault(candidate => + !candidate.IsMethodIndex + && candidate.Columns.Count > 0 + && !candidate.Columns[0].IsExpression + && candidate.Columns[0].ColumnIndex == ordinal + && (candidate.Unique && candidate.Columns.Count == 1 + || !hasStats + || TryGetSqliteStat1PrefixAverage(context, table.Name, candidate.Name, 1, out _))); + if (index is null) + return JoinCostParams.SelectivityEqualityUnindexed; + if (index.Unique && index.Columns.Count == 1) + return uniqueSelectivity; + if (!hasStats) + return JoinCostParams.SelectivityEqualityIndexed; + if (TryGetSqliteStat1IndexTotal(context, table.Name, index.Name, out var total) + && total > 0 + && TryGetSqliteStat1PrefixAverage(context, table.Name, index.Name, 1, out var average)) + { + return (double)average / total; + } + + return JoinCostParams.SelectivityEqualityUnindexed; + } + + case TursoConstraintOperator.Less + or TursoConstraintOperator.LessOrEqual + or TursoConstraintOperator.Greater + or TursoConstraintOperator.GreaterOrEqual: + return JoinCostParams.SelectivityRange; + case TursoConstraintOperator.Is: + return TursoCostParams.SelectivityIsNull; + case TursoConstraintOperator.IsNot: + return TursoCostParams.SelectivityIsNotNull; + default: + return JoinCostParams.SelectivityOther; + } + } + + /// The first number of an index's sqlite_stat1 row (its total rows). + private static bool TryGetSqliteStat1IndexTotal( + QueryContext context, + string tableName, + string indexName, + out long total) + { + total = 0; + return TryFindSqliteStat1Row(context, tableName, indexName, preferAnyIndex: false, out var parts) + && parts.Length >= 1 + && long.TryParse(parts[0], System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out total); + } + + /// + /// The product of the selectivities of a table's own filters (constraints whose other side + /// reads no table): Turso's constraint_output_multipliers for the first loop, and + /// build_self_constraint_selectivity for a hash build (join.rs:72-152, 1187-1208). + /// + private double EstimateTursoLocalSelectivity( + Expression? predicate, + NamedTableSource source, + EmbeddedTable table, + QueryContext context) + { + if (predicate is null) + return 1.0; + + var qualifier = source.Alias ?? source.Name; + var columns = new HashSet(table.Columns, StringComparer.OrdinalIgnoreCase); + var selectivity = 1.0; + foreach (var conjunct in IndexExpressionSemantics.SplitConjuncts(predicate)) + { + if (TryDescribeSemiAntiConstraint( + conjunctIndex: 0, + conjunct, + table, + qualifier, + columns, + source, + table, + context, + out var constraint) + && !constraint.DependsOnOuter) + { + selectivity *= constraint.Selectivity; + } + } + + return Math.Clamp(selectivity, 0.0, 1.0); + } +} diff --git a/src/Ahtola.Core/EmbeddedDatabase.SubqueryRewrites.cs b/src/Ahtola.Core/EmbeddedDatabase.SubqueryRewrites.cs index 40d7652b..e5af9d11 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.SubqueryRewrites.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.SubqueryRewrites.cs @@ -909,16 +909,16 @@ private bool TryRewriteSubqueryTermAsJoin( return false; } - // Every term that reaches out of the subquery must be a plain equality with all - // inner references on one side and all outer references on the other, so it can - // run as a join condition (unnest.rs:1423-1478). + // Every term that reaches out of the subquery must be one direct comparison with + // all inner references on one side and all outer references on the other, so it + // can run as a join condition (can_move_join_term, unnest.rs:1379-1400). if (referencesOuter) { // The equality synthesized for IN always spans both scopes; it is what makes // the join, not evidence that the subquery itself was correlated. if (!ReferenceEquals(conjunct, inEquality)) correlated = true; - if (!IsInnerOuterEquality(conjunct, innerQualifier, innerColumns)) + if (!IsInnerOuterComparison(conjunct, innerQualifier, innerColumns)) return false; } @@ -1090,17 +1090,34 @@ private static bool IsInnerColumnReference( } /// - /// True when the conjunct is inner = outer or outer = inner with each side - /// referencing exactly one of the two scopes (unnest.rs:1448-1478). Any other operator, or - /// a side mixing both scopes, cannot be evaluated as a join key. + /// True when the conjunct is one direct comparison (=, <>, <, + /// <=, >, >=, IS, IS NOT) with one side referencing + /// only the inner scope and the other only the outer scope. Ports v0.8.1 + /// is_inner_outer_comparison (unnest.rs:1402-1436, f7aac1fc5, which widened the + /// v0.8.0 =-only rule). A comparison operator yields NULL exactly when the + /// correlated subquery's WHERE would, and a semi/anti join treats NULL as "no match" the way + /// the subquery's WHERE did, so every listed operator keeps its three-valued answer when it + /// moves into the join condition. A side mixing both scopes still declines. /// - private static bool IsInnerOuterEquality( + private static bool IsInnerOuterComparison( Expression expression, string innerQualifier, HashSet innerColumns) { - if (expression is not BinaryExpression { Operator: BinaryOperator.Equal } equality) + if (expression is not BinaryExpression + { + Operator: BinaryOperator.Equal + or BinaryOperator.NotEqual + or BinaryOperator.LessThan + or BinaryOperator.LessThanOrEqual + or BinaryOperator.GreaterThan + or BinaryOperator.GreaterThanOrEqual + or BinaryOperator.Is + or BinaryOperator.IsNot, + } equality) + { return false; + } if (!TryClassifyJoinTerm(equality.Left, innerQualifier, innerColumns, out var leftInner, out var leftOuter) || !TryClassifyJoinTerm(equality.Right, innerQualifier, innerColumns, out var rightInner, out var rightOuter)) diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index df6a0eca..0df754ea 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -20902,6 +20902,12 @@ bool TryCompileAutomaticSemiJoinSelect( if (outerTarget is null || innerTarget is null) return false; + // This lowering builds an automatic index. When the ported cost model prefers + // another inner access (a declared index search, say), the evaluator runs that + // access instead, so EXPLAIN QUERY PLAN and execution stay the same plan. + if (PlanSemiOrAntiInnerAccess(join, leftPredicate: null, context) is { Kind: not SemiAntiInnerAccessKind.EphemeralIndex }) + return false; + var outerQualifier = outer.Alias ?? outer.Name; var innerQualifier = inner.Alias ?? inner.Name; var keyPairs = new List<(int Outer, int Inner)>(); @@ -38491,24 +38497,35 @@ private SourceData GetSemiOrAntiJoinRows( // declared collation it resolved before the rewrite. var innerContext = EnterCollationSource(context, source.Right); + // The inner access is chosen by the ported Turso join cost model (see + // EmbeddedDatabase.SemiAntiJoinAccess.cs); EXPLAIN QUERY PLAN describes the same choice. + var access = PlanSemiOrAntiInnerAccess(source, leftPredicate, context); + if (access is { Kind: SemiAntiInnerAccessKind.HashAnti }) + return GetHashAntiJoinRows(source, access, left, parameters, innerContext, maximumRows, outerRow); + // The un-rewritten correlated subquery reached its inner table through a cached - // transient hash probe. A matching declared index now takes precedence for the narrow - // conversion-free equality shape above; all other rewrites retain the hash probe. + // transient hash probe. A planned index search or automatic index is served by the same + // statement-cached equality probe (the full condition still decides every match); an + // unmodelled shape keeps the legacy declared-index-then-probe order. SourceData? materializedRight = null; var keepOnMatch = source.Kind == JoinKind.Semi; + var probeAccess = access is null + || access.Kind is SemiAntiInnerAccessKind.DeclaredIndexSearch or SemiAntiInnerAccessKind.EphemeralIndex; var rows = new List(); foreach (var leftRow in left.Rows) { context.CheckInterrupt(); - var probed = source.Condition is null + var probed = source.Condition is null || !probeAccess ? null - : TryGetDeclaredIndexLookupRows( - source.Right, - source.Condition, - parameters, - innerContext, - leftRow) + : (access is null + ? TryGetDeclaredIndexLookupRows( + source.Right, + source.Condition, + parameters, + innerContext, + leftRow) + : null) ?? TryGetTransientLookupRows( source.Right, source.Condition, @@ -38567,6 +38584,157 @@ private SourceData GetSemiOrAntiJoinRows( left.OmittedVirtualTablePredicates); } + /// + /// Runs a left anti hash join (Turso HashJoinType::LeftAnti): the outer rows are the + /// build input, hashed on the outer side of every planned equality key; the inner table is + /// read once as the probe; a build row is marked when some probe row in its bucket satisfies + /// the complete join condition; and the build rows no probe row marked are emitted in + /// their original order. The hash only chooses which pairs are tested, so the result is the + /// row-for-row answer of the nested anti-join: SQL equality never holds for a NULL key, so + /// a build row with a NULL key cannot match and is kept, and a probe row with one is skipped. + /// + private SourceData GetHashAntiJoinRows( + JoinTableSource source, + SemiAntiInnerAccess access, + SourceData left, + SqlValue[] parameters, + QueryContext innerContext, + long? maximumRows, + SourceRow? outerRow) + { + SourceData Emit(IReadOnlyList kept) + => new(left.Columns, kept, left.Collations, left.ColumnDefinitions, left.OmittedVirtualTablePredicates); + + if (left.Rows.Count == 0) + return Emit([]); + + var named = (NamedTableSource)source.Right; + var table = access.Table; + var innerOutputColumns = GetOutputColumns(named, innerContext); + var keys = new List<(Expression Column, TransientEqualityLookup Lookup)>(access.HashKeys.Count); + foreach (var key in access.HashKeys) + { + var columnOnLeft = key.Conjunct is BinaryExpression binary + && ReferenceEquals(binary.Left, key.ColumnSide); + if (!TryMatchTransientEquality( + key.ColumnSide, + key.Value, + columnOnLeft, + named, + table, + innerOutputColumns, + left.Rows[0], + out var lookup)) + { + throw new InvalidOperationException( + "A planned hash anti-join key is not canonicalizable for the build input."); + } + + keys.Add((key.ColumnSide, lookup)); + } + + var buckets = new Dictionary>(StringComparer.Ordinal); + var parts = new string[keys.Count]; + for (var position = 0; position < left.Rows.Count; position++) + { + innerContext.CheckInterrupt(); + var buildRow = left.Rows[position]; + if (!TryBuildHashAntiKey(keys, parts, key => EquiJoinHashIndex.CanonicalizeJoinKeyValue( + Evaluate(key.Lookup.ValueExpression, parameters, buildRow, innerContext), + key.Lookup.ValueConvertsTextToNumeric, + key.Lookup.ValueConvertsNumericToText, + key.Lookup.Collation), out var composite)) + { + continue; + } + + if (!buckets.TryGetValue(composite, out var bucket)) + { + bucket = []; + buckets[composite] = bucket; + } + + bucket.Add(position); + } + + var matched = new bool[left.Rows.Count]; + var remaining = left.Rows.Count; + var probeRows = GetSideSourceRows( + source.Right, + sidePredicate: null, + parameters, + innerContext, + outerRow, + sourceOrderBy: null); + foreach (var probeRow in probeRows.Rows) + { + innerContext.CheckInterrupt(); + if (remaining == 0) + break; + if (!TryBuildHashAntiKey(keys, parts, key => EquiJoinHashIndex.CanonicalizeJoinKeyValue( + Evaluate(key.Column, parameters, probeRow, innerContext), + key.Lookup.ColumnConvertsTextToNumeric, + key.Lookup.ColumnConvertsNumericToText, + key.Lookup.Collation), out var composite) + || !buckets.TryGetValue(composite, out var bucket)) + { + continue; + } + + foreach (var position in bucket) + { + if (matched[position]) + continue; + if (!IsTrue(Evaluate( + source.Condition!, + parameters, + probeRow with { Parent = left.Rows[position] }, + innerContext))) + { + continue; + } + + matched[position] = true; + remaining--; + } + } + + var kept = new List(remaining); + for (var position = 0; position < left.Rows.Count; position++) + { + if (matched[position]) + continue; + kept.Add(left.Rows[position]); + if (maximumRows is not null && kept.Count >= maximumRows.Value) + break; + } + + return Emit(kept); + + static bool TryBuildHashAntiKey( + List<(Expression Column, TransientEqualityLookup Lookup)> keys, + string[] parts, + Func<(Expression Column, TransientEqualityLookup Lookup), string?> canonicalize, + out string composite) + { + for (var index = 0; index < keys.Count; index++) + { + if (canonicalize(keys[index]) is not { } part) + { + composite = string.Empty; + return false; + } + + parts[index] = part; + } + + composite = parts.Length == 1 + ? parts[0] + : string.Concat(parts.Select(static part => part.Length.ToString(CultureInfo.InvariantCulture) + ":" + part)); + return true; + } + } + private static IReadOnlyList? CombineColumnDefinitions( SourceRow? left, SourceRow? right, diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index 89b1aaab..52b2b5b4 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -217,7 +217,3 @@ turso/explain-query-plan-json.sqltest::quotes-in-the-statement-are-escaped-in-th turso/explain-query-plan-json.sqltest::search-nodes-name-the-index-the-constraints-and-the-join | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, o.amount\n FROM users u\n LEFT JOIN order turso/explain-query-plan-json.sqltest::the-json-tree-matches-the-text-rows | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 4 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.age IN (SELECT ag turso/vacuum.sqltest::vacuum-rejects-query-only | database variant 1/1 (:temp:): error 'attempt to write a readonly database' does not match pattern 'Cannot execute VACUUM in query_only mode' -unnest-correlated.sqltest::correlated-comparison-uses-joins | database variant 1/1 (:memory:): expected 4 row(s), got 2 row(s); row 0 expected '1/0/0/SCAN outer_rows AS o' but was '0/0/0/MANAGED EVALUATOR FALLBACK' -unnest-correlated.sqltest::correlated-in-inequality-uses-indexed-semi-join | database variant 1/1 (:memory:): output does not match pattern '(?s)SCAN outer_rows AS o.*SEARCH i USING COVERING INDEX inner_key_amount' -unnest-correlated.sqltest::correlated-in-with-only-inequality-uses-indexed-semi-join | database variant 1/1 (:memory:): output does not match pattern '(?s)SCAN outer_rows AS o.*SEARCH i USING COVERING INDEX inner_key_amount \(key1<\?\)' -unnest-correlated.sqltest::not-exists-can-use-hash-anti-join | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN hash_anti_inner AS i.*SCAN hash_anti_outer AS o' diff --git a/src/Ahtola.Tests/SubqueryRewriteTests.cs b/src/Ahtola.Tests/SubqueryRewriteTests.cs index 47d36cf4..66821515 100644 --- a/src/Ahtola.Tests/SubqueryRewriteTests.cs +++ b/src/Ahtola.Tests/SubqueryRewriteTests.cs @@ -820,12 +820,39 @@ WHERE EXISTS (SELECT 1 FROM orders AS o } [Test] - public void DoesNotRewriteCorrelationPredicatesThatAreNotPlainEqualities() + public void RewritesDirectCorrelatedComparisonsThatAreNotEqualities() { - var queries = new[] + // Turso v0.8.1 (f7aac1fc5, unnest.rs is_inner_outer_comparison) moves any direct + // comparison (=, <>, <, <=, >, >=, IS, IS NOT) whose sides each read one scope. + var semiQueries = new[] { "SELECT c.id FROM customers AS c WHERE EXISTS (SELECT 1 FROM orders AS o WHERE o.customer_id > c.id) ORDER BY c.id;", "SELECT c.id FROM customers AS c WHERE EXISTS (SELECT 1 FROM orders AS o WHERE o.customer_id IS c.id) ORDER BY c.id;", + "SELECT c.id FROM customers AS c WHERE EXISTS (SELECT 1 FROM orders AS o WHERE o.customer_id = c.id AND o.total <> c.id) ORDER BY c.id;", + }; + foreach (var query in semiQueries) + { + AssertMatchesSqlite(OrdersSetup, query); + AssertRewrites(OrdersSetup, query, semiJoins: 1, antiJoins: 0); + } + + var antiQueries = new[] + { + "SELECT c.id FROM customers AS c WHERE NOT EXISTS (SELECT 1 FROM orders AS o WHERE o.customer_id >= c.id) ORDER BY c.id;", + "SELECT c.id FROM customers AS c WHERE NOT EXISTS (SELECT 1 FROM orders AS o WHERE o.customer_id = c.id AND o.total IS NOT c.id) ORDER BY c.id;", + }; + foreach (var query in antiQueries) + { + AssertMatchesSqlite(OrdersSetup, query); + AssertRewrites(OrdersSetup, query, semiJoins: 0, antiJoins: 1); + } + } + + [Test] + public void DoesNotRewriteCorrelationPredicatesThatAreNotDirectComparisons() + { + var queries = new[] + { "SELECT c.id FROM customers AS c WHERE EXISTS (SELECT 1 FROM orders AS o WHERE o.customer_id + c.id = 3) ORDER BY c.id;", """ SELECT c.id FROM customers AS c From 25ffee59f7fed6420951d532b73d314886f405ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 17:58:43 -0400 Subject: [PATCH 12/20] Plan IN-list and OR-implied IN index searches, compound multi-index OR, and semi/anti JSON estimates - Port Turso v0.8.1 in_filters_implied_by_or_branches (70f75beb7, lift_common_subexpressions.rs): an OR whose every branch pins a column to copyable literals implies column IN (values); when several columns of one table qualify only the sole leading-index column is used, and none when a compound index starts with two of them. The planner consumes the implied filter while choosing an access path; the statement itself is unchanged, so the original OR still filters every returned row. - TryPlanManagedIndexScan now searches an index by an explicit positive IN list or an implied IN filter on its plain ascending leading column (choose_best_in_seek_candidate), reported as Turso's (col=?) seek. - The OR-by-union planner replans compound AND branches with the compound-seek analysis and may use one index in several branches (MULTI-INDEX OR t (txy, txy)), matching consider_multi_index_union. - IndexCoversSelect understands IN lists and BETWEEN. - EXPLAIN QUERY PLAN FORMAT=JSON attaches the ported cost model's estimates (input/output rows, access and running cost, including cost_with_where_work with Turso's extra_steps) to planned semi/anti join nodes; search, multi-index and hash-join ops can now carry estimates. Closes multi_index_or_compound.sqltest compound-or-keeps-correlated-index-columns-together, nested-or-branches-use-one-index-search, nested-and-branches-use-one-index-search and turso/explain-query-plan-json.sqltest exists-is-reported-as-a-semi-join, not-exists-is-reported-as-an-anti-join. Co-Authored-By: Claude Opus 5.5 --- ...eddedDatabase.ExplainQueryPlanDescriber.cs | 55 +++- .../EmbeddedDatabase.OrTermInference.cs | 242 ++++++++++++++++++ .../EmbeddedDatabase.SemiAntiJoinAccess.cs | 35 ++- src/Ahtola.Core/EmbeddedDatabase.cs | 45 +++- .../managed-sqltest-expected-failures.txt | 5 - 5 files changed, 356 insertions(+), 26 deletions(-) create mode 100644 src/Ahtola.Core/EmbeddedDatabase.OrTermInference.cs diff --git a/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs b/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs index 99956a9f..de15523c 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs @@ -790,8 +790,29 @@ private bool TryDescribePlannedSemiAntiJoins( var outerDetail = outerPlan is null ? $"SCAN {outer.Name}" + (outer.Alias is null ? string.Empty : $" AS {outer.Alias}") : FormatManagedIndexExplainDetail(outerPlan, outerSelect); + // Estimates are reported only when the outer read is the full scan the ported cost + // model priced; an index-planned outer read is chosen by the managed heuristic planner. + EqpJsonEstimate? outerEstimate = null; + var outerRows = 0.0; + var runningCost = 0.0; + if (outerPlan is null) + { + (outerRows, runningCost) = EstimateSemiAntiOuterScan(outer, leftPredicate, context); + outerEstimate = new EqpJsonEstimate(1, outerRows, outerRows, runningCost, runningCost); + } + + EqpJsonEstimate? NextEstimate(SemiAntiInnerAccess access) + { + if (outerEstimate is null) + return null; + runningCost += access.Cost; + // A semi/anti join keeps or drops each outer row, so its output is its input + // (rows_after_join, join.rs:166-172). + return new EqpJsonEstimate(outerRows, 1, outerRows, access.Cost, runningCost); + } + var outerOp = outerPlan is null - ? (EqpJsonOp)new EqpJsonScanOp(outer.Name, outer.Alias, IndexName: null, Covering: false) + ? (EqpJsonOp)new EqpJsonScanOp(outer.Name, outer.Alias, IndexName: null, Covering: false, Estimate: outerEstimate) : BuildIndexScanOp(outerPlan, outerSelect); var planRows = new List(); @@ -805,14 +826,18 @@ void Add(string detail, EqpJsonOp op) var startIndex = 0; if (accesses[0].Kind == SemiAntiInnerAccessKind.HashAnti) { + // The build (outer) read is the first loop of the plan; the probe line comes first + // in the listing, as Turso prints a hash join. var probe = accesses[0].Inner; + var hashEstimate = NextEstimate(accesses[0]); Add( $"HASH JOIN {probe.Name}" + (probe.Alias is null ? string.Empty : $" AS {probe.Alias}"), - new EqpJsonHashJoinOp(probe.Name, probe.Alias, "anti")); + new EqpJsonHashJoinOp(probe.Name, probe.Alias, "anti", hashEstimate)); startIndex = 1; } - Add(outerDetail, outerOp); + planRows.Insert(startIndex == 1 ? 1 : 0, PlanRow(0, 0, outerDetail)); + planOps.Insert(startIndex == 1 ? 1 : 0, outerOp); for (var index = startIndex; index < accesses.Count; index++) { var access = accesses[index]; @@ -825,27 +850,27 @@ void Add(string detail, EqpJsonOp op) case SemiAntiInnerAccessKind.DeclaredIndexSearch: Add( $"SEARCH {alias} USING {(access.Covering ? "COVERING " : string.Empty)}INDEX {access.Index!.Name} ({string.Join(" AND ", constraints)})", - new EqpJsonSearchOp(inner.Name, inner.Alias, access.Index.Name, access.Covering, constraints, Join: joinMarker)); + new EqpJsonSearchOp(inner.Name, inner.Alias, access.Index.Name, access.Covering, constraints, Join: joinMarker, Estimate: NextEstimate(access))); break; case SemiAntiInnerAccessKind.DeclaredIndexScan: Add( $"SCAN {inner.Name}" + (inner.Alias is null ? string.Empty : $" AS {inner.Alias}") + $" USING {(access.Covering ? "COVERING " : string.Empty)}INDEX {access.Index!.Name}", - new EqpJsonScanOp(inner.Name, inner.Alias, access.Index.Name, access.Covering, joinMarker)); + new EqpJsonScanOp(inner.Name, inner.Alias, access.Index.Name, access.Covering, joinMarker, Estimate: NextEstimate(access))); break; case SemiAntiInnerAccessKind.EphemeralIndex: { var indexName = $"ephemeral_{inner.Name}_t{index * 2 + 3}"; Add( $"SEARCH {alias} USING COVERING INDEX {indexName} ({string.Join(" AND ", constraints)})", - new EqpJsonSearchOp(inner.Name, inner.Alias, indexName, Covering: true, constraints, Join: joinMarker, Ephemeral: true)); + new EqpJsonSearchOp(inner.Name, inner.Alias, indexName, Covering: true, constraints, Join: joinMarker, Ephemeral: true, Estimate: NextEstimate(access))); break; } case SemiAntiInnerAccessKind.TableScan: Add( $"SCAN {inner.Name}" + (inner.Alias is null ? string.Empty : $" AS {inner.Alias}"), - new EqpJsonScanOp(inner.Name, inner.Alias, IndexName: null, Covering: false, joinMarker)); + new EqpJsonScanOp(inner.Name, inner.Alias, IndexName: null, Covering: false, joinMarker, Estimate: NextEstimate(access))); break; default: // Only the first join of a chain can hash its (plain base table) outer input. @@ -853,7 +878,10 @@ void Add(string detail, EqpJsonOp op) } } - rows = planRows; + // Node ids follow the listing order. + rows = planRows + .Select((row, position) => PlanRow(position + 1, 0, row[3].AsText())) + .ToList(); ops = planOps; return true; } @@ -1099,7 +1127,8 @@ internal sealed record EqpJsonSearchOp( string? Join = null, bool Ephemeral = false, string SearchKind = "seek", - bool IsIntegerPrimaryKey = false) : EqpJsonOp + bool IsIntegerPrimaryKey = false, + EqpJsonEstimate? Estimate = null) : EqpJsonOp { public override string ToJson() { @@ -1116,6 +1145,7 @@ public override string ToJson() json.Append(",\"constraints\":[") .Append(string.Join(",", Constraints.Select(static c => EmbeddedDatabase.JsonEscape(c)))) .Append(']'); + Estimate?.AppendTo(json); return json.Append('}').ToString(); } } @@ -1125,7 +1155,8 @@ internal sealed record EqpJsonMultiIndexOp( string Table, IReadOnlyList Indexes, bool Union = true, - string? Alias = null) : EqpJsonOp + string? Alias = null, + EqpJsonEstimate? Estimate = null) : EqpJsonOp { public override string ToJson() { @@ -1136,6 +1167,7 @@ public override string ToJson() .Append("\",\"indexes\":[") .Append(string.Join(",", Indexes.Select(static name => EmbeddedDatabase.JsonEscape(name)))) .Append(']'); + Estimate?.AppendTo(json); return json.Append('}').ToString(); } } @@ -1144,12 +1176,13 @@ public override string ToJson() /// Mirrors EqpDetail::HashJoin: the build side of a hash join reads the whole table once /// before any probe. /// -internal sealed record EqpJsonHashJoinOp(string Table, string? Alias, string? Join = null) : EqpJsonOp +internal sealed record EqpJsonHashJoinOp(string Table, string? Alias, string? Join = null, EqpJsonEstimate? Estimate = null) : EqpJsonOp { public override string ToJson() { var json = new System.Text.StringBuilder("{\"type\":\"hash_join\","); AppendTableFields(json, Table, Alias, Join); + Estimate?.AppendTo(json); return json.Append('}').ToString(); } } diff --git a/src/Ahtola.Core/EmbeddedDatabase.OrTermInference.cs b/src/Ahtola.Core/EmbeddedDatabase.OrTermInference.cs new file mode 100644 index 00000000..d5d0d01f --- /dev/null +++ b/src/Ahtola.Core/EmbeddedDatabase.OrTermInference.cs @@ -0,0 +1,242 @@ +using Ahtola.Core.Parsing; + +namespace Ahtola.Core; + +/// +/// IN-list index searches, including the IN filters Turso v0.8.1 infers from +/// OR terms (optimizer/lift_common_subexpressions.rs rewrite_or_terms / +/// in_filters_implied_by_or_branches, commit 70f75beb7). +/// +/// For (x = 1 AND y = 10) OR (x = 2 AND y = 20) every branch pins x to one of +/// 1, 2, so x IN (1, 2) holds for every row the OR accepts. Turso adds that +/// implied filter to the WHERE clause (keeping the original OR) so the planner can +/// search an index on x. The managed planner never mutates the statement: it asks for +/// the implied filters while choosing an access path, and the complete WHERE — the original +/// OR included — still filters every row the search returns, so the implied filter can +/// only prune rows no branch accepts. +/// +/// +public sealed partial class EmbeddedDatabase +{ + /// + /// The IN filters implied by the top-level OR conjuncts of for one + /// table, ported from in_filters_implied_by_or_branches: a column qualifies when + /// every branch compares it with at least one copyable literal, the branches name more + /// than one distinct value, and — when several columns of the table qualify — it is the + /// single one that leads an index and no compound index starts with two of them. + /// + private static IReadOnlyList<(int ColumnOrdinal, IReadOnlyList Values)> GetImpliedOrInFilters( + Expression? where, + NamedTableSource source, + EmbeddedTable table) + { + if (where is null) + return []; + + List<(int ColumnOrdinal, IReadOnlyList Values)>? result = null; + foreach (var conjunct in IndexExpressionSemantics.SplitConjuncts(where)) + { + if (conjunct is not BinaryExpression { Operator: BinaryOperator.Or }) + continue; + + var branches = new List(); + CollectTopLevelOrLeaves(conjunct, branches); + var branchTerms = branches + .Select(static branch => IndexExpressionSemantics.SplitConjuncts(branch)) + .ToArray(); + + var candidates = new List(); + foreach (var term in branchTerms[0]) + { + if (TryGetColumnLiteralEquality(term, source, table, out var ordinal, out _) + && !candidates.Contains(ordinal)) + { + candidates.Add(ordinal); + } + } + + var filters = new List<(int ColumnOrdinal, IReadOnlyList Values)>(); + foreach (var ordinal in candidates) + { + var values = new List(); + var everyBranch = true; + foreach (var terms in branchTerms) + { + var found = false; + foreach (var term in terms) + { + if (!TryGetColumnLiteralEquality(term, source, table, out var termOrdinal, out var value) + || termOrdinal != ordinal) + { + continue; + } + + found = true; + if (!values.Any(existing => IndexExpressionSemantics.ExpressionsEqual(existing, value))) + values.Add(value); + } + + if (!found) + { + everyBranch = false; + break; + } + } + + // One value is the common-term rewrite's case, not an IN filter. + if (everyBranch && values.Count > 1) + filters.Add((ordinal, values)); + } + + if (filters.Count > 1) + { + var columns = filters.Select(static filter => filter.ColumnOrdinal).ToArray(); + var sole = SoleIndexedColumn(table, columns); + filters = sole is { } soleColumn + ? [.. filters.Where(filter => filter.ColumnOrdinal == soleColumn)] + : []; + } + + (result ??= []).AddRange(filters); + } + + return result ?? (IReadOnlyList<(int, IReadOnlyList)>)[]; + } + + /// lift_common_subexpressions.rs:208-233 sole_indexed_column. + private static int? SoleIndexedColumn(EmbeddedTable table, IReadOnlyList columns) + { + var indexes = table.Indexes.Where(static index => !index.IsMethodIndex).ToArray(); + if (indexes.Any(index => index.Columns.Count > 1 + && !index.Columns[0].IsExpression + && !index.Columns[1].IsExpression + && columns.Contains(index.Columns[0].ColumnIndex) + && columns.Contains(index.Columns[1].ColumnIndex))) + { + return null; + } + + var indexed = columns + .Where(column => indexes.Any(index => !index.IsPartial + && index.Columns.Count > 0 + && !index.Columns[0].IsExpression + && index.Columns[0].ColumnIndex == column)) + .ToArray(); + return indexed.Length == 1 ? indexed[0] : null; + } + + /// + /// lift_common_subexpressions.rs:239-284 column_literal_equality: column = + /// literal (either side) where the literal is a number, text, BLOB, TRUE/FALSE, or a + /// signed number. NULL never satisfies = and is not copied. + /// + private static bool TryGetColumnLiteralEquality( + Expression term, + NamedTableSource source, + EmbeddedTable table, + out int ordinal, + out Expression value) + { + ordinal = -1; + value = null!; + if (term is not BinaryExpression { Operator: BinaryOperator.Equal } equality) + return false; + + if (IsCopyableLiteral(equality.Right) && TryResolvePlainTableColumn(equality.Left, source, table, out ordinal)) + { + value = equality.Right; + return true; + } + + if (IsCopyableLiteral(equality.Left) && TryResolvePlainTableColumn(equality.Right, source, table, out ordinal)) + { + value = equality.Left; + return true; + } + + return false; + + static bool IsCopyableLiteral(Expression expression) + => expression switch + { + LiteralExpression { Value.Kind: SqlValueKind.Integer or SqlValueKind.Real or SqlValueKind.Text or SqlValueKind.Blob } => true, + ColumnExpression { BooleanKeyword: not null } => true, + UnaryExpression + { + Operator: UnaryOperator.Negate or UnaryOperator.Plus, + Operand: LiteralExpression { Value.Kind: SqlValueKind.Integer or SqlValueKind.Real }, + } => true, + _ => false, + }; + } + + /// A bare column reference that names a declared column of this table. + private static bool TryResolvePlainTableColumn( + Expression expression, + NamedTableSource source, + EmbeddedTable table, + out int ordinal) + { + ordinal = -1; + if (expression is not ColumnExpression { BooleanKeyword: null } column + || column.Qualifier is { } qualifier + && !string.Equals(qualifier, source.Alias ?? source.Name, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + return table.TryGetColumnIndex(column.UnqualifiedName ?? column.Name, out ordinal); + } + + /// + /// Whether an index whose leading key is can be searched by an + /// IN list (access_method.rs:505-645 choose_best_in_seek_candidate): an explicit + /// positive column IN (constant, …) conjunct, or an OR-implied IN filter, on that + /// plain ascending column, compared under the index's collation. + /// + private static bool WhereAllowsIndexInSearch( + Expression? where, + NamedTableSource source, + EmbeddedTable table, + EmbeddedIndexColumn leading) + { + if (where is null || leading.IsExpression || leading.ColumnIndex < 0 || leading.Descending) + return false; + + // open_loop copies the index collation onto the IN cursor, so the column's own + // collation must be the index's (access_method.rs:588-602). + var columnCollation = table.ColumnDefinitions[leading.ColumnIndex].Collation ?? "BINARY"; + if (!string.Equals( + columnCollation, + IndexExpressionSemantics.GetCollationName(table, leading), + StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + foreach (var conjunct in IndexExpressionSemantics.SplitConjuncts(where)) + { + if (conjunct is InExpression { Negated: false } inList + && inList.Values.Count > 0 + && TryResolvePlainTableColumn(inList.Value, source, table, out var ordinal) + && ordinal == leading.ColumnIndex + && inList.Values.All(IsInSearchConstant)) + { + return true; + } + } + + return GetImpliedOrInFilters(where, source, table) + .Any(filter => filter.ColumnOrdinal == leading.ColumnIndex); + + static bool IsInSearchConstant(Expression value) + => value switch + { + LiteralExpression or ParameterExpression => true, + ColumnExpression { BooleanKeyword: not null } => true, + UnaryExpression { Operator: UnaryOperator.Negate or UnaryOperator.Plus, Operand: var operand } + => operand is LiteralExpression, + _ => false, + }; + } +} diff --git a/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs b/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs index 374ff5ed..ab90fdef 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs @@ -162,7 +162,9 @@ private static IReadOnlyList BuildConstraintTexts(IReadOnlyList { if (!IsInnerColumnReference(column, innerQualifier, innerColumns)) @@ -351,10 +353,39 @@ or TursoConstraintOperator.Greater { var hashWithWhere = hash.Cost + EstimateSemiAntiWhereWork(hash, steps, inputRows); if (hashWithWhere < bestWithWhere) + { best = hash; + bestWithWhere = hashWithWhere; + } + } + + // add_where_cost folds the loop's WHERE work into the access method's own cost, which + // is what Turso reports as the node's access_cost. + return best with { Cost = bestWithWhere }; + } + + /// + /// The estimate of the first loop of a semi/anti chain: a full scan of the outer table + /// (estimate_scan_cost plus the WHERE work of its own filters, access_method.rs:687-707), + /// producing its row count reduced by those filters' selectivities (rows_after_join). + /// + private (double Rows, double Cost) EstimateSemiAntiOuterScan( + NamedTableSource outer, + Expression? predicate, + QueryContext context) + { + var table = context.Tables[outer.Name]; + var baseRows = EstimateTursoTableRows(outer.Name, context); + var extraSteps = 0; + if (predicate is not null) + { + foreach (var conjunct in IndexExpressionSemantics.SplitConjuncts(predicate)) + extraSteps += CountTursoWhereSteps(conjunct) - 1; } - return best; + var cost = TursoCostModel.EstimateScanCost(baseRows, 1.0) + + TursoCostModel.EstimateWhereWork(1.0, baseRows, consumedSteps: 0, extraSteps); + return (baseRows * EstimateTursoLocalSelectivity(predicate, outer, table, context), cost); } /// diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index 0df754ea..42531b6a 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -31056,6 +31056,13 @@ private static int CountTableSourceLeaves(TableSource? source) var leading = index.Columns[0]; var searchConstraint = GetIndexSearchConstraint(statement.Where, table, leading); var search = WhereUsesIndexTerm(statement.Where, table, leading); + if (!search && WhereAllowsIndexInSearch(statement.Where, source, table, leading)) + { + // An IN-list (explicit or implied by an OR, see EmbeddedDatabase.OrTermInference.cs) + // searches the leading key once per value; Turso reports that as an equality. + search = true; + searchConstraint = $"{leading.Name}=?"; + } var ordered = TryGetIndexOrderDirection(statement.OrderBy, table, index, out var reverse); var scansOnlyPartialPredicate = index.IsPartial && statement.OrderBy.Count == 0 @@ -31699,6 +31706,14 @@ private static bool ExpressionCoveredByIndex( return ExpressionCoveredByIndex(collation.Expression, table, covered); case FunctionExpression function: return function.Arguments.All(arg => ExpressionCoveredByIndex(arg, table, covered)); + case InExpression inList: + // An IN list only reads its left operand and constant values. + return ExpressionCoveredByIndex(inList.Value, table, covered) + && inList.Values.All(value => ExpressionCoveredByIndex(value, table, covered)); + case BetweenExpression between: + return ExpressionCoveredByIndex(between.Value, table, covered) + && ExpressionCoveredByIndex(between.Lower, table, covered) + && ExpressionCoveredByIndex(between.Upper, table, covered); default: // Subqueries, CASE, etc. are not covering-index safe without deeper analysis. return false; @@ -32310,9 +32325,12 @@ private int CompareMvccIndexRows( planned.Add(new ManagedOrIndexUnionBranch(index, index.Name, branch)); } - // Require at least one distinct index name so this is a real multi-index OR, - // not a single-index multi-equality that should use a plain index scan. - if (planned.Select(static item => item.Name).Distinct(StringComparer.OrdinalIgnoreCase).Count() < 2) + // Turso's OR-by-union (multi_index.rs consider_multi_index_union) replans every + // disjunct with the ordinary compound-seek analysis, so several branches may search + // the same index (MULTI-INDEX OR t (txy, txy)). A single-column OR of literals never + // reaches this point: its implied IN filter (EmbeddedDatabase.OrTermInference.cs) + // already chose an IN-list index search. Rowid-only unions are not modelled here. + if (planned.All(static item => item.Index is null)) return null; return new ManagedOrIndexUnionPlan(source, table, planned); @@ -32323,8 +32341,17 @@ private static bool IsRowidPrimaryKeyEquality( NamedTableSource source, Expression branch) { - if (!table.HasRowid - || branch is not BinaryExpression + if (!table.HasRowid) + return false; + + // A compound branch uses the rowid when one of its conjuncts is a rowid equality. + if (branch is BinaryExpression { Operator: BinaryOperator.And } conjunction) + { + return IsRowidPrimaryKeyEquality(table, source, conjunction.Left) + || IsRowidPrimaryKeyEquality(table, source, conjunction.Right); + } + + if (branch is not BinaryExpression { Operator: BinaryOperator.Equal or BinaryOperator.Is, } equality) @@ -32361,10 +32388,12 @@ private static bool TrySplitTopLevelOrBranches( { branches = []; CollectTopLevelOrLeaves(expression, branches); + // Each disjunct is an equality, or a conjunction the per-branch compound-seek analysis + // plans (multi_index.rs:1005-1075); its remaining conjuncts filter that branch's rows. return branches.Count >= 2 - && branches.All(branch => branch is BinaryExpression + && branches.All(static branch => branch is BinaryExpression { - Operator: BinaryOperator.Equal or BinaryOperator.Is + Operator: BinaryOperator.Equal or BinaryOperator.Is or BinaryOperator.And }); } @@ -32388,7 +32417,7 @@ private bool TryFindEqualityIndexForBranch( { index = null!; if (branch is not BinaryExpression binary - || binary.Operator is not (BinaryOperator.Equal or BinaryOperator.Is)) + || binary.Operator is not (BinaryOperator.Equal or BinaryOperator.Is or BinaryOperator.And)) { return false; } diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index 52b2b5b4..cc51598f 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -186,14 +186,11 @@ join/memory.sqltest::partial-stats-hash-join-uses-known-unique-key-count | datab join/memory.sqltest::range-prefix-still-allows-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN prefix_child AS child' join/memory.sqltest::unique-composite-hash-key-uses-one-key-estimate | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN composite_hash_child AS child".*"rows_per_input":5' join/memory.sqltest::unique-hash-key-does-not-apply-to-a-different-collation | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN collation_unique_probe".*"rows_per_input":0.1' -multi_index_or_compound.sqltest::compound-or-keeps-correlated-index-columns-together | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '1/0/0/MULTI-INDEX OR t (txy, txy)' but was '0/0/0/MANAGED EVALUATOR FALLBACK' multi_index_or_compound.sqltest::derived-or-filter-skips-correlated-columns | database variant 1/1 (:memory:): output does not match pattern '"rows_per_input":1000' multi_index_or_compound.sqltest::implied-or-filter-accepts-negative-numbers | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' multi_index_or_compound.sqltest::implied-or-filter-accepts-parenthesized-equalities | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' multi_index_or_compound.sqltest::implied-or-filter-keeps-compound-index-searches | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' multi_index_or_compound.sqltest::implied-or-filter-uses-only-indexed-column | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' -multi_index_or_compound.sqltest::nested-and-branches-use-one-index-search | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '1/0/0/SEARCH t USING INDEX tx (x=?)' but was '0/0/0/MANAGED EVALUATOR FALLBACK' -multi_index_or_compound.sqltest::nested-or-branches-use-one-index-search | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '1/0/0/SEARCH t USING COVERING INDEX tx (x=?)' but was '0/0/0/MANAGED EVALUATOR FALLBACK' sql-error-message-prefix.sqltest::database-full-error-has-no-prefix | database variant 1/1 (:memory:): expected an error but the statement succeeded turso/alter_column.sqltest::alter-column-rejects-index-only-rewrite-mvcc | database variant 1/1 (:memory:): expected an error but the statement succeeded turso/alter_column.sqltest::alter-column-rejects-index-rewrite-mvcc | database variant 1/1 (:memory:): expected an error but the statement succeeded @@ -210,9 +207,7 @@ turso/explain-query-plan-json.sqltest::a-from-clause-subquery-nests-its-body-und turso/explain-query-plan-json.sqltest::a-recursive-cte-reports-its-setup-and-step-phases | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH RECURSIVE cnt(x) AS (\n SELECT 1\n UNION AL turso/explain-query-plan-json.sqltest::an-or-over-two-indexes-lists-both-branches | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users\n WHERE id = 5 OR age = 30;","resu turso/explain-query-plan-json.sqltest::both-readers-of-a-shared-cte-point-at-one-materialization | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH spenders AS (\n SELECT user_id, sum(amount) AS tot -turso/explain-query-plan-json.sqltest::exists-is-reported-as-a-semi-join | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users u\n WHERE EXISTS (SELECT 1 FROM or turso/explain-query-plan-json.sqltest::list-and-scalar-subqueries-report-whether-they-are-correlated | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, (SELECT count(*) FROM orders o WHERE o.user_id -turso/explain-query-plan-json.sqltest::not-exists-is-reported-as-an-anti-join | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users u\n WHERE NOT EXISTS (SELECT 1 FRO turso/explain-query-plan-json.sqltest::quotes-in-the-statement-are-escaped-in-the-echo | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.name = 'a \"quote turso/explain-query-plan-json.sqltest::search-nodes-name-the-index-the-constraints-and-the-join | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, o.amount\n FROM users u\n LEFT JOIN order turso/explain-query-plan-json.sqltest::the-json-tree-matches-the-text-rows | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 4 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.age IN (SELECT ag From 3ba59a3ead8c11674eed2e8490f2a4e5b8d1df8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 18:49:31 -0400 Subject: [PATCH 13/20] Enforce PRAGMA max_page_count and report real page counts PRAGMA page_count now reports the real size for every database, and a connection's PRAGMA max_page_count ceiling is enforced with SQLite's SQLITE_FULL ("database or disk is full", result code 13) and its transaction semantics. - :memory: databases keep rows as heap objects, so their page count is measured by serializing the catalog through the same page builders a persisted image (VACUUM INTO / full rewrite) uses, via a private in-memory page planner store. Per-tree counts are cached by row-store lineage/revision; page_count is a high-water mark with freelist, and VACUUM compacts it. Values match bundled SQLite for the tested shapes. - File-backed databases: SqlitePager refuses a write transaction whose target size exceeds both the ceiling and the committed size (Turso Pager::allocate_page), before any frame is written. Explicit transactions plan the persist (pager PlanCommitsOnly probe) after each statement so the statement fails like SQLite's, not the COMMIT. - max_page_count is per connection, never persisted, clamps to the connection-visible page count, and treats 0/negative as a query. - In an explicit transaction a single-row write without a statement journal rolls back the whole transaction (sqlite3VdbeHalt); any other statement only rolls back itself. - MVCC: autocommit writes (which build pages at commit) and checkpoints are held to the ceiling; MVCC transaction COMMITs are not (Turso). - Removes the sql-error-message-prefix database-full ledger entry. Co-Authored-By: Claude Opus 5.5 --- .../Compilation/DdlStatementCompiler.cs | 7 +- .../EmbeddedDatabase.PlannerAccessPaths.cs | 1 - src/Ahtola.Core/EmbeddedDatabase.cs | 611 +++++++++++++++--- src/Ahtola.Core/EmbeddedFileStore.cs | 335 ++++++++-- src/Ahtola.Core/Storage/SqliteBtreePageIo.cs | 4 +- .../Storage/SqlitePageAllocator.cs | 4 +- src/Ahtola.Core/Storage/SqlitePager.cs | 33 + .../SqlitePagerCommitPlannedException.cs | 19 + .../managed-sqltest-expected-failures.txt | 1 - src/Ahtola.Tests/InMemoryPageModelTests.cs | 4 +- src/Ahtola.Tests/ManagedMaxPageCountTests.cs | 430 ++++++++++++ .../StorageAuditBoundedGapTests.cs | 5 +- 12 files changed, 1300 insertions(+), 154 deletions(-) create mode 100644 src/Ahtola.Core/Storage/SqlitePagerCommitPlannedException.cs create mode 100644 src/Ahtola.Tests/ManagedMaxPageCountTests.cs diff --git a/src/Ahtola.Core/Compilation/DdlStatementCompiler.cs b/src/Ahtola.Core/Compilation/DdlStatementCompiler.cs index 9ed1d9dc..54330642 100644 --- a/src/Ahtola.Core/Compilation/DdlStatementCompiler.cs +++ b/src/Ahtola.Core/Compilation/DdlStatementCompiler.cs @@ -18,9 +18,10 @@ namespace Ahtola.Core.Compilation; /// functions, so it cannot be a static rule. /// /// -/// Rejects the statement when creating this many additional b-trees would exceed -/// PRAGMA max_page_count. EXPLAIN passes a no-op: describing a program must not consult a -/// runtime storage limit, and it never allocates a page. +/// Notified of how many additional b-tree roots the statement creates. The engine passes a no-op: +/// PRAGMA max_page_count is a per-connection ceiling enforced against the pages the finished +/// statement's catalog actually needs (EmbeddedDatabase.EnsureCatalogFitsPageLimit and the +/// pager growth ceiling), which a per-root estimate made while compiling cannot know. /// /// The routed database index every schema instruction addresses. /// diff --git a/src/Ahtola.Core/EmbeddedDatabase.PlannerAccessPaths.cs b/src/Ahtola.Core/EmbeddedDatabase.PlannerAccessPaths.cs index 4e2f65b1..8a6e653a 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.PlannerAccessPaths.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.PlannerAccessPaths.cs @@ -35,7 +35,6 @@ private EmbeddedTable GetOrCreateSqliteStat4Table(SchemaCatalog catalog) throw new EmbeddedSqlException($"object name reserved for internal use: {SqliteStat4TableName}"); } - EnforceMaxPageCountForCatalogChange(1); var statistics = new EmbeddedTable( SqliteStat4TableName, [ diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index d1b78eec..d5138150 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -71,6 +71,24 @@ public EmbeddedSqlException(string message, Exception innerException) : base(mes internal InsertConflictAlgorithm? ConflictAlgorithm { get; } } +/// +/// SQLite's SQLITE_FULL (result code 13): a write needed the database to grow past its +/// PRAGMA max_page_count ceiling (Turso LimboError::DatabaseFull, raised by +/// Pager::allocate_page). The message is exactly SQLite's sqlite3_errmsg text, +/// with no "Database is full: " prefix. The failing statement is rolled back before the +/// exception surfaces, so nothing it wrote reaches the database image. +/// +internal sealed class EmbeddedDatabaseFullException : EmbeddedSqlException +{ + internal const string DatabaseFullMessage = "database or disk is full"; + internal const int SqliteFullResultCode = 13; + + public EmbeddedDatabaseFullException() + : base(DatabaseFullMessage, SqliteFullResultCode) + { + } +} + internal readonly record struct PragmaHeaderMetadata( int SchemaVersion, int UserVersion, @@ -360,18 +378,20 @@ public sealed partial class EmbeddedDatabase : IDisposable private readonly Dictionary _blobMutationGenerations = new(); private long _nextBlobMutationGeneration; - // Managed in-memory page model backing PRAGMA page_count / page_size / - // max_page_count on databases that have no file store. The memory database - // starts with zero pages (matching SQLite's empty pager) and materializes - // pages the way the pager would: the first header write creates the header - // page, and each table or index adds one page. Dropped b-trees move their - // pages onto the freelist rather than shrinking the database, so - // page_count is a high-water mark the way SQLite's header field is. + // Managed in-memory page model backing PRAGMA page_count / freelist_count / + // page_size / max_page_count on databases that have no file store. The memory + // database starts with zero pages (matching SQLite's empty pager); the first + // committed write materializes it, and from then on its size is the page image + // its catalog serializes to (counted by _inMemoryPagePlanner, the same page + // builders a persisted image uses). Released pages stay on the freelist rather + // than shrinking the database, so page_count is a high-water mark the way + // SQLite's header field is, until VACUUM compacts it. internal bool _inMemoryInitialized; internal int? _inMemoryPageSize; - private uint _inMemoryFreelistPages; private uint _inMemoryHighWaterPages; - private uint _maxPageCount = 4294967294; + private EmbeddedFileStore? _inMemoryPagePlanner; + private int _inMemoryPagePlannerPageSize; + private EmbeddedFileStore.FreshImagePageCountCache _inMemoryPageCountCache = new(); private readonly EmbeddedTransactionLock _transactionLock; /// /// Opt-in Turso MVCC store. When non-null, PRAGMA journal_mode reports @@ -500,6 +520,8 @@ public void Dispose() } _fileStore?.Dispose(); + _inMemoryPagePlanner?.Dispose(); + _inMemoryPagePlanner = null; } } @@ -692,12 +714,6 @@ internal MvStore? MvStore internal string DatabasePath => _databasePath; - internal uint MaxPageCount - { - get => _maxPageCount; - set => _maxPageCount = value; - } - internal IFileSystem FileSystem => _fileSystem ?? throw new InvalidOperationException("The managed database is not file-backed."); @@ -2220,7 +2236,8 @@ internal ExecutionResult Execute( SqliteSynchronousMode synchronousMode = SqliteSynchronousMode.Full, ManagedVirtualTableTransaction? virtualTableTransaction = null, ManagedSequenceSession? sequenceSession = null, - Func? describeJournalMode = null) + Func? describeJournalMode = null, + uint maxPageCount = SqlitePageLimits.DefaultMaximumPageCount) { var result = ExecuteCore( statement, @@ -2242,7 +2259,8 @@ internal ExecutionResult Execute( synchronousMode, virtualTableTransaction, sequenceSession, - describeJournalMode); + describeJournalMode, + maxPageCount); RecordChangeCounters(statement, result); return result; @@ -2286,7 +2304,8 @@ private ExecutionResult ExecuteCore( SqliteSynchronousMode synchronousMode = SqliteSynchronousMode.Full, ManagedVirtualTableTransaction? virtualTableTransaction = null, ManagedSequenceSession? sequenceSession = null, - Func? describeJournalMode = null) + Func? describeJournalMode = null, + uint maxPageCount = SqlitePageLimits.DefaultMaximumPageCount) { synchronousMode.Validate(nameof(synchronousMode)); ThrowIfRecursiveTriggerCallbackReentry(); @@ -2316,7 +2335,8 @@ private ExecutionResult ExecuteCore( synchronousMode, virtualTableTransaction, sequenceSession, - describeJournalMode)); + describeJournalMode, + maxPageCount)); } cancellationToken.ThrowIfCancellationRequested(); @@ -2346,9 +2366,17 @@ private ExecutionResult ExecuteCore( var ownsVirtualTableTransaction = virtualTableTransaction is null; var statementVirtualTableTransaction = virtualTableTransaction ?? new ManagedVirtualTableTransaction(); + // A connection-level max_page_count ceiling on a :memory: database is checked + // against the statement's finished catalog before it is published, which also + // needs a discardable working clone. File-backed databases (MVCC autocommit + // writes included) enforce it in the pager when the persist below writes. + var enforceInMemoryPageLimit = _fileStore is null + && _mvStore is null + && maxPageCount < SqlitePageLimits.DefaultMaximumPageCount; if ((cancellationToken.CanBeCanceled || commitGate is not null - || _virtualTables.Count != 0) + || _virtualTables.Count != 0 + || enforceInMemoryPageLimit) && statementMayMutate) { var cancellableWorking = new SchemaCatalog(_tables, _views, _triggers, _virtualTables).Clone(); @@ -2394,7 +2422,8 @@ private ExecutionResult ExecuteCore( else PersistFileCatalog( cancellableWorking, - busyTimeout: GetRemainingBusyTimeout(busyRetryDeadline)); + busyTimeout: GetRemainingBusyTimeout(busyRetryDeadline), + maxPageCount: maxPageCount); if (ownsVirtualTableTransaction) statementVirtualTableTransaction.Commit(cancellableWorking); virtualTableCommitCompleted = true; @@ -2413,6 +2442,11 @@ private ExecutionResult ExecuteCore( return cancellableResult; } + // SQLITE_FULL is raised while the statement allocates pages, before the + // implicit commit (and its hooks) ever run. + var inMemoryPublishedPages = enforceInMemoryPageLimit + ? EnsureCatalogFitsPageLimit(cancellableWorking, maxPageCount) + : null; if (ownsVirtualTableTransaction) statementVirtualTableTransaction.Sync(cancellableWorking); if (commitGate is not null && !commitGate()) @@ -2429,6 +2463,7 @@ private ExecutionResult ExecuteCore( } PublishCatalog(cancellableWorking); + RecordInMemoryPublishedPageCountLocked(inMemoryPublishedPages); } else { @@ -2436,7 +2471,8 @@ private ExecutionResult ExecuteCore( cancellableWorking, forceFullRewrite: cancellableResult.ForceFullCatalogRewrite, busyTimeout: GetRemainingBusyTimeout(busyRetryDeadline), - targetedIndexRebuild: cancellableResult.TargetedIndexRebuild); + targetedIndexRebuild: cancellableResult.TargetedIndexRebuild, + maxPageCount: maxPageCount); } if (ownsVirtualTableTransaction) @@ -2563,7 +2599,8 @@ private ExecutionResult ExecuteCore( { PersistFileCatalog( working, - busyTimeout: GetRemainingBusyTimeout(busyRetryDeadline)); + busyTimeout: GetRemainingBusyTimeout(busyRetryDeadline), + maxPageCount: maxPageCount); throw; } if (result.Changed) @@ -2572,7 +2609,8 @@ private ExecutionResult ExecuteCore( working, forceFullRewrite: result.ForceFullCatalogRewrite, busyTimeout: GetRemainingBusyTimeout(busyRetryDeadline), - targetedIndexRebuild: result.TargetedIndexRebuild); + targetedIndexRebuild: result.TargetedIndexRebuild, + maxPageCount: maxPageCount); } else if (!inTransaction) { @@ -3667,7 +3705,8 @@ internal void CommitTransaction( bool concurrent = false, bool containsSchemaChanges = false, SqliteSynchronousMode synchronousMode = SqliteSynchronousMode.Full, - IReadOnlySet? targetedIndexRebuildNames = null) + IReadOnlySet? targetedIndexRebuildNames = null, + uint maxPageCount = SqlitePageLimits.DefaultMaximumPageCount) { synchronousMode.Validate(nameof(synchronousMode)); @@ -3708,6 +3747,10 @@ internal void CommitTransaction( if (pragmaHeader is { } metadata) _inMemoryPragmaHeader = metadata; PublishCatalog(publishCatalog); + // Each statement was already measured against the ceiling; recording the + // exact size (a cache hit per tree) keeps the high-water mark honest. + if (maxPageCount < SqlitePageLimits.DefaultMaximumPageCount && _mvStore is null) + RecordInMemoryPublishedPageCountLocked(CountInMemoryLivePagesLocked(publishCatalog)); return; } @@ -3722,7 +3765,13 @@ internal void CommitTransaction( // schema-signature check, which safely widens to the full-catalog rewrite that // forceFullRewrite already forces for any REINDEX. var targetedIndexRebuild = ResolveTargetedIndexRebuild(targetedIndexRebuildNames, publishCatalog); - PersistFileCatalog(publishCatalog, pragmaHeader, forceFullRewrite, busyTimeout, targetedIndexRebuild: targetedIndexRebuild); + PersistFileCatalog( + publishCatalog, + pragmaHeader, + forceFullRewrite, + busyTimeout, + targetedIndexRebuild: targetedIndexRebuild, + maxPageCount: maxPageCount); } finally { @@ -4026,79 +4075,243 @@ internal int GetPageSize() return _fileStore is null ? _inMemoryPageSize ?? SqlitePageSize.Default : _fileCatalogVersion.PageSize; } - internal uint GetPageCount() + /// + /// PRAGMA page_count: the pages the database occupies. A file-backed database reports + /// its committed header size; a :memory: database reports the high-water mark of the + /// page image its catalog serializes to (see ). + /// + /// + /// The asking connection's uncommitted write-transaction catalog, if any. SQLite reports the + /// pager's current size, which includes pages the open transaction has already allocated. + /// + internal uint GetPageCount(SchemaCatalog? pendingCatalog = null) { lock (_gate) { if (_fileStore is null) { // An uninitialized in-memory database has no pages yet (SQLite - // reports 0 until the first page is written). Once initialized, - // the count is the high-water mark of allocated pages — SQLite's - // header field only grows; drops move pages onto the freelist. - if (!_inMemoryInitialized) + // reports 0 until the first page is written). + if (!_inMemoryInitialized && pendingCatalog is null) return 0; + var live = CountInMemoryLivePagesLocked(pendingCatalog ?? LiveCatalogLocked()); + if (pendingCatalog is not null) + return Math.Max(_inMemoryInitialized ? _inMemoryHighWaterPages : 0, live); + + FoldInMemoryHighWaterLocked(live); return _inMemoryHighWaterPages; } - return _fileCatalogVersion.DatabaseSizeInPages; + var committed = _fileCatalogVersion.DatabaseSizeInPages; + if (pendingCatalog is null || _mvStore is not null) + return committed; + + return Math.Max( + committed, + TryPlanFilePersistedPageCountLocked(pendingCatalog, pragmaHeader: null, forceFullRewrite: false) + ?? committed); } } /// - /// Reconciles the managed in-memory page model after a committed catalog change. - /// The model mirrors SQLite's pager: the header page count is a high-water mark - /// that only grows, and the freelist is the gap between the high-water mark and - /// the live b-tree count. Dropping a b-tree moves its pages onto the freelist - /// without shrinking PRAGMA page_count; creating one consumes freelist - /// pages first and only raises the mark beyond the previous peak. Callers must - /// hold _gate. + /// PRAGMA freelist_count. In memory this is the gap between the page high-water mark + /// and the pages the live image needs: SQLite never shrinks the database without VACUUM, so + /// pages a DROP or DELETE released stay allocated on the freelist. + /// + internal uint GetFreelistCount() + { + lock (_gate) + { + if (_fileStore is null) + { + if (!_inMemoryInitialized) + return 0; + + var live = CountInMemoryLivePagesLocked(LiveCatalogLocked()); + FoldInMemoryHighWaterLocked(live); + return _inMemoryHighWaterPages - live; + } + + return _fileCatalogVersion.FreelistPageCount; + } + } + + private SchemaCatalog LiveCatalogLocked() => new(_tables, _views, _triggers, _virtualTables); + + /// + /// Folds a cheap, exact lower bound into the in-memory page high-water mark after a + /// committed catalog change: page 1 plus one root page per table and index b-tree (no real + /// image can need fewer). The exact size is only paid for when something asks for it + /// (page_count, freelist_count, a max_page_count ceiling), because it + /// re-serializes every changed tree. Callers must hold _gate. /// private void ReconcileInMemoryPageModel() { - var live = 1; + var live = 1u; foreach (var table in _tables.Values) - live += 1 + table.Indexes.Count; + live += 1 + (uint)table.Indexes.Count; - if (live > _inMemoryHighWaterPages) - _inMemoryHighWaterPages = (uint)live; + FoldInMemoryHighWaterLocked(live); + } - _inMemoryFreelistPages = _inMemoryHighWaterPages - (uint)live; + private void FoldInMemoryHighWaterLocked(uint livePages) + { + if (livePages > _inMemoryHighWaterPages) + _inMemoryHighWaterPages = livePages; + } + + /// + /// The pages a :memory: catalog needs: the size of the complete SQLite image the + /// managed page writer would serialize it to (the same image VACUUM INTO or a backup + /// would persist), computed by a private page planner at this database's page size. + /// + /// + /// The in-memory engine keeps rows as heap objects and never builds b-tree pages, so there is + /// no pager whose size could be read. Rather than estimate, this runs the real table, index + /// and sqlite_schema page builders (caching each tree's count by row-store revision). A + /// catalog the page writer cannot represent falls back to the one-root-per-tree lower bound. + /// Callers must hold _gate. + /// + private uint CountInMemoryLivePagesLocked(SchemaCatalog catalog) + { + try + { + var planner = GetInMemoryPagePlannerLocked(); + planner.SetCollationResolver(_hasCustomCollations ? BuildCollationResolver() : null); + return planner.CountFreshImagePages( + catalog.Tables, + catalog.Views, + catalog.Triggers, + catalog.VirtualTables, + _inMemoryPageCountCache); + } + catch (Exception exception) when (exception is EmbeddedSqlException + or InvalidOperationException + or ArgumentException + or InvalidDataException + or OverflowException + or NotSupportedException) + { + var live = 1u; + foreach (var table in catalog.Tables.Values) + live += 1 + (uint)table.Indexes.Count; + return live; + } } - // An in-memory database models PRAGMA max_page_count at the catalog level: each new - // table or index b-tree needs one page (plus the header page while the database is - // still uninitialized), and SQLite fails the statement with "database or disk is - // full" when the allocation would exceed the limit. File-backed databases allocate - // real pages, so the pager owns enforcement there and the catalog check stays inert. - private void EnforceMaxPageCountForCatalogChange(int additionalPages) + private EmbeddedFileStore GetInMemoryPagePlannerLocked() { - if (_fileStore is not null) - return; + var pageSize = _inMemoryPageSize ?? SqlitePageSize.Default; + if (_inMemoryPagePlanner is { } planner && _inMemoryPagePlannerPageSize == pageSize) + return planner; - // Freelist pages satisfy new b-trees before the database grows (SQLite - // allocates from the freelist first), so only the growth beyond the current - // high-water mark counts against the limit. - var live = (long)_inMemoryHighWaterPages - _inMemoryFreelistPages; - var required = Math.Max((long)_inMemoryHighWaterPages, live + additionalPages) - + (_inMemoryInitialized ? 0 : 1); - if (required > MaxPageCount) - throw new EmbeddedSqlException("database or disk is full"); + _inMemoryPagePlanner?.Dispose(); + _inMemoryPagePlanner = null; + _inMemoryPageCountCache = new EmbeddedFileStore.FreshImagePageCountCache(); + // A private, empty rollback-journal image in its own in-memory file system: it is only + // ever asked to count pages, never to persist this database's catalog. + var created = EmbeddedFileStore.Open( + "ahtola-memory-page-accounting.db", + new InMemoryFileSystem(), + out _, + initialPageSize: pageSize, + initialTextEncoding: SqliteTextEncoding.Utf8, + createRollbackJournalMode: true); + _inMemoryPagePlanner = created; + _inMemoryPagePlannerPageSize = pageSize; + return created; } - internal uint GetFreelistCount() + /// + /// Rejects a finished statement whose catalog would need the database to grow past the + /// connection's PRAGMA max_page_count ceiling, with SQLite's SQLITE_FULL. SQLite + /// (and Turso's Pager::allocate_page) refuse the page that would cross the ceiling + /// while the statement runs; the managed engine builds pages only when a catalog is + /// persisted, so the statement's complete result is measured before it is published. Pages + /// on the freelist are reused first: only growth beyond the current size counts. + /// + /// + /// The page count the catalog needs, or when no ceiling applies (the + /// default ceiling, or MVCC, whose rows stay in the logical log until a checkpoint). + /// + internal uint? EnsureCatalogFitsPageLimit( + SchemaCatalog catalog, + uint maximumPageCount, + PragmaHeaderMetadata? pragmaHeader = null, + bool forceFullRewrite = false) { + if (maximumPageCount >= SqlitePageLimits.DefaultMaximumPageCount) + return null; + lock (_gate) { - // In-memory drops move pages onto the freelist the way SQLite's pager does; - // the file-backed path reads the real header field. + if (_mvStore is not null) + return null; + + uint current; + uint required; if (_fileStore is null) - return _inMemoryFreelistPages; - return _fileCatalogVersion.FreelistPageCount; + { + current = _inMemoryInitialized ? _inMemoryHighWaterPages : 0; + required = CountInMemoryLivePagesLocked(catalog); + } + else + { + current = _fileCatalogVersion.DatabaseSizeInPages; + if (TryPlanFilePersistedPageCountLocked(catalog, pragmaHeader, forceFullRewrite) is not { } planned) + return null; + required = planned; + } + + if (required > current && required > maximumPageCount) + throw new EmbeddedDatabaseFullException(); + return required; + } + } + + /// + /// The size, in pages, persisting would publish, or + /// when the persist path cannot plan it (it would fail on its own at + /// COMMIT, with its own error). Callers must hold _gate. + /// + private uint? TryPlanFilePersistedPageCountLocked( + SchemaCatalog catalog, + PragmaHeaderMetadata? pragmaHeader, + bool forceFullRewrite) + { + try + { + return _fileStore!.PlanPersistedPageCount( + catalog.Tables, + catalog.Views, + catalog.Triggers, + catalog.VirtualTables, + pragmaHeader, + forceFullRewrite, + previousTables: _tables); + } + catch (Exception exception) when (exception is EmbeddedSqlException + or InvalidOperationException + or ArgumentException + or InvalidDataException + or OverflowException + or NotSupportedException) + { + return null; } } + /// + /// Records the exact page count a just-published :memory: catalog was measured at, so + /// the high-water mark (and therefore the freelist) reflects pages it allocated. + /// + private void RecordInMemoryPublishedPageCountLocked(uint? livePages) + { + if (_fileStore is null && livePages is { } pages) + FoldInMemoryHighWaterLocked(pages); + } + internal SqliteTextEncoding GetTextEncoding() { lock (_gate) @@ -4280,7 +4493,8 @@ internal MvccCheckpointResult RunMvccCheckpoint( string? mode, TimeSpan busyTimeout = default, SqliteSynchronousMode synchronousMode = SqliteSynchronousMode.Full, - MvccTxId? permittedTransaction = null) + MvccTxId? permittedTransaction = null, + uint maxPageCount = SqlitePageLimits.DefaultMaximumPageCount) { synchronousMode.Validate(nameof(synchronousMode)); lock (_gate) @@ -4333,12 +4547,17 @@ internal MvccCheckpointResult RunMvccCheckpoint( } else { + // MVCC keeps committed rows in the logical log, so this is where they first + // need b-tree pages and where the checkpointing connection's + // max_page_count applies (Turso fails the checkpoint with DatabaseFull and + // keeps the log intact; nothing below has run yet). PersistFileCatalog( merged, pragmaHeader: null, forceFullRewrite: false, busyTimeout, - checkpointAfterCommit: false); + checkpointAfterCommit: false, + maxPageCount: maxPageCount); } stateMachine.Enter(MvccCheckpointPhase.PersistPageWal); @@ -4526,7 +4745,8 @@ internal SqliteJournalMode SwitchJournalMode( internal void MigratePageSize( int pageSize, TimeSpan busyTimeout = default, - SqliteSynchronousMode synchronousMode = SqliteSynchronousMode.Full) + SqliteSynchronousMode synchronousMode = SqliteSynchronousMode.Full, + uint maxPageCount = SqlitePageLimits.DefaultMaximumPageCount) { synchronousMode.Validate(nameof(synchronousMode)); lock (_gate) @@ -4559,7 +4779,7 @@ internal void MigratePageSize( foreach (var table in _tables.Values) _ = table.Rows; if (pageSize == _fileCatalogVersion.PageSize) - _fileStore.Compact(); + _fileStore.CompactWithinPageLimit(maxPageCount); else _fileStore.MigratePageSize(pageSize, _tables, _views, _triggers, _virtualTables); _fileStore.AdoptCommittedTables(_tables); @@ -4569,6 +4789,22 @@ internal void MigratePageSize( } } + /// + /// VACUUM on a :memory: database. The heap catalog is already compact, so the only + /// observable effect is SQLite's: the freelist is released and PRAGMA page_count drops + /// to the pages the live image needs. + /// + internal void CompactInMemoryPageModel() + { + lock (_gate) + { + if (_fileStore is not null || !_inMemoryInitialized) + return; + + _inMemoryHighWaterPages = CountInMemoryLivePagesLocked(LiveCatalogLocked()); + } + } + internal void VacuumInto( string destinationPath, int pageSize, @@ -4723,7 +4959,8 @@ private void PersistFileCatalog( bool forceFullRewrite = false, TimeSpan busyTimeout = default, bool checkpointAfterCommit = true, - IReadOnlyList<(string TableName, EmbeddedTable Table, EmbeddedIndex Index)>? targetedIndexRebuild = null) + IReadOnlyList<(string TableName, EmbeddedTable Table, EmbeddedIndex Index)>? targetedIndexRebuild = null, + uint maxPageCount = SqlitePageLimits.DefaultMaximumPageCount) { if (_fileStore is null || _fileSystem is null || _fileCatalogWriteLock is null) throw new InvalidOperationException("The managed file catalog persistence state is not initialized."); @@ -4744,7 +4981,8 @@ private void PersistFileCatalog( pragmaHeader, forceFullRewrite, previousTables: _tables, - targetedIndexRebuild: targetedIndexRebuild) + targetedIndexRebuild: targetedIndexRebuild, + maximumPageCount: maxPageCount) : _fileStore.PersistForMvccCheckpoint( catalog.Tables, catalog.Views, @@ -4752,7 +4990,8 @@ private void PersistFileCatalog( catalog.VirtualTables, pragmaHeader, forceFullRewrite, - previousTables: _tables); + previousTables: _tables, + maximumPageCount: maxPageCount); PublishCatalog(catalog, committedVersion); } catch (EmbeddedPostCommitMaintenanceException) @@ -6551,19 +6790,18 @@ private ExecutionResult ExecuteCreateTypeDefinition( /// The connection-dependent facts and checks a DDL compilation needs. /// /// The schema the statement is resolved against. - /// - /// Whether the compilation enforces PRAGMA max_page_count. EXPLAIN passes - /// : describing a program allocates no page and must not consult a runtime - /// storage limit. - /// - private DdlCompilationContext CreateDdlCompilationContext( - SchemaCatalog catalog, - bool enforceMaxPageCount = true) + /// + /// The compiler's page-reservation hook is inert here: PRAGMA max_page_count is a + /// per-connection ceiling enforced against the pages the statement's finished catalog + /// actually needs (see and the pager growth + /// ceiling), not against a per-object estimate made while compiling. + /// + private DdlCompilationContext CreateDdlCompilationContext(SchemaCatalog catalog) => new( catalog, GetPragmaHeaderMetadata().SchemaVersion, ValidateCheckConstraintFunctions, - enforceMaxPageCount ? EnforceMaxPageCountForCatalogChange : static _ => { }, + static _ => { }, Database: 0, HasCollation, IsRegisteredScalarFunction, @@ -7458,7 +7696,6 @@ private EmbeddedTable GetOrCreateSqliteStat1Table(SchemaCatalog catalog) throw new EmbeddedSqlException($"object name reserved for internal use: {SqliteStat1TableName}"); } - EnforceMaxPageCountForCatalogChange(1); var statistics = new EmbeddedTable( SqliteStat1TableName, [ @@ -28524,7 +28761,7 @@ UnaryExpression unary when TryMapArithmeticOperator(unary.Operator, out _) /// /// The compilation context EXPLAIN uses. It resolves names against the same schema execution - /// would see, but never enforces PRAGMA max_page_count: describing a program allocates no page. + /// would see; describing a program allocates no page. /// private DdlCompilationContext CreateExplainDdlCompilationContext(QueryContext context) => CreateDdlCompilationContext( @@ -28540,8 +28777,7 @@ context.VirtualTables is null ? new Dictionary(StringComparer.OrdinalIgnoreCase) : new Dictionary( context.VirtualTables, - StringComparer.OrdinalIgnoreCase)), - enforceMaxPageCount: false); + StringComparer.OrdinalIgnoreCase))); /// /// The CREATE TABLE a CREATE TABLE AS SELECT lowers to, with its columns derived from the @@ -57615,6 +57851,10 @@ public sealed partial class EmbeddedConnection : IDisposable private bool _deferForeignKeys; private bool _recursiveTriggers; private readonly Dictionary _cacheSizes = []; + // PRAGMA max_page_count is a property of this connection's pager for each schema, as in + // SQLite and Turso (Pager::max_page_count): never persisted, never shared with another + // connection on the same database, and back to the default for every new connection. + private readonly Dictionary _maxPageCounts = []; private readonly Dictionary _cacheSpills = []; private int _tempStore; private bool _ignoreCheckConstraints; @@ -59105,6 +59345,7 @@ public void ResetForPooling() _ignoreCheckConstraints = false; _requireWhere = false; _cacheSizes.Clear(); + _maxPageCounts.Clear(); _cacheSpills.Clear(); _synchronousModes.Clear(); ReleaseExclusiveLockingModes(); @@ -59536,6 +59777,7 @@ private static void DisposeDatabaseLeases(IEnumerable? leases) private void ResetTemporaryDatabase() { _cacheSizes.Remove(_tempDatabase); + _maxPageCounts.Remove(_tempDatabase); _cacheSpills.Remove(_tempDatabase); _synchronousModes.Remove(_tempDatabase); ReleaseExclusiveLockingMode(_tempDatabase); @@ -59814,7 +60056,8 @@ or AttachDatabaseStatement vdbeExecutionOptions: vdbeExecutionOptions, synchronousMode: GetSynchronousMode(routed.Database), sequenceSession: _sequenceSession, - describeJournalMode: DescribeJournalModeForSchema); + describeJournalMode: DescribeJournalModeForSchema, + maxPageCount: GetMaxPageCount(routed.Database)); } catch (Exception failure) when (failure is not EmbeddedConflictFailException @@ -59876,6 +60119,23 @@ or AttachDatabaseStatement describeJournalMode: DescribeJournalModeForSchema); if (routedMayMutate) cancellationToken.ThrowIfCancellationRequested(); + // SQLite allocates a statement's pages while it runs and fails it with + // SQLITE_FULL there; the managed engine writes pages only at COMMIT, so the + // finished statement catalog is measured against this connection's + // max_page_count before the transaction adopts it. + if (routedMayMutate && result.Changed) + { + _ = routed.Database.EnsureCatalogFitsPageLimit( + statementCatalog, + GetMaxPageCount(routed.Database), + routed.Database.IsFileBacked + && !transactionState.HasSnapshotPragmaHeader + && !transactionState.HasSchemaChanges + && !EmbeddedDatabase.MayChangeSchema(routed.Statement) + ? null + : transactionState.PragmaHeader, + transactionState.ForceFullCatalogRewrite || result.ForceFullCatalogRewrite); + } // The catalog overload used for transactional statements does not // record change counters itself, so mirror the autocommit path here: // changes()/total_changes() must observe in-transaction writes. @@ -60153,7 +60413,7 @@ or AttachDatabaseStatement ReleaseConcurrentSchemaGateIfUnused(); throw; } - catch + catch (Exception failure) { if (changeDataCaptureSnapshot is { } snapshot) changeDataCapture?.Restore(snapshot); @@ -60166,6 +60426,16 @@ or AttachDatabaseStatement if (statementOverlayCheckpoint is not null) transactionState?.Overlay?.RestoreCheckpoint(statementOverlayCheckpoint); ReleaseConcurrentSchemaGateIfUnused(); + // SQLITE_FULL without a statement journal rolls back the whole transaction + // (sqlite3VdbeHalt); with one, only the statement. + if (failure is EmbeddedDatabaseFullException + && transactionState is not null + && _transactionDatabases is not null + && RollsBackTransactionOnDatabaseFull(routed.Statement, transactionState.Catalog)) + { + ResetTransactionState(); + FireRollbackHook(); + } throw; } finally @@ -60453,6 +60723,7 @@ private ExecutionResult ExecuteDetach(DetachDatabaseStatement statement) _attachedDatabases.Remove(statement.Alias); _pendingPageSizes.Remove(attachment.Database); _cacheSizes.Remove(attachment.Database); + _maxPageCounts.Remove(attachment.Database); _cacheSpills.Remove(attachment.Database); _synchronousModes.Remove(attachment.Database); ReleaseExclusiveLockingMode(attachment.Database); @@ -60512,7 +60783,7 @@ private ExecutionResult ExecutePragmaEncoding(PragmaEncodingStatement statement) private ExecutionResult ExecutePragmaPageCount(PragmaPageCountStatement statement) { var database = ResolvePragmaDatabase(statement.Schema); - return new ExecutionResult(["page_count"], [[SqlValue.Integer(database.GetPageCount())]], 0); + return new ExecutionResult(["page_count"], [[SqlValue.Integer(GetConnectionVisiblePageCount(database))]], 0); } private ExecutionResult ExecutePragmaFreelistCount(PragmaFreelistCountStatement statement) @@ -64538,7 +64809,8 @@ private void BeginConcurrentSchemaChange() "TRUNCATE", BusyTimeout, GetSynchronousMode(_database), - permittedTransaction: txId); + permittedTransaction: txId, + maxPageCount: GetMaxPageCount(_database)); if (checkpoint.Busy) throw new EmbeddedBusyException(); } @@ -64738,7 +65010,8 @@ private void CommitTransaction() // coexisting change is persisted atomically alongside the rebuilt index. targetedIndexRebuildNames: state.HasNonTargetedIndexRebuildChange ? null - : state.TargetedIndexRebuildNames); + : state.TargetedIndexRebuildNames, + maxPageCount: GetCommitPageLimit(database)); } catch (EmbeddedPostCommitMaintenanceException failure) { @@ -65195,18 +65468,147 @@ private ExecutionResult ExecutePragmaCacheSpill(PragmaCacheSpillStatement statem private ExecutionResult ExecutePragmaMaxPageCount(PragmaMaxPageCountStatement statement) { var database = ResolvePragmaDatabase(statement.Schema); - if (statement.Value is null) - return new ExecutionResult(["max_page_count"], [[SqlValue.Integer(database.MaxPageCount)]], 0); + // SQLite's pragma.c clamps the argument to 0..0xfffffffe, and OP_MaxPgcnt (Turso + // op_max_pgcnt) treats 0 - so also any negative request - as a query. Any other request + // becomes the new ceiling, but never below the pages the database already occupies + // (Turso Pager::set_max_page_count; SQLite sqlite3BtreeLastPage), counting pages this + // connection's open transaction has already allocated. + var requested = statement.Value is { } value + ? value <= 0 ? 0u : (uint)Math.Min(value, (long)SqlitePageLimits.AbsoluteMaximumPageCount) + : 0u; + if (requested == 0) + return new ExecutionResult(["max_page_count"], [[SqlValue.Integer(GetMaxPageCount(database))]], 0); + + var newMax = Math.Max(requested, GetConnectionVisiblePageCount(database)); + _maxPageCounts[database] = newMax; + return new ExecutionResult(["max_page_count"], [[SqlValue.Integer(newMax)]], 0); + } - // Turso treats 0 as a no-op query and clamps any other request to at - // least the current database size (set_max_page_count). - if (statement.Value.Value == 0) - return new ExecutionResult(["max_page_count"], [[SqlValue.Integer(database.MaxPageCount)]], 0); + /// This connection's PRAGMA max_page_count ceiling for . + private uint GetMaxPageCount(EmbeddedDatabase database) + => _maxPageCounts.GetValueOrDefault(database, SqlitePageLimits.DefaultMaximumPageCount); - var requested = statement.Value.Value < 0 ? 0 : (uint)statement.Value.Value; - var newMax = Math.Max(requested, database.GetPageCount()); - database.MaxPageCount = newMax; - return new ExecutionResult(["max_page_count"], [[SqlValue.Integer(newMax)]], 0); + /// + /// The ceiling an explicit transaction's COMMIT is held to. An MVCC transaction publishes its + /// logical-log frame before the managed engine materializes its pages, so (as in Turso, where + /// the MVCC commit never touches the page ceiling) it must not fail on the ceiling after the + /// log already made it durable; its pages are held to the ceiling at checkpoint instead. + /// Autocommit MVCC writes build their pages directly, so they keep the ceiling. + /// + private uint GetCommitPageLimit(EmbeddedDatabase database) + => database.IsMvccEnabled ? SqlitePageLimits.DefaultMaximumPageCount : GetMaxPageCount(database); + + /// + /// PRAGMA page_count as this connection sees it: the committed size, or, inside a write + /// transaction that changed , the size including the pages that + /// transaction has allocated (SQLite reports the pager's current size). + /// + private uint GetConnectionVisiblePageCount(EmbeddedDatabase database) + { + var state = GetTransactionState(database); + return database.GetPageCount(state is { HasChanges: true } ? state.Catalog : null); + } + + /// + /// Whether SQLite would roll back the whole transaction, not just the statement, when this + /// statement hits SQLITE_FULL inside an explicit transaction. + /// + /// + /// sqlite3VdbeHalt treats SQLITE_FULL as a special error: it rolls back only the + /// statement when the statement runs under a statement journal, and otherwise rolls back the + /// entire transaction. SQLite (and Turso's translate/stmt_journal.rs) skips the + /// statement journal for a write that can only touch a single row, because such a write is + /// atomic on its own: a one-row INSERT ... VALUES without triggers, REPLACE, UPSERT or + /// AUTOINCREMENT, and an UPDATE/DELETE addressed to one rowid without triggers, + /// REPLACE or foreign keys. Every other statement (multi-row DML, DDL, CREATE INDEX) keeps + /// the transaction open with only its own changes undone. + /// + private bool RollsBackTransactionOnDatabaseFull(ParsedStatement statement, EmbeddedDatabase.SchemaCatalog catalog) + { + switch (statement) + { + case InsertStatement insert: + { + if (insert.Source is not null + || insert.Rows.Count > 1 + || insert.Upsert is not null + || insert.ConflictAlgorithm == InsertConflictAlgorithm.Replace + || !catalog.Tables.TryGetValue(insert.TableName, out var table) + || table.IsAutoIncrement + || HasTriggers(catalog, insert.TableName) + || UsesReplaceConstraint(table)) + { + return false; + } + + return true; + } + case UpdateStatement update: + return update.From is null + && update.ConflictAlgorithm != InsertConflictAlgorithm.Replace + && catalog.Tables.TryGetValue(update.TableName, out var updated) + && !UsesReplaceConstraint(updated) + && !HasTriggers(catalog, update.TableName) + && !HasForeignKeyRelationships(catalog, update.TableName, updated) + && AddressesSingleRowid(update.Where, updated); + case DeleteStatement delete: + return catalog.Tables.TryGetValue(delete.TableName, out var deleted) + && !HasTriggers(catalog, delete.TableName) + && !HasForeignKeyRelationships(catalog, delete.TableName, deleted) + && AddressesSingleRowid(delete.Where, deleted); + default: + return false; + } + + bool HasTriggers(EmbeddedDatabase.SchemaCatalog schema, string tableName) + => schema.Triggers.Values.Any(trigger => + string.Equals(trigger.TableName, tableName, StringComparison.OrdinalIgnoreCase)) + || (_tempDatabase.HasTriggers + && _tempDatabase.SnapshotTriggers().Any(trigger => + string.Equals(trigger.TableName, tableName, StringComparison.OrdinalIgnoreCase))); + + bool HasForeignKeyRelationships(EmbeddedDatabase.SchemaCatalog schema, string tableName, EmbeddedTable table) + => _foreignKeys + && (table.ForeignKeys.Count != 0 + || schema.Tables.Values.Any(other => other.ForeignKeys.Any(foreignKey => + string.Equals(foreignKey.ParentTable, tableName, StringComparison.OrdinalIgnoreCase)))); + + static bool UsesReplaceConstraint(EmbeddedTable table) + => table.TablePrimaryKeyConflictAlgorithm == InsertConflictAlgorithm.Replace + || table.Indexes.Any(index => index.ConflictAlgorithm == InsertConflictAlgorithm.Replace) + || table.ColumnDefinitions.Any(column => + column.PrimaryKeyConflictAlgorithm == InsertConflictAlgorithm.Replace + || column.UniqueConflictAlgorithm == InsertConflictAlgorithm.Replace + || column.NotNullConflictAlgorithm == InsertConflictAlgorithm.Replace); + + static bool AddressesSingleRowid(Expression? where, EmbeddedTable table) + { + if (where is not BinaryExpression { Operator: BinaryOperator.Equal } equality) + return false; + + return (IsRowidReference(equality.Left, table) && IsConstant(equality.Right)) + || (IsRowidReference(equality.Right, table) && IsConstant(equality.Left)); + } + + static bool IsRowidReference(Expression expression, EmbeddedTable table) + { + if (expression is not ColumnExpression column || table.WithoutRowid) + return false; + + var name = column.UnqualifiedName ?? column.Name; + if (name.Equals("rowid", StringComparison.OrdinalIgnoreCase) + || name.Equals("_rowid_", StringComparison.OrdinalIgnoreCase) + || name.Equals("oid", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return table.RowidAliasColumnIndex >= 0 + && name.Equals(table.Columns[table.RowidAliasColumnIndex], StringComparison.OrdinalIgnoreCase); + } + + static bool IsConstant(Expression expression) + => expression is LiteralExpression or ParameterExpression; } private ExecutionResult ExecutePragmaIgnoreCheckConstraints(PragmaIgnoreCheckConstraintsStatement statement) @@ -65284,7 +65686,8 @@ private ExecutionResult ExecutePragmaWalCheckpoint(PragmaWalCheckpointStatement var result = database.RunMvccCheckpoint( statement.Mode, BusyTimeout, - GetSynchronousMode(database)); + GetSynchronousMode(database), + maxPageCount: GetMaxPageCount(database)); return new ExecutionResult( columns, [[ @@ -65563,7 +65966,10 @@ private ExecutionResult ExecuteVacuum(VacuumStatement statement, SqlValue[] para if (database.IsReadOnly) throw new EmbeddedSqlException("attempt to write a readonly database"); if (!database.IsFileBacked && statement.Into is null) + { + database.CompactInMemoryPageModel(); return ExecutionResult.Empty; + } // VACUUM rewrites the whole database, so it has to lose to a connection // holding a write transaction just like any other write. @@ -65583,7 +65989,8 @@ private ExecutionResult ExecuteVacuum(VacuumStatement statement, SqlValue[] para database.MigratePageSize( targetPageSize, busyTimeout: BusyTimeout, - synchronousMode: GetSynchronousMode(database)); + synchronousMode: GetSynchronousMode(database), + maxPageCount: GetMaxPageCount(database)); } else { diff --git a/src/Ahtola.Core/EmbeddedFileStore.cs b/src/Ahtola.Core/EmbeddedFileStore.cs index 92ce9317..609fd03e 100644 --- a/src/Ahtola.Core/EmbeddedFileStore.cs +++ b/src/Ahtola.Core/EmbeddedFileStore.cs @@ -3031,24 +3031,33 @@ public FileCatalogVersion Persist( PragmaHeaderMetadata? pragmaHeader = null, bool forceFullRewrite = false, IReadOnlyDictionary? previousTables = null, - IReadOnlyList<(string TableName, EmbeddedTable Table, EmbeddedIndex Index)>? targetedIndexRebuild = null) - => PersistCore( - tables, - views, - triggers, - virtualTables, - reclaimTrailingPages: false, - incrementSchemaCookie: false, - pragmaHeader, - forceFullRewrite, - previousTables, - targetedIndexRebuild: targetedIndexRebuild); + IReadOnlyList<(string TableName, EmbeddedTable Table, EmbeddedIndex Index)>? targetedIndexRebuild = null, + uint maximumPageCount = SqlitePageLimits.DefaultMaximumPageCount) + => WithGrowthCeiling( + maximumPageCount, + () => PersistCore( + tables, + views, + triggers, + virtualTables, + reclaimTrailingPages: false, + incrementSchemaCookie: false, + pragmaHeader, + forceFullRewrite, + previousTables, + targetedIndexRebuild: targetedIndexRebuild)); /// /// Materializes an MVCC checkpoint into pager WAL pages without reclaiming /// those frames. The checkpoint state machine must first backfill the main /// store and retire the logical log before it resets the WAL. /// + /// + /// is the checkpointing connection's + /// PRAGMA max_page_count: MVCC keeps committed rows in the logical log, so the page + /// ceiling is only reached here, when they are materialized into b-tree pages (Turso's + /// checkpoint fails with DatabaseFull and leaves the log intact). + /// internal FileCatalogVersion PersistForMvccCheckpoint( IReadOnlyDictionary tables, IReadOnlyDictionary views, @@ -3056,18 +3065,100 @@ internal FileCatalogVersion PersistForMvccCheckpoint( IReadOnlyDictionary virtualTables, PragmaHeaderMetadata? pragmaHeader = null, bool forceFullRewrite = false, - IReadOnlyDictionary? previousTables = null) - => PersistCore( - tables, - views, - triggers, - virtualTables, - reclaimTrailingPages: false, - incrementSchemaCookie: false, - pragmaHeader, - forceFullRewrite, - previousTables, - checkpointAfterCommit: false); + IReadOnlyDictionary? previousTables = null, + uint maximumPageCount = SqlitePageLimits.DefaultMaximumPageCount) + => WithGrowthCeiling( + maximumPageCount, + () => PersistCore( + tables, + views, + triggers, + virtualTables, + reclaimTrailingPages: false, + incrementSchemaCookie: false, + pragmaHeader, + forceFullRewrite, + previousTables, + checkpointAfterCommit: false)); + + /// + /// Computes the database size, in pages, that persisting this catalog would publish, + /// without writing, locking or publishing anything: the exact persist path runs (the + /// incremental b-tree mutation, a bounded leaf rewrite, or the full catalog rewrite, whichever + /// the real commit would pick) and is stopped at the pager's write entry point. Returns the + /// committed size when the catalog needs no page write at all. + /// + /// + /// This is how statement-level PRAGMA max_page_count enforcement sees the pages an + /// explicit transaction would need: the managed engine only writes pages at COMMIT, while + /// SQLite allocates them while the statement runs and fails that statement with + /// SQLITE_FULL. + /// + internal uint PlanPersistedPageCount( + IReadOnlyDictionary tables, + IReadOnlyDictionary views, + IReadOnlyDictionary triggers, + IReadOnlyDictionary virtualTables, + PragmaHeaderMetadata? pragmaHeader, + bool forceFullRewrite, + IReadOnlyDictionary? previousTables) + { + ThrowIfDisposed(); + ThrowIfPostCommitMaintenanceFaulted(); + if (_pager.PlanCommitsOnly) + throw new InvalidOperationException("A managed persist plan is already in progress."); + + // Every persist path stops at the pager before it publishes anything, but the plan must + // never be able to move this store's committed bookkeeping even if a future path learns + // to return without a page write. + var committedTables = _committedTables; + var header = _header; + var tableRootPages = _tableRootPages; + var indexRootPages = _indexRootPages; + var lastSchemaSignature = _lastSchemaSignature; + _pager.PlanCommitsOnly = true; + try + { + _ = PersistCore( + tables, + views, + triggers, + virtualTables, + reclaimTrailingPages: false, + incrementSchemaCookie: false, + pragmaHeader, + forceFullRewrite, + previousTables); + return _pager.CommittedPageCount; + } + catch (SqlitePagerCommitPlannedException planned) + { + return planned.TargetDatabaseSizeInPages; + } + finally + { + _pager.PlanCommitsOnly = false; + _committedTables = committedTables; + _header = header; + _tableRootPages = tableRootPages; + _indexRootPages = indexRootPages; + _lastSchemaSignature = lastSchemaSignature; + } + } + + private T WithGrowthCeiling(uint maximumPageCount, Func persist) + { + var previous = _pager.MaximumPageCount; + _pager.MaximumPageCount = maximumPageCount; + try + { + return persist(); + } + finally + { + _pager.MaximumPageCount = previous; + } + } /// /// Builds a private, not-yet-published VACUUM destination image in one @@ -3107,19 +3198,26 @@ private FileCatalogVersion PersistForVacuumTarget( /// the managed writer can represent, then checkpoints and physically removes /// its retired suffix. /// - internal void Compact() + internal void Compact() => CompactWithinPageLimit(SqlitePageLimits.DefaultMaximumPageCount); + + /// + /// held to the requesting connection's PRAGMA max_page_count. + /// + internal void CompactWithinPageLimit(uint maximumPageCount) { ThrowIfDisposed(); var catalog = Load(); - _ = PersistCore( - catalog.Tables, - catalog.Views, - catalog.Triggers, - catalog.VirtualTables, - reclaimTrailingPages: true, - incrementSchemaCookie: true, - pragmaHeader: null, - forceFullRewrite: true); + _ = WithGrowthCeiling( + maximumPageCount, + () => PersistCore( + catalog.Tables, + catalog.Views, + catalog.Triggers, + catalog.VirtualTables, + reclaimTrailingPages: true, + incrementSchemaCookie: true, + pragmaHeader: null, + forceFullRewrite: true)); } internal FileCatalogVersion UpdatePragmaHeader(PragmaHeaderMetadata metadata) @@ -3794,6 +3892,165 @@ private static void CollectRewritePlanPages( image[SchemaRootPage] = schemaTree.RootPage; } + /// + /// Counts the pages a complete, compacted image of this catalog occupies when this store + /// serializes it: page 1 (the sqlite_schema root), one root per table and index b-tree, + /// every interior, leaf and overflow page those trees need, and any extra sqlite_schema + /// pages. The trees are built by the same page builders a full rewrite (VACUUM, VACUUM INTO, + /// page-size migration) uses, so the result is exactly the size that rewrite would publish + /// for this store's page size, reserved space and text encoding. + /// + /// + /// A :memory: database keeps its rows as heap objects and has no pager of its own, so + /// this is the honest source of its PRAGMA page_count and the size its + /// max_page_count ceiling is checked against. Per-tree counts are cached by row-store + /// lineage and revision, so a statement only re-serializes the trees it changed. + /// + internal uint CountFreshImagePages( + IReadOnlyDictionary tables, + IReadOnlyDictionary views, + IReadOnlyDictionary triggers, + IReadOnlyDictionary virtualTables, + FreshImagePageCountCache cache) + { + ThrowIfDisposed(); + ArgumentNullException.ThrowIfNull(cache); + + // Root pages are reserved exactly as PersistCore reserves them for a compacted rewrite + // (tables in name order, then each table's indexes), because sqlite_schema stores the + // root page numbers as varints and its own size depends on them. + var tableNames = tables.Keys.OrderBy(name => name, StringComparer.OrdinalIgnoreCase).ToArray(); + var rootPages = new Dictionary(StringComparer.OrdinalIgnoreCase); + var indexRootPages = new Dictionary(StringComparer.OrdinalIgnoreCase); + var nextRootPage = SchemaRootPage + 1; + foreach (var name in tableNames) + rootPages[name] = nextRootPage++; + foreach (var name in tableNames) + { + foreach (var index in tables[name].Indexes) + indexRootPages.TryAdd(index.Name, nextRootPage++); + } + + var livePages = new HashSet(StringComparer.OrdinalIgnoreCase); + long total = nextRootPage - 1; + foreach (var name in tableNames) + { + var table = tables[name]; + total += cache.GetTreePages( + FreshImagePageCountCache.TableKey(name, table), + livePages, + () => CountTableTreePages(name, table)) - 1; + foreach (var index in table.Indexes) + { + total += cache.GetTreePages( + FreshImagePageCountCache.IndexKey(name, table, index), + livePages, + () => CountIndexTreePages(name, table, index)) - 1; + } + } + + cache.RetainOnly(livePages); + var schemaAllocator = new CountingPageAllocator(); + _ = BuildSchemaTree( + BuildSchemaEntries(tables, views, triggers, virtualTables, rootPages, indexRootPages), + schemaAllocator); + total += schemaAllocator.Count; + return total >= uint.MaxValue ? uint.MaxValue : (uint)total; + } + + private uint CountTableTreePages(string name, EmbeddedTable table) + { + var allocator = new CountingPageAllocator(); + _ = allocator.ReservePage(); + _ = table.WithoutRowid + ? BuildWithoutRowidTableTree(name, table, allocator) + : BuildTableTree(name, table, allocator); + return allocator.Count; + } + + private uint CountIndexTreePages(string tableName, EmbeddedTable table, EmbeddedIndex index) + { + var allocator = new CountingPageAllocator(); + _ = allocator.ReservePage(); + _ = BuildIndexTree(tableName, table, index, allocator); + return allocator.Count; + } + + /// + /// Counts page reservations without assigning real page numbers. Every page-number field a + /// b-tree stores (child pointers, overflow links) is a fixed four bytes, so the number of + /// pages a tree needs does not depend on which numbers it is given. + /// + private sealed class CountingPageAllocator : IIndexRebuildPageAllocator + { + public uint Count { get; private set; } + + public uint ReservePage() + { + Count = checked(Count + 1); + // Any non-zero number that is not page 1 is a valid placeholder. + return Count + SchemaRootPage; + } + } + + /// + /// Per-tree page counts for , keyed by the tree's name, + /// definition and the row-store lineage/revision that produced it. A statement's working + /// clone shares its source's lineage and revision until it mutates rows, so unchanged trees + /// are never re-serialized. + /// + internal sealed class FreshImagePageCountCache + { + private readonly Dictionary _trees = + new(StringComparer.OrdinalIgnoreCase); + + internal static (string Slot, string Key) TableKey(string name, EmbeddedTable table) + => ("t\u0001" + name, + string.Join( + '\u0001', + table.Rows.LineageId, + table.Rows.Revision, + table.WithoutRowid, + table.Strict, + table.Sql, + table.ColumnDefinitions.Length)); + + internal static (string Slot, string Key) IndexKey(string tableName, EmbeddedTable table, EmbeddedIndex index) + => ("i\u0001" + index.Name, + string.Join( + '\u0001', + tableName, + table.Rows.LineageId, + table.Rows.Revision, + table.Sql, + index.Unique, + index.Origin, + index.WhereSql, + index.Sql, + index.Method, + string.Join( + '\u0002', + index.Columns.Select(column => + $"{column.Name}\u0003{column.ColumnIndex}\u0003{column.Collation}\u0003{column.Descending}\u0003{column.NullPlacement}\u0003{column.ExpressionSql}")))); + + internal uint GetTreePages((string Slot, string Key) key, ISet live, Func count) + { + live.Add(key.Slot); + if (_trees.TryGetValue(key.Slot, out var cached) && string.Equals(cached.Key, key.Key, StringComparison.Ordinal)) + return cached.Pages; + + var pages = count(); + _trees[key.Slot] = (key.Key, pages); + return pages; + } + + internal void RetainOnly(ISet live) + { + foreach (var slot in _trees.Keys.Where(slot => !live.Contains(slot)).ToArray()) + _trees.Remove(slot); + } + } + /// The number of changed rows above which a complete rewrite is preferred. /// /// A bulk change touches most of the database anyway, and one rewrite packs @@ -11137,7 +11394,7 @@ null or LiteralExpression or CurrentTimeExpression or ColumnExpression or RaiseE private PreparedTableTree BuildWithoutRowidTableTree( string name, EmbeddedTable table, - RebuildPageAllocator allocator) + IIndexRebuildPageAllocator allocator) { var primaryKeySchema = ValidateWithoutRowidTableRepresentable(name, table); var comparer = CreatePrimaryKeyComparer(primaryKeySchema); @@ -11167,7 +11424,7 @@ private PreparedTableTree BuildWithoutRowidTableTree( private PreparedTableTree BuildTableTree( string name, EmbeddedTable table, - RebuildPageAllocator allocator) + IIndexRebuildPageAllocator allocator) { var leafGroups = PartitionTableLeafCells(name, table); var overflowPages = new List(); @@ -11389,7 +11646,7 @@ private SqliteTableLeafCell CreateTableLeafPlanningCell(long rowId, ReadOnlySpan private byte[] BuildTableLeafPage( IReadOnlyList cells, - RebuildPageAllocator allocator, + IIndexRebuildPageAllocator allocator, ICollection overflowPages) { var builder = new SqliteTableLeafPageBuilder(_pageSize, _usableSpace, isFirstPage: false); @@ -12138,7 +12395,7 @@ private byte[] BuildIndexLeafPage( private SqliteTableLeafCell CreateTableLeafCell( long rowId, byte[] record, - RebuildPageAllocator allocator, + IIndexRebuildPageAllocator allocator, ICollection overflowPages) { var layout = SqlitePayloadLayout.Calculate( @@ -12796,7 +13053,7 @@ private bool IsCollationNameResolvable(string name) private PreparedSchemaTree BuildSchemaTree( IReadOnlyList entries, - RebuildPageAllocator allocator) + IIndexRebuildPageAllocator allocator) { var cells = new List(entries.Count); var overflowPages = new List(); diff --git a/src/Ahtola.Core/Storage/SqliteBtreePageIo.cs b/src/Ahtola.Core/Storage/SqliteBtreePageIo.cs index 4dbdc4f3..49ed9217 100644 --- a/src/Ahtola.Core/Storage/SqliteBtreePageIo.cs +++ b/src/Ahtola.Core/Storage/SqliteBtreePageIo.cs @@ -215,8 +215,8 @@ private void EnsureWithinGrowthCeiling(uint pageNumber) { if (pageNumber > MaximumPageCount) { - throw new InvalidOperationException( - $"SQLite database is full: page {pageNumber} exceeds the maximum page count {MaximumPageCount}."); + // SQLITE_FULL: Turso Pager::allocate_page returns LimboError::DatabaseFull. + throw new EmbeddedDatabaseFullException(); } } diff --git a/src/Ahtola.Core/Storage/SqlitePageAllocator.cs b/src/Ahtola.Core/Storage/SqlitePageAllocator.cs index f28e01c7..48382596 100644 --- a/src/Ahtola.Core/Storage/SqlitePageAllocator.cs +++ b/src/Ahtola.Core/Storage/SqlitePageAllocator.cs @@ -133,8 +133,8 @@ public SqlitePageAllocation Allocate() var pageNumber = _nextPageNumber; if (pageNumber > MaximumPageCount) { - throw new InvalidOperationException( - $"SQLite database is full: page {pageNumber} exceeds the maximum page count {MaximumPageCount}."); + // SQLITE_FULL: Turso Pager::allocate_page returns LimboError::DatabaseFull. + throw new EmbeddedDatabaseFullException(); } _nextPageNumber = Advance(pageNumber); diff --git a/src/Ahtola.Core/Storage/SqlitePager.cs b/src/Ahtola.Core/Storage/SqlitePager.cs index 8bc9e325..d288311c 100644 --- a/src/Ahtola.Core/Storage/SqlitePager.cs +++ b/src/Ahtola.Core/Storage/SqlitePager.cs @@ -266,6 +266,36 @@ public uint CommittedPageCount } } + /// + /// The growth ceiling a write transaction must respect: the owning connection's + /// PRAGMA max_page_count (Turso Pager::max_page_count). A transaction whose + /// target size exceeds both this ceiling and the committed size is rejected with + /// SQLITE_FULL before a single frame is written, mirroring Turso + /// Pager::allocate_page refusing the page that would cross the limit. A database + /// that is already larger than the ceiling (grown by another connection) may still be + /// rewritten in place; only growth is refused. + /// + internal uint MaximumPageCount { get; set; } = SqlitePageLimits.DefaultMaximumPageCount; + + /// + /// When set, every write entry point reports the target size it was asked to reach by + /// throwing instead of writing anything, + /// so a caller can learn how large a pending mutation would make the database without + /// committing it (statement-level max_page_count enforcement inside an explicit + /// transaction, where pages are only written at COMMIT). + /// + internal bool PlanCommitsOnly { get; set; } + + private void ThrowIfCommitIsPlannedOrExceedsGrowthCeiling(uint targetDatabaseSizeInPages) + { + if (PlanCommitsOnly) + throw new SqlitePagerCommitPlannedException(targetDatabaseSizeInPages); + + var ceiling = MaximumPageCount; + if (targetDatabaseSizeInPages > ceiling && targetDatabaseSizeInPages > CommittedPageCount) + throw new EmbeddedDatabaseFullException(); + } + /// /// The maximum number of clean main-database page images this pager retains. /// WAL-overlay and transaction images are not part of this cache. @@ -1410,6 +1440,7 @@ private SqlitePagerReadTransaction BeginReadTransactionWithWalIndexReadMark(Time /// public SqlitePagerTransaction BeginTransaction(uint targetDatabaseSizeInPages, TimeSpan? busyTimeout = null) { + ThrowIfCommitIsPlannedOrExceedsGrowthCeiling(targetDatabaseSizeInPages); var configuredBusyTimeout = ResolveBusyTimeout(busyTimeout); var lockStopwatch = configuredBusyTimeout == Timeout.InfiniteTimeSpan ? null @@ -1517,6 +1548,7 @@ internal SqlitePagerTransaction BeginExclusiveRewriteTransaction( uint targetDatabaseSizeInPages, TimeSpan? busyTimeout = null) { + ThrowIfCommitIsPlannedOrExceedsGrowthCeiling(targetDatabaseSizeInPages); var configuredBusyTimeout = ResolveBusyTimeout(busyTimeout); var lockStopwatch = configuredBusyTimeout == Timeout.InfiniteTimeSpan ? null @@ -2193,6 +2225,7 @@ internal void WriteUnpublishedVacuumImage( Func> getPageImage) { ArgumentNullException.ThrowIfNull(getPageImage); + ThrowIfCommitIsPlannedOrExceedsGrowthCeiling(pageCount); using var checkpointLock = _lockManager.EnterCheckpoint( TimeSpan.Zero, useExternalCoordinator: false); diff --git a/src/Ahtola.Core/Storage/SqlitePagerCommitPlannedException.cs b/src/Ahtola.Core/Storage/SqlitePagerCommitPlannedException.cs new file mode 100644 index 00000000..3448f4af --- /dev/null +++ b/src/Ahtola.Core/Storage/SqlitePagerCommitPlannedException.cs @@ -0,0 +1,19 @@ +namespace Ahtola.Core.Storage; + +/// +/// Raised by a in mode at the +/// point a write would begin, carrying the database size that write was about to publish. +/// Nothing has been written, locked or staged in the pager when it is thrown. +/// +/// +/// Deliberately derives from rather than +/// or : persist paths +/// catch those to fall back to a different page layout, and a planning probe must travel +/// straight back to the caller that armed it instead of being mistaken for a layout failure. +/// +internal sealed class SqlitePagerCommitPlannedException(uint targetDatabaseSizeInPages) + : Exception("A planned SQLite pager commit was intercepted before it wrote any page.") +{ + /// The database size, in pages, the intercepted commit would have published. + public uint TargetDatabaseSizeInPages { get; } = targetDatabaseSizeInPages; +} diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index 3e2bdfa1..90809eda 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -201,7 +201,6 @@ partial_idx.sqltest::partial-index-or-right-term-with-range-search | database va partial_idx.sqltest::partial-index-or-term-with-covering-scan | database variant 1/1 (:memory:): expected 3 row(s), got 3 row(s); row 0 expected '1/0/0/SCAN t USING COVERING INDEX idx' but was '1/0/0/SCAN t' partial_idx.sqltest::partial-index-or-term-with-equality-search | database variant 1/1 (:memory:): expected 2 row(s), got 2 row(s); row 0 expected '1/0/0/SEARCH t USING INDEX idx (a=?)' but was '1/0/0/SCAN t' partial_idx.sqltest::partial-index-or-term-with-order | database variant 1/1 (:memory:): expected 3 row(s), got 4 row(s); row 0 expected '1/0/0/SCAN t USING INDEX idx' but was '1/0/0/SCAN t' -sql-error-message-prefix.sqltest::database-full-error-has-no-prefix | database variant 1/1 (:memory:): expected an error but the statement succeeded turso/alter_column.sqltest::alter-column-rejects-index-only-rewrite-mvcc | database variant 1/1 (:memory:): expected an error but the statement succeeded turso/alter_column.sqltest::alter-column-rejects-index-rewrite-mvcc | database variant 1/1 (:memory:): expected an error but the statement succeeded turso/alter_column.sqltest::alter-column-rewrites-expression-index-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' diff --git a/src/Ahtola.Tests/InMemoryPageModelTests.cs b/src/Ahtola.Tests/InMemoryPageModelTests.cs index 581684cb..a3858fb7 100644 --- a/src/Ahtola.Tests/InMemoryPageModelTests.cs +++ b/src/Ahtola.Tests/InMemoryPageModelTests.cs @@ -78,14 +78,14 @@ public void MaxPageCountRejectsGrowthBeyondTheLimit() { using var connection = new EmbeddedDatabase().Connect(); Execute(connection, "PRAGMA max_page_count=1"); - Assert.Throws( + Assert.Catch( () => Execute(connection, "CREATE TABLE t(a)")); using var connection2 = new EmbeddedDatabase().Connect(); Execute(connection2, "PRAGMA max_page_count=2"); Execute(connection2, "CREATE TABLE t(a)"); // Header + one root fit the limit; the second table does not. - Assert.Throws( + Assert.Catch( () => Execute(connection2, "CREATE TABLE t2(b)")); } diff --git a/src/Ahtola.Tests/ManagedMaxPageCountTests.cs b/src/Ahtola.Tests/ManagedMaxPageCountTests.cs new file mode 100644 index 00000000..dfb1e04e --- /dev/null +++ b/src/Ahtola.Tests/ManagedMaxPageCountTests.cs @@ -0,0 +1,430 @@ +using AwesomeAssertions; +using Ahtola.Core; +using Microsoft.Data.Sqlite; + +namespace Ahtola.Tests; + +/// +/// PRAGMA page_count accuracy and PRAGMA max_page_count enforcement, differential +/// against the bundled SQLite where the two engines can run the same script. SQLite semantics +/// covered: the ceiling is per connection and never persisted; setting it clamps to the current +/// size; a write that would grow the database past it fails with SQLITE_FULL ("database or +/// disk is full", result code 13); the failing statement is rolled back, and inside an explicit +/// transaction a single-row write without a statement journal rolls back the whole transaction +/// while any other statement only rolls back itself. MVCC (Turso) only hits the ceiling at +/// checkpoint. +/// +public sealed class ManagedMaxPageCountTests +{ + private const string Full = "ERR:database or disk is full"; + private const string Rows100x400 = + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM c WHERE i < 100) " + + "INSERT INTO t SELECT zeroblob(400) FROM c"; + + private string _directory = null!; + + [SetUp] + public void SetUp() + { + _directory = Path.Combine(Path.GetTempPath(), "ahtola-max-page-count-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_directory); + } + + [TearDown] + public void TearDown() + { + try + { + Directory.Delete(_directory, recursive: true); + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } + } + + private static IEnumerable MemoryPageCountScripts() + { + yield return new TestCaseData( + (object)new[] + { + "PRAGMA page_count", "PRAGMA page_size=1024", "CREATE TABLE t(a)", "PRAGMA page_count", + Rows100x400, "PRAGMA page_count", "PRAGMA freelist_count", + "DELETE FROM t WHERE rowid > 50", "PRAGMA page_count", "PRAGMA freelist_count", + "VACUUM", "PRAGMA page_count", "PRAGMA freelist_count", + }).SetName("MemoryPageCount_RowsDeleteVacuum_1024"); + yield return new TestCaseData( + (object)new[] + { + "CREATE TABLE t(a INTEGER PRIMARY KEY, b TEXT)", + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM c WHERE i < 2000) " + + "INSERT INTO t SELECT i, 'row ' || i FROM c", + "PRAGMA page_count", "CREATE INDEX ib ON t(b)", "PRAGMA page_count", + "DROP INDEX ib", "PRAGMA page_count", "PRAGMA freelist_count", + }).SetName("MemoryPageCount_IndexedTable_4096"); + yield return new TestCaseData( + (object)new[] + { + "PRAGMA page_size=1024", "CREATE TABLE t(a)", "INSERT INTO t VALUES (zeroblob(4000))", + "PRAGMA page_count", "CREATE TABLE u(a, b)", "CREATE TABLE v(a, b)", "PRAGMA page_count", + }).SetName("MemoryPageCount_OverflowAndSchema_1024"); + } + + [TestCaseSource(nameof(MemoryPageCountScripts))] + public void InMemoryPageCountMatchesSqlite(string[] script) + => RunAhtolaMemory(script).Should().Equal(RunSqlite(script, "Data Source=:memory:")); + + [TestCase("delete")] + [TestCase("wal")] + public void FileBackedPageCountMatchesSqlite(string journalMode) + { + string[] script = + [ + $"PRAGMA journal_mode={journalMode}", "CREATE TABLE t(a)", "PRAGMA page_count", + Rows100x400.Replace("zeroblob(400)", "zeroblob(1500)", StringComparison.Ordinal), + "PRAGMA page_count", "CREATE INDEX ta ON t(length(a))", "PRAGMA page_count", + ]; + + var ahtola = RunAhtolaFile(script, Path.Combine(_directory, "ahtola.db")); + var sqlite = RunSqlite(script, $"Data Source={Path.Combine(_directory, "sqlite.db")};Pooling=False"); + ahtola.Should().Equal(sqlite); + ahtola[1].Should().Be("2"); + } + + [Test] + public void MaxPageCountGetSetAndClampMatchSqlite() + { + string[] script = + [ + "PRAGMA max_page_count", "PRAGMA max_page_count = 1000", "PRAGMA max_page_count", + "PRAGMA max_page_count = 0", "PRAGMA max_page_count = -5", "PRAGMA max_page_count", + "CREATE TABLE t(a)", Rows100x400, "PRAGMA page_count", "PRAGMA max_page_count = 1", + "PRAGMA max_page_count = 99999999999", "PRAGMA max_page_count", + ]; + + var ahtola = RunAhtolaMemory(script); + ahtola.Should().Equal(RunSqlite(script, "Data Source=:memory:")); + ahtola[0].Should().Be("4294967294"); + ahtola[^1].Should().Be("4294967294"); + } + + [Test] + public void MaxPageCountIsPerConnectionAndNotPersisted() + { + var path = Path.Combine(_directory, "limit.db"); + using (var database = EmbeddedDatabase.OpenFile(path)) + using (var first = database.Connect()) + using (var second = database.Connect()) + { + Execute(first, "CREATE TABLE t(a)"); + Scalar(first, "PRAGMA max_page_count = 2").Should().Be("2"); + Scalar(second, "PRAGMA max_page_count").Should().Be("4294967294"); + + // Only the connection that set the ceiling is held to it. + Execute(second, "INSERT INTO t VALUES (zeroblob(20000))"); + int.Parse(Scalar(first, "PRAGMA page_count")).Should().BeGreaterThan(2); + Scalar(first, "PRAGMA max_page_count").Should().Be("2"); + Catch(first, "INSERT INTO t VALUES (zeroblob(20000))").Should().Be(Full); + } + + using var reopened = EmbeddedDatabase.OpenFile(path); + using var connection = reopened.Connect(); + Scalar(connection, "PRAGMA max_page_count").Should().Be("4294967294"); + } + + [Test] + public void CorpusDatabaseFullCaseRaisesSqliteFullWithoutPrefix() + { + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + Execute(connection, "PRAGMA page_size = 1024"); + Execute(connection, "PRAGMA max_page_count = 2"); + Execute(connection, "CREATE TABLE f (x)"); + + var failure = Assert.Catch( + () => Execute(connection, "INSERT INTO f VALUES (zeroblob(4000))"))!; + failure.Message.Should().Be("database or disk is full"); + failure.SqliteErrorCode.Should().Be(13); + Scalar(connection, "SELECT count(*) FROM f").Should().Be("0"); + Scalar(connection, "PRAGMA page_count").Should().Be("2"); + Scalar(connection, "PRAGMA integrity_check").Should().Be("ok"); + } + + private static IEnumerable TransactionScripts() + { + const string Setup = "PRAGMA page_size=1024"; + const string FiftyRows = + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM c WHERE i < 50) " + + "INSERT INTO f SELECT printf('%.*c', 300, 'x') || i FROM c"; + yield return Script( + "SingleRowInsertRollsBackTheTransaction", + Setup, "CREATE TABLE f(x)", "PRAGMA max_page_count=4", "BEGIN", "INSERT INTO f VALUES (1)", + "INSERT INTO f VALUES (zeroblob(8000))", "SELECT count(*) FROM f", "COMMIT", "SELECT count(*) FROM f"); + yield return Script( + "MultiRowInsertRollsBackOnlyTheStatement", + Setup, "CREATE TABLE f(x UNIQUE)", "PRAGMA max_page_count=4", "BEGIN", "INSERT INTO f VALUES (1)", + FiftyRows, "SELECT count(*) FROM f", "COMMIT", "SELECT count(*) FROM f"); + yield return Script( + "CreateTableRollsBackOnlyTheStatement", + Setup, "CREATE TABLE f(x)", "PRAGMA max_page_count=3", "BEGIN", "INSERT INTO f VALUES (1)", + "CREATE TABLE g(y)", "CREATE TABLE h(y)", "SELECT count(*) FROM f", "COMMIT", + "SELECT count(*) FROM f", "SELECT count(*) FROM sqlite_schema"); + yield return Script( + "CreateIndexRollsBackOnlyTheStatement", + Setup, "CREATE TABLE f(x)", "INSERT INTO f VALUES (1)", "PRAGMA max_page_count=2", "BEGIN", + "INSERT INTO f VALUES (2)", "CREATE INDEX fx ON f(x)", "SELECT count(*) FROM f", "COMMIT", + "SELECT count(*) FROM f", "SELECT count(*) FROM sqlite_schema"); + yield return Script( + "MultiRowUpdateRollsBackOnlyTheStatement", + Setup, "CREATE TABLE f(x)", "INSERT INTO f VALUES (1)", "INSERT INTO f VALUES (2)", + "PRAGMA max_page_count=2", "BEGIN", "INSERT INTO f VALUES (3)", "UPDATE f SET x = zeroblob(3000)", + "SELECT count(*), sum(length(x)) FROM f", "COMMIT", "SELECT count(*) FROM f"); + yield return Script( + "SingleRowidUpdateRollsBackTheTransaction", + Setup, "CREATE TABLE f(x)", "INSERT INTO f VALUES (1)", "INSERT INTO f VALUES (2)", + "PRAGMA max_page_count=2", "BEGIN", "INSERT INTO f VALUES (3)", + "UPDATE f SET x = zeroblob(3000) WHERE rowid = 1", "SELECT count(*), sum(length(x)) FROM f", + "COMMIT", "SELECT count(*) FROM f"); + yield return Script( + "AutocommitCreateTableAtTheCeiling", + "PRAGMA max_page_count=1", "CREATE TABLE t(a)", "PRAGMA max_page_count=2", "CREATE TABLE t(a)", + "CREATE TABLE t2(b)", "SELECT count(*) FROM sqlite_schema", "PRAGMA page_count"); + + static TestCaseData Script(string name, params string[] statements) + => new TestCaseData((object)statements).SetName("InMemoryTransaction_" + name); + } + + [TestCaseSource(nameof(TransactionScripts))] + public void InMemoryDatabaseFullTransactionSemanticsMatchSqlite(string[] script) + { + var ahtola = RunAhtolaMemory(script); + ahtola.Should().Equal(RunSqlite(script, "Data Source=:memory:")); + ahtola.Should().Contain(Full); + } + + [TestCase("delete")] + [TestCase("wal")] + public void FileBackedDatabaseFullRollsBackAndStaysReopenable(string journalMode) + { + var path = Path.Combine(_directory, $"full-{journalMode}.db"); + using (var database = EmbeddedDatabase.OpenFile(path)) + using (var connection = database.Connect()) + { + Execute(connection, $"PRAGMA journal_mode={journalMode}"); + Execute(connection, "CREATE TABLE t(a)"); + Execute(connection, "INSERT INTO t VALUES (1)"); + Scalar(connection, "PRAGMA page_count").Should().Be("2"); + Scalar(connection, "PRAGMA max_page_count = 3").Should().Be("3"); + + // Autocommit INSERT, CREATE INDEX and CREATE TABLE that need to grow past the ceiling. + Catch(connection, "INSERT INTO t VALUES (zeroblob(20000))").Should().Be(Full); + Execute(connection, "CREATE TABLE u(a)"); + Catch(connection, "CREATE INDEX ua ON u(a)").Should().Be(Full); + Catch(connection, "CREATE TABLE v(a)").Should().Be(Full); + Scalar(connection, "PRAGMA page_count").Should().Be("3"); + Scalar(connection, "SELECT count(*) FROM sqlite_schema").Should().Be("2"); + + // Explicit transaction: a multi-row statement is undone alone and COMMIT keeps the rest. + Execute(connection, "BEGIN"); + Execute(connection, "INSERT INTO t VALUES (2)"); + Catch( + connection, + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM c WHERE i < 20) " + + "INSERT INTO t SELECT zeroblob(2000) FROM c") + .Should().Be(Full); + Execute(connection, "INSERT INTO t VALUES (3)"); + Execute(connection, "COMMIT"); + + // A single-row write without a statement journal takes the transaction with it. + Execute(connection, "BEGIN"); + Execute(connection, "INSERT INTO t VALUES (4)"); + Catch(connection, "INSERT INTO t VALUES (zeroblob(20000))").Should().Be(Full); + Catch(connection, "COMMIT").Should().Be("ERR:cannot commit - no transaction is active"); + + Scalar(connection, "SELECT group_concat(a) FROM t WHERE typeof(a) = 'integer'").Should().Be("1,2,3"); + Scalar(connection, "PRAGMA page_count").Should().Be("3"); + Scalar(connection, "PRAGMA integrity_check").Should().Be("ok"); + + // Raising the ceiling lets the same write through. + Scalar(connection, "PRAGMA max_page_count = 20").Should().Be("20"); + Execute(connection, "INSERT INTO t VALUES (zeroblob(20000))"); + } + + using (var reopened = EmbeddedDatabase.OpenFile(path)) + using (var connection = reopened.Connect()) + { + Scalar(connection, "PRAGMA integrity_check").Should().Be("ok"); + Scalar(connection, "SELECT count(*) FROM t").Should().Be("4"); + } + + using var sqlite = new SqliteConnection($"Data Source={path};Pooling=False"); + sqlite.Open(); + using var command = sqlite.CreateCommand(); + command.CommandText = "PRAGMA integrity_check"; + command.ExecuteScalar().Should().Be("ok"); + } + + [Test] + public void MvccAutocommitWriteIsHeldToTheCeiling() + { + // Managed MVCC autocommit writes build their b-tree pages as they commit (nothing waits in + // the logical log), so, like every other page allocation, they are held to the ceiling. + var path = Path.Combine(_directory, "mvcc-autocommit.db"); + using (var database = EmbeddedDatabase.OpenFile(path)) + using (var connection = database.Connect()) + { + Execute(connection, "PRAGMA journal_mode=mvcc"); + Execute(connection, "CREATE TABLE t(x TEXT)"); + Execute(connection, "PRAGMA wal_checkpoint(TRUNCATE)"); + Scalar(connection, "PRAGMA max_page_count = 5").Should().Be("5"); + Catch(connection, SixtyLargeRows).Should().Be(Full); + Scalar(connection, "SELECT count(*) FROM t").Should().Be("0"); + Execute(connection, "INSERT INTO t VALUES ('small')"); + Execute(connection, "PRAGMA wal_checkpoint(TRUNCATE)"); + Scalar(connection, "PRAGMA integrity_check").Should().Be("ok"); + } + + using var reopened = EmbeddedDatabase.OpenFile(path); + using var reopenedConnection = reopened.Connect(); + Scalar(reopenedConnection, "SELECT count(*) FROM t").Should().Be("1"); + Scalar(reopenedConnection, "PRAGMA integrity_check").Should().Be("ok"); + } + + [Test] + public void MvccConcurrentCommitIsNotHeldToTheCeiling() + { + // Turso: an MVCC commit only appends to the logical log and never consults + // max_page_count; the ceiling applies when pages are materialized. The managed commit + // publishes the log frame before it materializes pages, so it must not fail afterwards. + var path = Path.Combine(_directory, "mvcc-concurrent.db"); + using (var database = EmbeddedDatabase.OpenFile(path)) + using (var connection = database.Connect()) + { + Execute(connection, "PRAGMA journal_mode=mvcc"); + Execute(connection, "CREATE TABLE t(x TEXT)"); + Execute(connection, "PRAGMA wal_checkpoint(TRUNCATE)"); + Scalar(connection, "PRAGMA max_page_count = 5").Should().Be("5"); + Execute(connection, "BEGIN CONCURRENT"); + Execute(connection, SixtyLargeRows); + Execute(connection, "COMMIT"); + Scalar(connection, "SELECT count(*) FROM t").Should().Be("60"); + Scalar(connection, "PRAGMA max_page_count").Should().Be("5"); + Execute(connection, "PRAGMA wal_checkpoint(TRUNCATE)"); + Scalar(connection, "SELECT count(*) FROM t").Should().Be("60"); + } + + using var reopened = EmbeddedDatabase.OpenFile(path); + using var reopenedConnection = reopened.Connect(); + Scalar(reopenedConnection, "PRAGMA max_page_count").Should().Be("4294967294"); + Scalar(reopenedConnection, "SELECT count(*) FROM t").Should().Be("60"); + Scalar(reopenedConnection, "PRAGMA integrity_check").Should().Be("ok"); + } + + private const string SixtyLargeRows = + "WITH RECURSIVE c(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM c WHERE i < 60) " + + "INSERT INTO t SELECT printf('%.*c', 1800, 'x') FROM c"; + + private static List RunAhtolaMemory(IEnumerable script) + { + using var database = new EmbeddedDatabase(); + using var connection = database.Connect(); + return RunAhtola(connection, script); + } + + private static List RunAhtolaFile(IEnumerable script, string path) + { + using var database = EmbeddedDatabase.OpenFile(path); + using var connection = database.Connect(); + return RunAhtola(connection, script); + } + + private static List RunAhtola(EmbeddedConnection connection, IEnumerable script) + { + var output = new List(); + foreach (var sql in script) + { + try + { + using var statement = connection.Prepare(sql); + while (statement.Step() == StatementStepResult.Row) + { + output.Add(string.Join( + ",", + Enumerable.Range(0, statement.ColumnCount).Select(index => Show(statement.GetValue(index))))); + } + } + catch (EmbeddedSqlException exception) + { + if (exception.Message == "database or disk is full") + exception.SqliteErrorCode.Should().Be(13); + output.Add("ERR:" + exception.Message); + } + } + + return output; + } + + private static List RunSqlite(IEnumerable script, string connectionString) + { + using var connection = new SqliteConnection(connectionString); + connection.Open(); + var output = new List(); + foreach (var sql in script) + { + try + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + output.Add(string.Join( + ",", + Enumerable.Range(0, reader.FieldCount).Select(index => Convert.ToString(reader.GetValue(index))))); + } + } + catch (SqliteException exception) + { + output.Add("ERR:" + exception.Message + .Replace($"SQLite Error {exception.SqliteErrorCode}: '", string.Empty, StringComparison.Ordinal) + .TrimEnd('.', '\'')); + } + } + + return output; + } + + private static string Show(SqlValue value) => value.Kind switch + { + SqlValueKind.Null => string.Empty, + SqlValueKind.Integer => value.AsInteger().ToString(System.Globalization.CultureInfo.InvariantCulture), + SqlValueKind.Text => value.AsText(), + _ => value.ToString() ?? string.Empty, + }; + + private static void Execute(EmbeddedConnection connection, string sql) + { + using var statement = connection.Prepare(sql); + while (statement.Step() == StatementStepResult.Row) + { + } + } + + private static string Scalar(EmbeddedConnection connection, string sql) + { + using var statement = connection.Prepare(sql); + return statement.Step() == StatementStepResult.Row + ? Show(statement.GetValue(0)) + : throw new InvalidOperationException($"{sql} produced no row."); + } + + private static string Catch(EmbeddedConnection connection, string sql) + { + var failure = Assert.Catch(() => Execute(connection, sql))!; + if (failure.Message == "database or disk is full") + failure.SqliteErrorCode.Should().Be(13); + return "ERR:" + failure.Message; + } +} diff --git a/src/Ahtola.Tests/StorageAuditBoundedGapTests.cs b/src/Ahtola.Tests/StorageAuditBoundedGapTests.cs index 1503b838..778de7c6 100644 --- a/src/Ahtola.Tests/StorageAuditBoundedGapTests.cs +++ b/src/Ahtola.Tests/StorageAuditBoundedGapTests.cs @@ -1,5 +1,6 @@ using System.Buffers.Binary; using AwesomeAssertions; +using Ahtola.Core; using Ahtola.Core.Storage; namespace Ahtola.Tests; @@ -497,7 +498,7 @@ public void AllocationPathsEnforceTheDatabaseGrowthCeiling() allocator.MaximumPageCount.Should().Be(6u); allocator.Allocate().PageNumber.Should().Be(5u); allocator.Allocate().PageNumber.Should().Be(6u); - Assert.Throws(() => allocator.Allocate()); + Assert.Catch(() => allocator.Allocate())!.Message.Should().Be("database or disk is full"); var io = new SqliteStagedBtreePageIo( _ => new byte[SqlitePageSize.Minimum], @@ -506,7 +507,7 @@ public void AllocationPathsEnforceTheDatabaseGrowthCeiling() usableSpace: SqlitePageSize.Minimum); io.SetMaximumPageCount(5).Should().Be(5u); io.AllocatePage().Should().Be(5u); - Assert.Throws(() => io.AllocatePage()); + Assert.Catch(() => io.AllocatePage())!.Message.Should().Be("database or disk is full"); io.SetMaximumPageCount(1).Should().Be(5u); } From 82d61addac8a4be8de66f00c46b2f759db213241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 19:07:20 -0400 Subject: [PATCH 14/20] Cost compiled joins with the Turso v0.8.1 join planner and describe the executed join tree Port the per-step costing of optimizer/join.rs join_lhs_and_rhs and access_method.rs (find_best_access_method_for_btree, the temporary-index fallback, try_hash_join_access_method, estimate_hash_join_cost, cost_with_where_work) into JoinOrderEnumerator.EvaluateTursoStep: declared index seeks, INTEGER PRIMARY KEY lookups, the ephemeral index over a full scan, left-deep hash joins that build the last placed table (materialized behind a longer prefix up to MAX_MATERIALIZED_BUILD_ROWS), outer-join Poisson output estimates and the cross-product penalty. Two-table LEFT/FULL base-table joins are planned the same way, and VdbeHashJoinRuntime now keeps unmatched rows for either build side so LEFT/FULL can hash their preserved input. EXPLAIN QUERY PLAN for a compiled join now walks the OpenJoinCursor plan it will execute and renders Turso's EqpDetail rows (SCAN / SEARCH ... USING [COVERING] INDEX (a=? AND b=?) / SEARCH ... USING INTEGER PRIMARY KEY / HASH JOIN / MATERIALIZE hash build input / LEFT-JOIN / USE SORTER FOR ORDER BY). Shapes Turso would not produce for a table with INDEXED BY or NOT INDEXED (temporary index, hash join) keep the previous placeholder description. Unit tests that pinned the old single-row EQP, the automatic-index choice or the old build-side heuristic now assert the plan the Turso cost model selects. Co-Authored-By: Claude Opus 5.5 --- .../JoinOrdering/JoinOrderEnumerator.cs | 406 +++++++++++++++++- .../JoinOrdering/JoinOrderModel.cs | 62 ++- .../EmbeddedDatabase.JoinOrderRewrites.cs | 207 ++++++++- src/Ahtola.Core/EmbeddedDatabase.cs | 240 ++++++++++- .../Execution/VdbeHashJoinRuntime.cs | 53 ++- src/Ahtola.Core/Execution/VdbeProgram.cs | 20 +- .../CompiledJoinVdbeSqlRoutingTests.cs | 11 +- .../managed-sqltest-expected-failures.txt | 6 - .../ExplainQueryPlanFormatJsonTests.cs | 177 +++++--- .../IndexSeekJoinDirectAccessTests.cs | 8 +- src/Ahtola.Tests/IndexSeekJoinTests.cs | 21 +- .../PlannerAccessPathDepthTests.cs | 35 +- src/Ahtola.Tests/PlannerStat1JoinCostTests.cs | 24 +- .../WithoutRowidIndexSeekJoinTests.cs | 18 +- 14 files changed, 1138 insertions(+), 150 deletions(-) diff --git a/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderEnumerator.cs b/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderEnumerator.cs index e20f2125..04d904cc 100644 --- a/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderEnumerator.cs +++ b/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderEnumerator.cs @@ -3,6 +3,14 @@ namespace Ahtola.Core.Compilation.JoinOrdering; +/// The outer-join kind of a step costed by the Turso model. +internal enum TursoOuterJoin +{ + None, + Left, + Full, +} + /// /// Arbitrary-N join-order enumeration for a single plain-INNER segment. /// @@ -45,6 +53,9 @@ internal static class JoinOrderEnumerator private const double CostEpsilon = 1e-9; + /// join.rs:65-67 MAX_MATERIALIZED_BUILD_ROWS. + private const double MaximumMaterializedBuildRows = 200_000.0; + /// /// Upper bound on how many mutually incomparable partial plans one DP state keeps. Reaching /// it is a pathological shape, not a normal one; the cap keeps prepare-time work bounded. @@ -91,16 +102,16 @@ public static JoinOrderPlan EvaluateOrder( var shapes = new JoinStepShape[memberOrder.Length]; var indexAccesses = new JoinIndexAccessChoice?[memberOrder.Length]; var first = segment.Members[memberOrder[0]]; - var cardinality = Math.Max(1.0, first.RowCount); + var (firstCost, cardinality) = EvaluateFirstStep(segment, memberOrder[0]); var cost = HasForcedIndexCandidate(first) ? double.PositiveInfinity - : JoinCostModel.EstimateFullScanCost(first.RowCount, scanCount: 1.0); + : firstCost; var placed = 1UL << memberOrder[0]; for (var step = 1; step < memberOrder.Length; step++) { var member = memberOrder[step]; - var evaluation = EvaluateStep(segment, placed, member, cardinality); + var evaluation = EvaluateStep(segment, placed, member, cardinality, memberOrder[step - 1]); shapes[step] = evaluation.Shape; indexAccesses[step] = evaluation.IndexAccess; cost += evaluation.StepCost; @@ -135,11 +146,12 @@ private static JoinOrderPlan ComputeDynamicProgramming(JoinSegment segment, Join continue; var state = ((1 << member) * count) + member; + var (memberCost, memberCardinality) = EvaluateFirstStep(segment, member); frontiers[state] = [ new JoinOrderDpEntry( - JoinCostModel.EstimateFullScanCost(segment.Members[member].RowCount, scanCount: 1.0), - Math.Max(1.0, segment.Members[member].RowCount), + memberCost, + memberCardinality, [member], [JoinStepShape.NestedLoop], [null]), @@ -175,7 +187,7 @@ private static JoinOrderPlan ComputeDynamicProgramming(JoinSegment segment, Join if ((mask & (1 << next)) != 0) continue; - var evaluation = EvaluateStep(segment, (ulong)mask, next, entry.Cardinality); + var evaluation = EvaluateStep(segment, (ulong)mask, next, entry.Cardinality, entry.Order[^1]); var nextCost = entry.Cost + evaluation.StepCost; if (nextCost >= upperBound + CostEpsilon) continue; @@ -299,7 +311,7 @@ private static JoinOrderPlan ComputeGreedy(JoinSegment segment, JoinOrderPlan id if (HasForcedIndexCandidate(segment.Members[member])) continue; - var candidate = JoinCostModel.EstimateFullScanCost(segment.Members[member].RowCount, scanCount: 1.0); + var candidate = EvaluateFirstStep(segment, member).Cost; if (candidate >= firstCost - CostEpsilon) continue; @@ -314,7 +326,7 @@ private static JoinOrderPlan ComputeGreedy(JoinSegment segment, JoinOrderPlan id shapes[0] = JoinStepShape.NestedLoop; used[firstMember] = true; var placed = 1UL << firstMember; - var cardinality = Math.Max(1.0, segment.Members[firstMember].RowCount); + var cardinality = EvaluateFirstStep(segment, firstMember).Cardinality; var cost = firstCost; for (var step = 1; step < count; step++) @@ -329,7 +341,7 @@ private static JoinOrderPlan ComputeGreedy(JoinSegment segment, JoinOrderPlan id if (used[member]) continue; - var evaluation = EvaluateStep(segment, placed, member, cardinality); + var evaluation = EvaluateStep(segment, placed, member, cardinality, order[step - 1]); // Strict improvement only, so the lowest member index wins every tie. if (bestMember >= 0 && evaluation.StepCost >= bestCost - CostEpsilon) continue; @@ -385,6 +397,34 @@ private static bool IsBetter(double cost, int[] order, double incumbentCost, int return order.Length < incumbentOrder.Length; } + /// + /// The cost and output cardinality of the plan's first (outer-most) member: a full scan. + /// Under the Turso model this is estimate_scan_cost plus the WHERE work of the + /// member's own ready filters, producing the rows those filters keep + /// (join.rs:156-190 rows_after_join, access_method.rs:687-707). + /// + internal static (double Cost, double Cardinality) EvaluateFirstStep(JoinSegment segment, int member) + { + var rows = Math.Max(1.0, segment.Members[member].RowCount); + if (!segment.UseTursoCostModel) + return (JoinCostModel.EstimateFullScanCost(segment.Members[member].RowCount, scanCount: 1.0), rows); + + var memberBit = 1UL << member; + var extraSteps = 0; + var selectivity = 1.0; + foreach (var term in segment.Terms) + { + if (term.TableMask != memberBit) + continue; + extraSteps += term.WhereExtraSteps; + selectivity *= term.TursoLocalSelectivity; + } + + var cost = TursoCostModel.EstimateScanCost(rows, 1.0) + + TursoCostModel.EstimateWhereWork(1.0, rows, consumedSteps: 0, extraSteps); + return (cost, rows * selectivity); + } + /// /// Scores adding onto a partial plan covering /// with cardinality . @@ -393,8 +433,12 @@ private static StepEvaluation EvaluateStep( JoinSegment segment, ulong placedMask, int member, - double leftCardinality) + double leftCardinality, + int lastMember = -1) { + if (segment.UseTursoCostModel) + return EvaluateTursoStep(segment, placedMask, member, leftCardinality, lastMember: lastMember); + var memberBit = 1UL << member; var candidateMask = placedMask | memberBit; var rightRows = Math.Max(1.0, segment.Members[member].RowCount); @@ -675,7 +719,347 @@ private static bool TryGetEqualityMatchRows( return false; } - private readonly record struct StepEvaluation( + /// + /// One left-deep step costed with the ported Turso v0.8.1 join planner + /// (optimizer/join.rs join_lhs_and_rhs, access_method.rs + /// find_best_access_method_for_btree and try_hash_join_access_method), mapped + /// onto the shapes the managed operator executes: + /// + /// a full scan of the member per outer row (estimate_scan_cost with the outer + /// cardinality as the scan count) → ; + /// a declared index seek on an outer-bound equality prefix + /// (estimate_index_cost) → ; + /// Turso's temporary index when the best B-tree access is a full scan + /// (scan + estimate_ephemeral_index_build_cost + per-seek work) → the operator's hash + /// of the right input, ; + /// a hash join building the previous table and probing this one + /// (estimate_hash_join_cost, only while the outer side is that single table, the + /// left-deep build Turso allows without materialization) → + /// . + /// + /// Every alternative carries the WHERE work of the terms that become ready at this step + /// (cost_with_where_work), and a step with no join term pays Turso's cross-product + /// penalty. + /// + internal static StepEvaluation EvaluateTursoStep( + JoinSegment segment, + ulong placedMask, + int member, + double inputCardinality, + TursoOuterJoin outerJoin = TursoOuterJoin.None, + int lastMember = -1) + { + var memberBit = 1UL << member; + var candidateMask = placedMask | memberBit; + var target = segment.Members[member]; + var rows = Math.Max(1.0, target.RowCount); + + // Terms that become ready at this step, and the constraints they put on the member. + var ready = new List(); + var joinEqualities = new List<(int Term, int Ordinal, double Selectivity, bool Hashable)>(); + var hasJoinTerm = false; + var memberLocalSelectivity = 1.0; + for (var index = 0; index < segment.Terms.Count; index++) + { + var term = segment.Terms[index]; + if ((term.TableMask & memberBit) == 0 || (term.TableMask & ~candidateMask) != 0) + continue; + ready.Add(index); + if ((term.TableMask & placedMask) != 0) + hasJoinTerm = true; + if (term.TableMask == memberBit) + memberLocalSelectivity *= term.TursoLocalSelectivity; + + if (!term.IsEquality || term.CostOnly) + continue; + if (term.EqualityRightMask == memberBit && (term.EqualityLeftMask & ~placedMask) == 0 && term.EqualityLeftMask != 0) + { + joinEqualities.Add((index, term.EqualityRightColumnOrdinal, Sanitize(term.TursoRightSelectivity), term.EqualityCollation is not null)); + } + else if (term.EqualityLeftMask == memberBit && (term.EqualityRightMask & ~placedMask) == 0 && term.EqualityRightMask != 0) + { + joinEqualities.Add((index, term.EqualityLeftColumnOrdinal, Sanitize(term.TursoLeftSelectivity), term.EqualityCollation is not null)); + } + } + + // The selectivity of the ready constraints an access path does not consume + // (constraint_output_multipliers, join.rs:94-152). + double RemainingSelectivity(IReadOnlyCollection consumed) + { + var selectivity = memberLocalSelectivity; + foreach (var equality in joinEqualities) + { + if (!consumed.Contains(equality.Term)) + selectivity *= equality.Selectivity; + } + + return selectivity; + } + + double WhereWork(IReadOnlyCollection consumed, double rowsPerOuterRow) + { + var consumedSteps = 0; + var remainingSteps = 0; + foreach (var index in ready) + { + if (consumed.Contains(index)) + consumedSteps += segment.Terms[index].WhereExtraSteps; + else + remainingSteps += segment.Terms[index].WhereExtraSteps; + } + + return TursoCostModel.EstimateWhereWork(inputCardinality, rowsPerOuterRow, consumedSteps, remainingSteps); + } + + var crossPenalty = hasJoinTerm ? 0.0 : inputCardinality * rows * memberLocalSelectivity; + + // An outer join emits every outer row at least once: the Poisson estimate of + // rows_after_join (join.rs:192-248) keeps a row whose matches all miss. + double OuterRows(double rowsPerOuterRow, double remainingSelectivity) + => outerJoin == TursoOuterJoin.None + ? inputCardinality * rowsPerOuterRow * remainingSelectivity + : inputCardinality * (rowsPerOuterRow * remainingSelectivity + Math.Exp(-(rowsPerOuterRow * remainingSelectivity))); + + // choose_best_btree_candidate: the plain scan, then each declared index seek. + IReadOnlyCollection noTerms = []; + var bestShape = JoinStepShape.NestedLoop; + JoinIndexAccessChoice? bestIndex = null; + var bestBtreeCost = TursoCostModel.EstimateScanCost(rows, inputCardinality); + var bestRowsPerOuter = rows; + IReadOnlyCollection bestConsumed = noTerms; + var forcedOnly = HasForcedIndexCandidate(target); + if (forcedOnly) + bestBtreeCost = double.PositiveInfinity; + + var candidates = target.IndexCandidates ?? []; + for (var candidateIndex = 0; candidateIndex < candidates.Count; candidateIndex++) + { + var candidate = candidates[candidateIndex]; + if (candidate.Automatic) + continue; + + var termIndices = new List(candidate.Columns.Count); + var selectivityProduct = 1.0; + foreach (var column in candidate.Columns) + { + var bound = -1; + foreach (var equality in joinEqualities) + { + if (!termIndices.Contains(equality.Term) + && CanBindIndexColumn(segment.Terms[equality.Term], placedMask, memberBit, column)) + { + bound = equality.Term; + selectivityProduct *= equality.Selectivity; + break; + } + } + + if (bound < 0) + break; + termIndices.Add(bound); + } + + if (termIndices.Count == 0) + continue; + + var unique = candidate.Unique && termIndices.Count >= candidate.Columns.Count; + var rowsPerSeek = unique + ? 1.0 + : termIndices.Count <= candidate.RowsPerPrefix.Count + ? Math.Max(candidate.RowsPerPrefix[termIndices.Count - 1], 1.0) + : Math.Max(selectivityProduct * rows, 1.0); + var info = new TursoIndexInfo( + candidate.Unique, + candidate.Columns.Count, + candidate.Covering, + TursoCostModel.IndexLeafRowsPerPage(candidate.Columns.Count, candidate.TableColumnCount, candidate.HasRowIdAlias)); + var cost = TursoCostModel.EstimateIndexCost( + rows, + TursoCostModel.EstimateBtreeDepth(rows, JoinCostParams.RowsPerTablePage), + info, + inputCardinality, + rowsPerSeek); + if (cost < bestBtreeCost || candidate.Forced && bestShape != JoinStepShape.IndexSeekRight) + { + bestBtreeCost = cost; + bestShape = JoinStepShape.IndexSeekRight; + bestIndex = new JoinIndexAccessChoice(candidateIndex, [.. termIndices], rowsPerSeek); + bestRowsPerOuter = rowsPerSeek; + bestConsumed = termIndices; + } + } + + // The rowid candidate: an outer-bound equality on the INTEGER PRIMARY KEY is a unique + // point lookup (estimate_cost_for_scan_or_seek with the rowid IndexInfo). The operator + // serves it by hashing the right input on that key once, i.e. a rowid-keyed lookup. + var rowidSeek = false; + if (!forcedOnly && target.RowidAliasOrdinal >= 0) + { + foreach (var equality in joinEqualities) + { + if (equality.Ordinal != target.RowidAliasOrdinal || !equality.Hashable) + continue; + + var cost = TursoCostModel.EstimateIndexCost( + rows, + TursoCostModel.EstimateBtreeDepth(rows, JoinCostParams.RowsPerTablePage), + new TursoIndexInfo(Unique: true, ColumnCount: 1, Covering: true, JoinCostParams.RowsPerTablePage), + inputCardinality, + 1.0); + if (cost < bestBtreeCost) + { + bestBtreeCost = cost; + bestShape = JoinStepShape.HashBuildRight; + bestIndex = null; + bestRowsPerOuter = 1.0; + bestConsumed = [equality.Term]; + rowidSeek = true; + } + + break; + } + } + + if (double.IsPositiveInfinity(bestBtreeCost)) + { + return new StepEvaluation(JoinStepShape.NestedLoop, double.PositiveInfinity, inputCardinality * rows, null); + } + + var bestCost = bestBtreeCost + WhereWork(bestConsumed, bestRowsPerOuter) + crossPenalty; + var bestOutput = OuterRows(bestRowsPerOuter, RemainingSelectivity(bestConsumed)); + + // The temporary index is only considered over a plain full scan and never for a FULL + // join, which has no nested-loop form (access_method.rs:903-987). + var hashableKeys = joinEqualities.Where(static equality => equality.Hashable).ToArray(); + if (bestShape == JoinStepShape.NestedLoop && !forcedOnly && hashableKeys.Length != 0 + && outerJoin != TursoOuterJoin.Full) + { + var consumed = hashableKeys.Select(static equality => equality.Term).ToArray(); + var rowsPerSeek = Math.Max( + hashableKeys.Aggregate(1.0, static (product, equality) => product * equality.Selectivity) * rows, + 1.0); + var cost = TursoCostModel.EstimateScanCost(rows, 1.0) + + TursoCostModel.EstimateEphemeralIndexBuildCost(rows) + + inputCardinality * JoinCostParams.CpuCostPerSeek + + inputCardinality * rowsPerSeek * JoinCostParams.CpuCostPerRow + + WhereWork(consumed, rowsPerSeek) + + crossPenalty; + if (cost < bestCost) + { + bestCost = cost; + bestShape = JoinStepShape.HashBuildRight; + bestIndex = null; + bestOutput = OuterRows(rowsPerSeek, RemainingSelectivity(consumed)); + } + } + + // The left-deep hash join builds the last placed table and probes this one + // (join.rs:586-1000). With one placed table the build reads it directly; behind a longer + // prefix Turso materializes the build input from the prefix's rows (the operator hashes + // that joined prefix), which is only allowed up to MAX_MATERIALIZED_BUILD_ROWS. An index + // that can seek the probe key keeps the index path (probe_index_can_seek_join_key), and + // so does a selective declared-index or rowid probe seek. + var placedCount = BitOperations.PopCount(placedMask); + var buildMember = placedCount == 1 ? BitOperations.TrailingZeroCount(placedMask) : lastMember; + var materialize = placedCount > 1; + var buildKeys = buildMember < 0 + ? [] + : hashableKeys + .Where(key => + { + var term = segment.Terms[key.Term]; + var otherMask = term.EqualityLeftMask == memberBit ? term.EqualityRightMask : term.EqualityLeftMask; + return otherMask == 1UL << buildMember; + }) + .ToArray(); + if (buildMember >= 0 + && buildKeys.Length != 0 + && bestShape != JoinStepShape.IndexSeekRight + && !rowidSeek + && !forcedOnly + && !buildKeys.Any(key => target.IndexLeadingColumnOrdinals.Contains(key.Ordinal)) + && (!materialize || inputCardinality <= MaximumMaterializedBuildRows || outerJoin == TursoOuterJoin.Full)) + { + var build = segment.Members[buildMember]; + var buildRows = Math.Max(1.0, build.RowCount); + var buildBit = 1UL << buildMember; + var buildSelectivity = 1.0; + foreach (var term in segment.Terms) + { + if (term.TableMask == buildBit) + buildSelectivity *= term.TursoLocalSelectivity; + } + + // A plain build column that an index could seek keeps the index path unless the + // probe side has no constant filter (can_replace_build_index_with_hash). + var hashCanReplaceBuildIndex = segment.Terms.All(term => term.TableMask != memberBit); + var buildKeyIndexed = false; + var buildKeyUnique = false; + foreach (var key in buildKeys) + { + var term = segment.Terms[key.Term]; + var buildOrdinal = term.EqualityLeftMask == buildBit + ? term.EqualityLeftColumnOrdinal + : term.EqualityRightColumnOrdinal; + if (buildOrdinal == build.RowidAliasOrdinal && buildOrdinal >= 0) + { + buildKeyIndexed = true; + buildKeyUnique = true; + } + else if (build.IndexedColumnOrdinals.Contains(buildOrdinal)) + { + buildKeyIndexed = true; + } + } + + if (hashCanReplaceBuildIndex || !buildKeyIndexed) + { + var consumed = buildKeys.Select(static equality => equality.Term).ToArray(); + // should_materialize: the build is the prefix's estimated rows. + var buildCardinality = materialize + ? inputCardinality + : buildRows * Math.Clamp(buildSelectivity, 0.0, 1.0); + var probeSelectivity = buildKeys.Aggregate(1.0, static (product, equality) => product * equality.Selectivity); + var distinctBuildKeys = materialize ? Math.Min(inputCardinality, buildRows) : buildRows; + var rowsPerBuildRow = buildKeyUnique + ? rows / Math.Max(distinctBuildKeys, 1.0) + : rows * probeSelectivity; + // HashJoinType estimates (access_method.rs:1413-1420): an outer join emits each + // build row at least once, a FULL join also every unmatched probe row. + var rowsPerOuterRow = outerJoin switch + { + TursoOuterJoin.Left => Math.Max(rowsPerBuildRow, 1.0), + TursoOuterJoin.Full => Math.Max(Math.Max(rowsPerBuildRow, 1.0), rows / Math.Max(buildCardinality, 1.0)), + _ => rowsPerBuildRow, + }; + var cost = TursoCostModel.EstimateHashJoinCost( + buildCardinality, + rows, + buildCardinality * rowsPerOuterRow, + keepsUnmatchedBuildRows: outerJoin != TursoOuterJoin.None, + probeMultiplier: 1.0) + + (materialize ? buildCardinality * JoinCostParams.CpuCostPerRow : 0.0) + + WhereWork(consumed, rowsPerOuterRow) + + crossPenalty; + // FULL OUTER requires the hash join for its unmatched-probe scan. + if (cost < bestCost || outerJoin == TursoOuterJoin.Full) + { + bestCost = cost; + bestShape = JoinStepShape.HashBuildLeft; + bestIndex = null; + bestOutput = outerJoin == TursoOuterJoin.None + ? inputCardinality * rowsPerBuildRow * RemainingSelectivity(consumed) + : inputCardinality * rowsPerOuterRow; + } + } + } + return new StepEvaluation(bestShape, bestCost, bestOutput, bestIndex); + + static double Sanitize(double selectivity) + => double.IsNaN(selectivity) ? JoinCostParams.SelectivityEqualityUnindexed : selectivity; + } + + internal readonly record struct StepEvaluation( JoinStepShape Shape, double StepCost, double OutputCardinality, diff --git a/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderModel.cs b/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderModel.cs index 0f450e2d..afd45b27 100644 --- a/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderModel.cs +++ b/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderModel.cs @@ -17,7 +17,27 @@ internal sealed record JoinSegmentMember( int OriginalIndex, double RowCount, int ColumnWidth, - IReadOnlyList? IndexCandidates = null); + IReadOnlyList? IndexCandidates = null) +{ + /// + /// Table ordinal of the member's INTEGER PRIMARY KEY (rowid alias) column, or -1. Only the + /// Turso cost model reads it (a rowid-alias hash build key keeps the index path, + /// access_method.rs:1360-1389). + /// + public int RowidAliasOrdinal { get; init; } = -1; + + /// + /// Table ordinals that any declared index of the member contains. Only the Turso cost + /// model reads it (an indexed plain build column keeps the index path). + /// + public IReadOnlyCollection IndexedColumnOrdinals { get; init; } = []; + + /// + /// Table ordinals that lead a declared index of the member (Turso's + /// probe_index_can_seek_join_key, access_method.rs:1563-1592). + /// + public IReadOnlyCollection IndexLeadingColumnOrdinals { get; init; } = []; +} /// /// One persisted or automatic index usable as an outer-bound join access. Forced is true @@ -113,7 +133,35 @@ internal sealed record JoinPredicateTerm( bool EqualityRightConvertsTextToNumeric = false, bool EqualityRightConvertsNumericToText = false, string? EqualityCollation = null, - string? EqualitySeekCollation = null); + string? EqualitySeekCollation = null) +{ + /// + /// Turso estimate_selectivity of this equality seen as a constraint on the left + /// operand's member (constraints.rs:367-437), or NaN when not computed. + /// + public double TursoLeftSelectivity { get; init; } = double.NaN; + + /// The same figure for the right operand's member. + public double TursoRightSelectivity { get; init; } = double.NaN; + + /// + /// Turso selectivity of a single-member constraint whose other side reads no table + /// (a local filter), or 1 when the term is not such a constraint. + /// + public double TursoLocalSelectivity { get; init; } = 1.0; + + /// + /// where_expr_steps(term) - 1 (join.rs:2338-2345): the WHERE work the term adds to + /// the loop where it becomes ready, beyond the one simple condition a row already pays for. + /// + public int WhereExtraSteps { get; init; } + + /// + /// True for a WHERE conjunct that only feeds the cost model: it is never attached to a + /// synthesized join node, because the surviving WHERE evaluates it. + /// + public bool CostOnly { get; init; } +} /// /// Exact index and equality terms selected for one IndexSeekRight step. @@ -129,7 +177,15 @@ internal sealed record JoinIndexAccessChoice( /// internal sealed record JoinSegment( IReadOnlyList Members, - IReadOnlyList Terms); + IReadOnlyList Terms) +{ + /// + /// When true, steps are costed with the ported Turso v0.8.1 join planner + /// ( EvaluateTursoStep) instead of the narrowed + /// executor-shape model. + /// + public bool UseTursoCostModel { get; init; } +} /// /// One enumerated left-deep plan: the member order, the physical shape chosen for each step, and diff --git a/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs b/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs index 3f00d99d..738ab23d 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs @@ -222,10 +222,10 @@ private bool TryRewriteJoinOrderNode( map[left.SlotMap.Length + index] = left.SlotMap.Length + right.SlotMap[index]; var changed = left.Changed || right.Changed; - result = new JoinOrderRewrittenSource( - changed ? join with { Left = left.Source, Right = right.Source } : join, - map, - changed); + var rebuilt = changed ? join with { Left = left.Source, Right = right.Source } : join; + if (TryPlanTursoOuterJoinNode(rebuilt, state, isRoot)) + changed = true; + result = new JoinOrderRewrittenSource(rebuilt, map, changed); return true; } @@ -277,24 +277,46 @@ private bool TryRewriteJoinOrderSegment( return false; } - terms.Add(term); + terms.Add(WithTursoTermEstimates(term, conjunct, sources, infos, state.Context)); placements.Add(new JoinOrderTermPlacement(conjunct, term.TableMask)); } var pushedWhereTerms = 0; + List? costOnlyConjuncts = null; if (isRoot && state.Where is not null) { foreach (var conjunct in SplitJoinOrderConjunction(state.Where)) { if (!TryCreatePushableJoinOrderWhereTerm(conjunct, infos, state.Context, out var term)) + { + (costOnlyConjuncts ??= []).Add(conjunct); continue; + } - terms.Add(term); + terms.Add(WithTursoTermEstimates(term, conjunct, sources, infos, state.Context)); placements.Add(new JoinOrderTermPlacement(conjunct, term.TableMask)); pushedWhereTerms++; } } + // The rest of the WHERE clause is never attached to a synthesized node (the surviving + // WHERE evaluates it), but Turso charges its evaluation to the loop where it becomes + // ready. Such terms follow every placement, so placement indices stay aligned. + if (costOnlyConjuncts is not null) + { + foreach (var conjunct in costOnlyConjuncts) + { + if (!TryResolveJoinOrderMask(conjunct, infos, out var mask) || mask == 0) + continue; + terms.Add(WithTursoTermEstimates( + new JoinPredicateTerm(mask, IsEquality: false, 0, 0, 0.0, 0.0, Selectivity: 1.0) { CostOnly = true }, + conjunct, + sources, + infos, + state.Context)); + } + } + var members = new JoinSegmentMember[infos.Length]; for (var index = 0; index < infos.Length; index++) { @@ -308,15 +330,33 @@ private bool TryRewriteJoinOrderSegment( indexCandidates = [.. indexCandidates, automatic]; } + var memberTable = infos[index].Table; members[index] = new JoinSegmentMember( index, infos[index].RowCount, infos[index].Width, - indexCandidates); + indexCandidates) + { + RowidAliasOrdinal = memberTable?.RowidAliasColumnIndex ?? -1, + IndexedColumnOrdinals = memberTable is null + ? [] + : memberTable.Indexes + .Where(static index => !index.IsMethodIndex) + .SelectMany(static index => index.Columns) + .Where(static column => !column.IsExpression) + .Select(static column => column.ColumnIndex) + .ToHashSet(), + IndexLeadingColumnOrdinals = memberTable is null + ? [] + : memberTable.Indexes + .Where(static index => !index.IsMethodIndex && index.Columns.Count > 0 && !index.Columns[0].IsExpression) + .Select(static index => index.Columns[0].ColumnIndex) + .ToHashSet(), + }; } Interlocked.Increment(ref _joinOrderSegmentsConsidered); - var plan = JoinOrderEnumerator.Compute(new JoinSegment(members, terms)); + var plan = JoinOrderEnumerator.Compute(new JoinSegment(members, terms) { UseTursoCostModel = true }); if (plan is null) { Interlocked.Increment(ref _joinOrderDeclines); @@ -347,6 +387,110 @@ private bool TryRewriteJoinOrderSegment( return true; } + /// + /// Chooses the right input's access for a two-table LEFT or FULL join of base tables with + /// the ported Turso step model (): a hash + /// join building the preserved left table and probing the right (Turso + /// HashJoinType::LeftOuter/FullOuter), a declared index seek, or the operator's + /// hash of the right input (Turso's temporary index). The decision is recorded for the + /// builder; true when one was recorded. + /// + private bool TryPlanTursoOuterJoinNode(JoinTableSource join, JoinOrderRewriteState state, bool isRoot) + { + if (join.Kind is not (JoinKind.Left or JoinKind.Full) + || join.Left is not NamedTableSource + || join.Right is not NamedTableSource + || join.Condition is null + || join.UsingColumns is not null + || join.Natural) + { + return false; + } + + var conjuncts = SplitJoinOrderConjunction(join.Condition).ToList(); + var sources = new List { join.Left, join.Right }; + var infos = new JoinOrderMemberInfo[2]; + for (var index = 0; index < 2; index++) + { + if (!TryDescribeJoinOrderMember(sources[index], conjuncts, state.Context, out var info)) + return false; + infos[index] = info; + } + + var terms = new List(); + var placements = new List(); + foreach (var conjunct in conjuncts) + { + if (!TryCreateJoinOrderTerm(conjunct, infos, state.Context, out var term)) + return false; + terms.Add(WithTursoTermEstimates(term, conjunct, sources, infos, state.Context)); + placements.Add(new JoinOrderTermPlacement(conjunct, term.TableMask)); + } + + // WHERE filters on the preserved table narrow the rows that reach the right input. + if (isRoot && state.Where is not null) + { + foreach (var conjunct in SplitJoinOrderConjunction(state.Where)) + { + if (TryResolveJoinOrderMask(conjunct, infos, out var mask) && mask == 1UL) + { + terms.Add(WithTursoTermEstimates( + new JoinPredicateTerm(mask, IsEquality: false, 0, 0, 0.0, 0.0, Selectivity: 1.0) { CostOnly = true }, + conjunct, + sources, + infos, + state.Context)); + } + } + } + + var members = new JoinSegmentMember[2]; + for (var index = 0; index < 2; index++) + { + var memberTable = infos[index].Table; + members[index] = new JoinSegmentMember(index, infos[index].RowCount, infos[index].Width, infos[index].IndexCandidates) + { + RowidAliasOrdinal = memberTable?.RowidAliasColumnIndex ?? -1, + IndexedColumnOrdinals = memberTable is null + ? [] + : memberTable.Indexes + .Where(static index => !index.IsMethodIndex) + .SelectMany(static index => index.Columns) + .Where(static column => !column.IsExpression) + .Select(static column => column.ColumnIndex) + .ToHashSet(), + IndexLeadingColumnOrdinals = memberTable is null + ? [] + : memberTable.Indexes + .Where(static index => !index.IsMethodIndex && index.Columns.Count > 0 && !index.Columns[0].IsExpression) + .Select(static index => index.Columns[0].ColumnIndex) + .ToHashSet(), + }; + } + + var segment = new JoinSegment(members, terms) { UseTursoCostModel = true }; + var (_, outerRows) = JoinOrderEnumerator.EvaluateFirstStep(segment, 0); + var step = JoinOrderEnumerator.EvaluateTursoStep( + segment, + placedMask: 1UL, + member: 1, + outerRows, + join.Kind == JoinKind.Full ? TursoOuterJoin.Full : TursoOuterJoin.Left); + switch (step.Shape) + { + case JoinStepShape.HashBuildLeft: + state.HashBuildRight[join] = false; + return true; + case JoinStepShape.IndexSeekRight when join.Kind == JoinKind.Left && step.IndexAccess is { } access: + state.IndexSeeks[join] = new CompiledJoinIndexSelection( + members[1].IndexCandidates![access.CandidateIndex], + access.EqualityTermIndices.Select(index => placements[index].Expression).ToArray()); + return true; + default: + return false; + } + } + /// /// Collects the maximal plain-INNER run rooted at in left-to-right /// FROM order together with every ON conjunct it contributes. A non-qualifying child becomes @@ -1625,6 +1769,53 @@ private sealed class JoinOrderMemberInfo( public IReadOnlyList IndexCandidates { get; } = indexCandidates; } + /// + /// Adds the Turso cost-model figures to one segment term: the estimate_selectivity + /// of an equality from each operand's member, the selectivity of a single-member filter, + /// and the term's where_expr_steps - 1 WHERE work. + /// + private JoinPredicateTerm WithTursoTermEstimates( + JoinPredicateTerm term, + Expression conjunct, + List sources, + JoinOrderMemberInfo[] infos, + QueryContext context) + { + var leftSelectivity = double.NaN; + var rightSelectivity = double.NaN; + if (term.IsEquality) + { + leftSelectivity = MemberEqualitySelectivity(term.EqualityLeftMask, term.EqualityLeftColumnOrdinal); + rightSelectivity = MemberEqualitySelectivity(term.EqualityRightMask, term.EqualityRightColumnOrdinal); + } + + var localSelectivity = 1.0; + if (System.Numerics.BitOperations.PopCount(term.TableMask) == 1) + { + var member = System.Numerics.BitOperations.TrailingZeroCount(term.TableMask); + if (sources[member] is NamedTableSource named && infos[member].Table is { } table) + localSelectivity = EstimateTursoLocalSelectivity(conjunct, named, table, context); + } + + return term with + { + TursoLeftSelectivity = leftSelectivity, + TursoRightSelectivity = rightSelectivity, + TursoLocalSelectivity = localSelectivity, + WhereExtraSteps = CountTursoWhereSteps(conjunct) - 1, + }; + + double MemberEqualitySelectivity(ulong mask, int ordinal) + { + if (mask == 0 || ordinal < 0) + return double.NaN; + var member = System.Numerics.BitOperations.TrailingZeroCount(mask); + return infos[member].Table is { } table && ordinal < table.Columns.Length + ? EstimateTursoColumnSelectivity(table, ordinal, TursoConstraintOperator.Equal, context) + : double.NaN; + } + } + private readonly record struct JoinOrderTermPlacement(Expression Expression, ulong Mask); private sealed record CompiledJoinIndexSelection( diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index 42531b6a..fe7f51e1 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -22508,7 +22508,7 @@ SourceRow CombinedRow(VdbeJoinRow row) => CreateCompiledJoinSourceRow( if (indexSelection is not null) { - if (kind is not (VdbeJoinKind.Inner or VdbeJoinKind.Semi or VdbeJoinKind.Anti) + if (kind is not (VdbeJoinKind.Inner or VdbeJoinKind.Left or VdbeJoinKind.Semi or VdbeJoinKind.Anti) || !TryCreateCompiledJoinIndexScanPlan( join, indexSelection, @@ -22544,7 +22544,17 @@ SourceRow CombinedRow(VdbeJoinRow row) => CreateCompiledJoinSourceRow( // INNER equijoin: hash-build the smaller estimated side (default still right). // OUTER joins keep hash-build-right so unmatched-side semantics stay correct. var hashBuildRight = true; - if (kind is VdbeJoinKind.Inner && equiProbe is not null) + if (kind is VdbeJoinKind.Left or VdbeJoinKind.Full + && equiProbe is not null + && hashBuildRightOverrides is not null + && hashBuildRightOverrides.TryGetValue(join, out var outerChoice)) + { + // The cost-based stage chose to hash the preserved left input and probe the right + // (Turso's LeftOuter/FullOuter hash join); VdbeHashJoinRuntime emits the unmatched + // build rows after the probe scan. + hashBuildRight = outerChoice; + } + else if (kind is VdbeJoinKind.Inner && equiProbe is not null) { // A node the cost-based join-order stage synthesized carries its own build-side // decision, scored against the executable shapes in JoinCostModel.EstimateStepCost. @@ -23478,6 +23488,7 @@ private static bool IndexKeyPrefixMatches(SqliteIndexRecordComparer comparer, Sq var leftColumns = GetOutputColumns(join.Left, context); var rightColumns = GetOutputColumns(join.Right, context); var keys = new List(); + List<(int Ordinal, string Name)>? rightKeyColumns = []; var pending = new Stack(); pending.Push(join.Condition); while (pending.Count > 0) @@ -23500,6 +23511,7 @@ private static bool IndexKeyPrefixMatches(SqliteIndexRecordComparer comparer, Sq key.RightConvertsTextToNumeric, key.RightConvertsNumericToText, key.Collation)); + rightKeyColumns?.Add((key.RightColumn.Index, key.RightColumn.Name)); continue; } @@ -23539,6 +23551,7 @@ private static bool IndexKeyPrefixMatches(SqliteIndexRecordComparer comparer, Sq RightConvertsTextToNumeric: false, RightConvertsNumericToText: false, Collation: "BINARY")); + rightKeyColumns = null; } if (keys.Count == 0) @@ -23567,7 +23580,16 @@ private static bool IndexKeyPrefixMatches(SqliteIndexRecordComparer comparer, Sq return new VdbeJoinEquiProbe( left => BuildKey(left, leftSide: true), - right => BuildKey(right, leftSide: false)); + right => BuildKey(right, leftSide: false)) + { + // Turso orders a temporary index's key columns by table position. + RightKeyColumns = rightKeyColumns?.OrderBy(static column => column.Ordinal).Select(static column => column.Name).ToArray(), + RightKeyIsRowid = rightKeyColumns is [var onlyKey] + && join.Right is NamedTableSource rightNamed + && context.Tables.TryGetValue(rightNamed.Name, out var rightTable) + && rightTable.RowidAliasColumnIndex >= 0 + && rightTable.RowidAliasColumnIndex == onlyKey.Ordinal, + }; static bool ExpressionBelongsToSource( Expression expression, @@ -29696,6 +29718,20 @@ private ExecutionResult ExecuteExplainQueryPlanText( compiledJoinCandidate = compiledJoinProgram.Program.Instructions .OfType() .FirstOrDefault()?.Plan.Root; + if (compiledJoinCandidate is not null + && TryDescribeCompiledJoinPlan( + compiledJoinCandidate, + compiledJoinSelect, + compiledJoinProgram.Program, + out var describedJoin)) + { + ops = describedJoin.Select(static node => (EqpJsonOp?)node.Op).ToArray(); + return new ExecutionResult( + ExplainQueryPlanColumns(), + describedJoin.Select((node, index) => PlanRow(index + 1, node.Parent + 1, node.Detail)).ToArray(), + 0); + } + var searches = GetCompiledJoinIndexSearchDescriptions(compiledJoinProgram.Program); if (searches.Count > 0) { @@ -29889,6 +29925,204 @@ void AddScan(VdbeJoinScanPlan scan, string? marker) return nodes; } + /// + /// Describes a compiled OpenJoinCursor plan tree step by step in its left-deep + /// execution order, formatted like Turso's EqpDetail (core/translate/eqp.rs Display): + /// the outer leaf as SCAN; an index seek as SEARCH id USING [COVERING ]INDEX name + /// (a=? AND b=?); a right input the operator hashes once and probes per outer row as + /// the automatic index it is (SEARCH id USING COVERING INDEX ephemeral_table_tN (…), + /// Turso's temporary-index access, N being the table's FROM position); a left input it + /// hashes before streaming the right as HASH JOIN right followed by the build's + /// SCAN, or behind a longer prefix as MATERIALIZE hash build input over that + /// prefix; a keyless right input as a nested SCAN. LEFT and FULL joins carry + /// LEFT-JOIN. Any node or shape outside that set (derived rows, RIGHT joins, a FULL + /// join that is not a hash join, a hash or automatic index Turso would not build for a table + /// with an index directive, grouping or DISTINCT) returns so the + /// caller keeps its narrower description rather than guess. + /// + private static bool TryDescribeCompiledJoinPlan( + VdbeJoinPlanNode root, + SelectStatement select, + VdbeProgram program, + out List<(string Detail, EqpJsonOp Op, int Parent)> nodes) + { + nodes = []; + if (select.GroupBy.Count != 0 || select.Distinct || select.Having is not null + || !TryGetNamedJoinLeaves(select.Source, out var leaves)) + { + return false; + } + + var described = new List<(string Detail, EqpJsonOp Op, int Parent)>(); + if (!Describe(root, parent: -1)) + return false; + + if (select.OrderBy.Count != 0) + { + if (!program.Instructions.OfType().Any()) + return false; + described.Add(("USE SORTER FOR ORDER BY", new EqpJsonOrderByOp(), -1)); + } + + nodes = described; + return true; + + bool Describe(VdbeJoinPlanNode node, int parent) + { + if (node is VdbeJoinScanPlan outer) + { + described.Add(( + $"SCAN {outer.TableName}" + (outer.Alias is null ? string.Empty : $" AS {outer.Alias}"), + new EqpJsonScanOp(outer.TableName, outer.Alias, IndexName: null, Covering: false), + parent)); + return true; + } + + if (node is not VdbeJoinOperatorPlan + { + Kind: VdbeJoinKind.Inner or VdbeJoinKind.Left or VdbeJoinKind.Full, + } join) + { + return false; + } + + var marker = GetCompiledJoinMarker(join.Kind); + var suffix = join.Kind is VdbeJoinKind.Left or VdbeJoinKind.Full ? " LEFT-JOIN" : string.Empty; + // A FULL join is only modelled as the hash join that builds its left input. + if (join.Kind == VdbeJoinKind.Full && (join.EquiProbe is null || join.HashBuildRight)) + return false; + if (join.Right is IVdbeJoinSeekPlan && join.Right.SearchMetadata is { } seek) + { + if (!Describe(join.Left, parent)) + return false; + var identifier = seek.Alias ?? seek.TableName; + var indexName = seek.Ephemeral + ? EphemeralIndexName(seek.TableName, seek.Alias) + : seek.IndexName; + if (indexName is null) + return false; + var covering = seek.Ephemeral || seek.Covering; + described.Add(( + $"SEARCH {identifier} USING {(covering ? "COVERING " : string.Empty)}INDEX {indexName} ({string.Join(" AND ", seek.Constraints)}){suffix}", + new EqpJsonSearchOp(seek.TableName, seek.Alias, indexName, covering, seek.Constraints, marker, seek.Ephemeral), + parent)); + return true; + } + + if (join.Right is not VdbeJoinScanPlan right) + return false; + + var rightWithAlias = right.TableName + (right.Alias is null ? string.Empty : $" AS {right.Alias}"); + if (join.EquiProbe is null) + { + if (!Describe(join.Left, parent)) + return false; + described.Add(($"SCAN {rightWithAlias}{suffix}", new EqpJsonScanOp(right.TableName, right.Alias, IndexName: null, Covering: false, marker), parent)); + return true; + } + + if (join.HashBuildRight) + { + if (!Describe(join.Left, parent)) + return false; + + // Hashing the right input once on its INTEGER PRIMARY KEY is a rowid lookup. + if (join.EquiProbe.RightKeyIsRowid) + { + described.Add(( + $"SEARCH {right.Alias ?? right.TableName} USING INTEGER PRIMARY KEY (rowid=?){suffix}", + new EqpJsonSearchOp(right.TableName, right.Alias, IndexName: null, Covering: false, ["rowid=?"], marker, IsIntegerPrimaryKey: true), + parent)); + return true; + } + + // Otherwise the right rows hashed once on the join key and probed per outer row + // are an automatic index over the right input. Turso never builds one for a table + // carrying INDEXED BY / NOT INDEXED (access_method.rs:916), so that shape has no + // Turso description. + if (HasIndexDirective(right.TableName, right.Alias) + || join.EquiProbe.RightKeyColumns is not { Count: > 0 } keyColumns + || EphemeralIndexName(right.TableName, right.Alias) is not { } indexName) + { + return false; + } + + var constraints = keyColumns.Select(static column => $"{column}=?").ToArray(); + described.Add(( + $"SEARCH {right.Alias ?? right.TableName} USING COVERING INDEX {indexName} ({string.Join(" AND ", constraints)}){suffix}", + new EqpJsonSearchOp(right.TableName, right.Alias, indexName, Covering: true, constraints, marker, Ephemeral: true), + parent)); + return true; + } + + // Hash-build-left: the left input is hashed once and the right streams as the probe + // (Turso's left-deep hash join builds the previous table and probes the new one). + // Turso rejects a hash join whose build or probe table carries INDEXED BY / NOT + // INDEXED (access_method.rs:1474-1479 has_indexed_by_directives). + if (HasIndexDirective(right.TableName, right.Alias)) + return false; + if (join.Left is VdbeJoinScanPlan build) + { + if (HasIndexDirective(build.TableName, build.Alias)) + return false; + described.Add(($"HASH JOIN {rightWithAlias}", new EqpJsonHashJoinOp(right.TableName, right.Alias, marker), parent)); + described.Add(( + $"SCAN {build.TableName}" + (build.Alias is null ? string.Empty : $" AS {build.Alias}"), + new EqpJsonScanOp(build.TableName, build.Alias, IndexName: null, Covering: false), + parent)); + return true; + } + + // Behind a longer prefix the build input is the prefix's joined rows, materialized + // before the probe (Turso's materialize_build_input, EqpDetail::HashBuild), named + // after the prefix's last table. + if (join.Left is not VdbeJoinOperatorPlan prefix + || LastTable(prefix) is not { } last + || HasIndexDirective(last.Table, last.Alias)) + { + return false; + } + + var materializeIndex = described.Count; + described.Add(( + $"MATERIALIZE hash build input for {last.Table}" + (last.Alias is null ? string.Empty : $" AS {last.Alias}"), + new EqpJsonCompiledJoinHashBuildOp(last.Table, last.Alias), + parent)); + if (!Describe(prefix, materializeIndex)) + return false; + described.Add(($"HASH JOIN {rightWithAlias}", new EqpJsonHashJoinOp(right.TableName, right.Alias, marker), parent)); + return true; + } + + static (string Table, string? Alias)? LastTable(VdbeJoinOperatorPlan prefix) + => prefix.Right switch + { + VdbeJoinScanPlan scan => (scan.TableName, scan.Alias), + { SearchMetadata: { } metadata } => (metadata.TableName, metadata.Alias), + _ => null, + }; + + bool HasIndexDirective(string tableName, string? alias) + { + var qualifier = alias ?? tableName; + return leaves.Any(leaf => leaf.IndexDirective is not null + && string.Equals(leaf.Alias ?? leaf.Name, qualifier, StringComparison.OrdinalIgnoreCase)); + } + + string? EphemeralIndexName(string tableName, string? alias) + { + // Turso names a temporary index ephemeral_{table}_{internal id}; a FROM table's + // internal id is its 1-based position in the statement's FROM list. + var qualifier = alias ?? tableName; + for (var position = 0; position < leaves.Count; position++) + { + if (string.Equals(leaves[position].Alias ?? leaves[position].Name, qualifier, StringComparison.OrdinalIgnoreCase)) + return $"ephemeral_{tableName}_t{position + 1}"; + } + + return null; + } + } private static string? GetCompiledJoinMarker(VdbeJoinKind kind) => kind switch { VdbeJoinKind.Inner => "inner", diff --git a/src/Ahtola.Core/Execution/VdbeHashJoinRuntime.cs b/src/Ahtola.Core/Execution/VdbeHashJoinRuntime.cs index 61f1c9ed..032587f8 100644 --- a/src/Ahtola.Core/Execution/VdbeHashJoinRuntime.cs +++ b/src/Ahtola.Core/Execution/VdbeHashJoinRuntime.cs @@ -229,7 +229,7 @@ private static IEnumerable EnumerateCore( { HashSpill? spill = null; var residency = new PartitionResidencyCache(); - var trackUnmatchedBuild = buildIsRight && plan.Kind is VdbeJoinKind.Right or VdbeJoinKind.Full; + var trackUnmatchedBuild = KeepsUnmatchedBuild(plan, buildIsRight); var spillInfrastructureBytes = VdbeManagedFootprint.EstimateHashSpillInfrastructure( context.Options.TemporaryDirectory, PartitionCount, @@ -403,9 +403,9 @@ IEnumerable DrainPendingBatch() } } - if (!matchedProbe && buildIsRight && plan.Kind is VdbeJoinKind.Left or VdbeJoinKind.Full) + if (!matchedProbe && KeepsUnmatchedProbe(plan, buildIsRight)) { - yield return Combine(probe, NullRow(buildNode)); + yield return NullExtendProbe(probe, buildNode, buildIsRight); if (maximumRows is { } maximum && ++emitted >= maximum) yield break; } @@ -500,7 +500,7 @@ IEnumerable DrainPendingBatch() context.ThrowIfCancellationRequested(); if (matched![index]) continue; - yield return Combine(nullProbe, buffered[index].Row); + yield return NullExtendBuild(buffered[index].Row, nullProbe, buildIsRight); if (maximumRows is { } maximum && ++emitted >= maximum) yield break; } @@ -516,7 +516,7 @@ IEnumerable DrainPendingBatch() var build = lease.Entry; if (spill.IsMatched(build.Ordinal, context)) continue; - combined = Combine(nullProbe, build.Row); + combined = NullExtendBuild(build.Row, nullProbe, buildIsRight); } yield return combined; if (maximumRows is { } maximum && ++emitted >= maximum) @@ -673,9 +673,9 @@ IEnumerable EmitOneIndex(int localIndex) outputRetainedBytes[localIndex] = 0; } } - else if (buildIsRight && plan.Kind is VdbeJoinKind.Left or VdbeJoinKind.Full) + else if (KeepsUnmatchedProbe(plan, buildIsRight)) { - yield return Combine(batch[localIndex].Probe!, NullRow(buildNode)); + yield return NullExtendProbe(batch[localIndex].Probe!, buildNode, buildIsRight); } } else @@ -706,8 +706,8 @@ IEnumerable EmitOneIndex(int localIndex) } } - if (!matchedProbe && buildIsRight && plan.Kind is VdbeJoinKind.Left or VdbeJoinKind.Full) - yield return Combine(probe, NullRow(buildNode)); + if (!matchedProbe && KeepsUnmatchedProbe(plan, buildIsRight)) + yield return NullExtendProbe(probe, buildNode, buildIsRight); } ReleaseInput(localIndex); @@ -777,7 +777,7 @@ bool ReserveSlot(string key, VdbeJoinRow probe, int candidateCount, bool needsNu var localIndex = localIndices[slot]; var key = batch[localIndex].Key!; var probe = batch[localIndex].Probe!; - var needsNullExtension = buildIsRight && plan.Kind is VdbeJoinKind.Left or VdbeJoinKind.Full; + var needsNullExtension = KeepsUnmatchedProbe(plan, buildIsRight); if (!ReserveSlot(key, probe, resolution.Loaded.CountFor(key), needsNullExtension, out var reserved)) continue; // Deferred whole to individual streaming; nothing evaluated. @@ -795,7 +795,7 @@ bool ReserveSlot(string key, VdbeJoinRow probe, int candidateCount, bool needsNu matchedOrdinals?.Add(build.Ordinal); } if (matches is null && needsNullExtension) - (matches ??= []).Add(Combine(probe, NullRow(buildNode))); + (matches ??= []).Add(NullExtendProbe(probe, buildNode, buildIsRight)); } catch { @@ -812,7 +812,7 @@ bool ReserveSlot(string key, VdbeJoinRow probe, int candidateCount, bool needsNu var localIndex = localIndices[slot]; var key = batch[localIndex].Key!; var probe = batch[localIndex].Probe!; - var needsNullExtension = buildIsRight && plan.Kind is VdbeJoinKind.Left or VdbeJoinKind.Full; + var needsNullExtension = KeepsUnmatchedProbe(plan, buildIsRight); if (!ReserveSlot(key, probe, resolution.Index.Find(key).Count, needsNullExtension, out var reserved)) continue; @@ -844,7 +844,7 @@ bool ReserveSlot(string key, VdbeJoinRow probe, int candidateCount, bool needsNu matchedOrdinals?.Add(ordinal); } if (matches is null && needsNullExtension) - (matches ??= []).Add(Combine(probe, NullRow(buildNode))); + (matches ??= []).Add(NullExtendProbe(probe, buildNode, buildIsRight)); } catch { @@ -879,7 +879,7 @@ bool ReserveSlot(string key, VdbeJoinRow probe, int candidateCount, bool needsNu var localIndex = localIndices[slot]; var key = batch[localIndex].Key!; var probe = batch[localIndex].Probe!; - var needsNullExtension = buildIsRight && plan.Kind is VdbeJoinKind.Left or VdbeJoinKind.Full; + var needsNullExtension = KeepsUnmatchedProbe(plan, buildIsRight); if (!ReserveSlot(key, probe, partitionEntryCount, needsNullExtension, out var reserved)) continue; @@ -922,11 +922,11 @@ bool ReserveSlot(string key, VdbeJoinRow probe, int candidateCount, bool needsNu if (reservedBySlot[slot] == 0) continue; var localIndex = localIndices[slot]; - var needsNullExtension = buildIsRight && plan.Kind is VdbeJoinKind.Left or VdbeJoinKind.Full; + var needsNullExtension = KeepsUnmatchedProbe(plan, buildIsRight); var matches = matchesBySlot[slot]; if (matches is null && needsNullExtension) { - (matches ??= []).Add(Combine(batch[localIndex].Probe!, NullRow(buildNode))); + (matches ??= []).Add(NullExtendProbe(batch[localIndex].Probe!, buildNode, buildIsRight)); matchesBySlot[slot] = matches; } } @@ -1216,6 +1216,27 @@ private static bool Matches( : plan.Condition(build, probe, combined); } + /// + /// Whether the preserved side of an outer join is the build input, whose unmatched rows are + /// emitted after the probe scan (Turso HashJoinType::keeps_unmatched_build_rows). + /// + private static bool KeepsUnmatchedBuild(VdbeJoinOperatorPlan plan, bool buildIsRight) + => buildIsRight + ? plan.Kind is VdbeJoinKind.Right or VdbeJoinKind.Full + : plan.Kind is VdbeJoinKind.Left or VdbeJoinKind.Full; + + /// Whether an unmatched probe row is emitted null-extended as it streams. + private static bool KeepsUnmatchedProbe(VdbeJoinOperatorPlan plan, bool buildIsRight) + => buildIsRight + ? plan.Kind is VdbeJoinKind.Left or VdbeJoinKind.Full + : plan.Kind is VdbeJoinKind.Right or VdbeJoinKind.Full; + + private static VdbeJoinRow NullExtendProbe(VdbeJoinRow probe, VdbeJoinPlanNode buildNode, bool buildIsRight) + => buildIsRight ? Combine(probe, NullRow(buildNode)) : Combine(NullRow(buildNode), probe); + + private static VdbeJoinRow NullExtendBuild(VdbeJoinRow build, VdbeJoinRow nullProbe, bool buildIsRight) + => buildIsRight ? Combine(nullProbe, build) : Combine(build, nullProbe); + private static VdbeJoinRow Combine(VdbeJoinRow build, VdbeJoinRow probe, bool buildIsRight) => buildIsRight ? Combine(probe, build) diff --git a/src/Ahtola.Core/Execution/VdbeProgram.cs b/src/Ahtola.Core/Execution/VdbeProgram.cs index acb4d5a3..4a82a22b 100644 --- a/src/Ahtola.Core/Execution/VdbeProgram.cs +++ b/src/Ahtola.Core/Execution/VdbeProgram.cs @@ -1871,6 +1871,19 @@ public VdbeJoinEquiProbe( public Func BuildLeftKey { get; } public Func BuildRightKey { get; } + + /// + /// The right-side column names of the hash key, in key order, when every key is a plain + /// column of the right input (EXPLAIN QUERY PLAN names them as the automatic index's + /// constraints); when any key is an expression. + /// + internal IReadOnlyList? RightKeyColumns { get; init; } + + /// + /// True when the only hash key is the right table's INTEGER PRIMARY KEY, so hashing the + /// right input is a rowid-keyed point lookup. + /// + internal bool RightKeyIsRowid { get; init; } } /// An INNER, LEFT, RIGHT, FULL, SEMI, or ANTI node in a materializing join plan. @@ -1894,8 +1907,11 @@ kind is VdbeJoinKind.Semi or VdbeJoinKind.Anti { if (!Enum.IsDefined(kind)) throw new ArgumentOutOfRangeException(nameof(kind)); - if (!hashBuildRight && kind is not VdbeJoinKind.Inner) - throw new ArgumentException("Hash-build-left is only valid for INNER joins.", nameof(hashBuildRight)); + // Hash-build-left hashes the left input and streams the right as the probe. For a LEFT or + // FULL join the build is the preserved side, whose unmatched rows VdbeHashJoinRuntime + // emits after the probe scan (Turso HashJoinType::LeftOuter / FullOuter). + if (!hashBuildRight && kind is not (VdbeJoinKind.Inner or VdbeJoinKind.Left or VdbeJoinKind.Full)) + throw new ArgumentException("Hash-build-left is only valid for INNER, LEFT, or FULL joins.", nameof(hashBuildRight)); if (!hashBuildRight && equiProbe is null) throw new ArgumentException("Hash-build-left requires an equijoin probe.", nameof(hashBuildRight)); diff --git a/src/Ahtola.Tests/CompiledJoinVdbeSqlRoutingTests.cs b/src/Ahtola.Tests/CompiledJoinVdbeSqlRoutingTests.cs index b3f9dd87..3730979d 100644 --- a/src/Ahtola.Tests/CompiledJoinVdbeSqlRoutingTests.cs +++ b/src/Ahtola.Tests/CompiledJoinVdbeSqlRoutingTests.cs @@ -72,8 +72,11 @@ public void ThreeWayInnerJoinRoutesThroughMaterializingJoinCursor() ReadRows( connection, "EXPLAIN QUERY PLAN SELECT a.av, b.bv, c.cv FROM a JOIN b ON a.id=b.id JOIN c ON b.id=c.id;") - .Should().ContainSingle() - .Which[3].Should().Be(SqlValue.Text("MANAGED COMPILED VDBE")); + .Select(row => row[3].AsText()) + .Should().Equal( + "SCAN a", + "SEARCH b USING COVERING INDEX ephemeral_b_t2 (id=?)", + "SEARCH c USING COVERING INDEX ephemeral_c_t3 (id=?)"); } [Test] @@ -475,8 +478,8 @@ public void RoutedAndFallbackProjectionErrorsKeepTheSameCallbackOrder() ReadRows( routed, "EXPLAIN QUERY PLAN SELECT fail_on_two(l.id) FROM l JOIN r ON l.id=r.id;") - .Should().ContainSingle().Which[3] - .Should().Be(SqlValue.Text("MANAGED COMPILED VDBE")); + .Select(row => row[3].AsText()) + .Should().Equal("SCAN l", "SEARCH r USING COVERING INDEX ephemeral_r_t2 (id=?)"); ReadRows( fallback, "EXPLAIN QUERY PLAN SELECT fail_on_two(l.id) FROM l JOIN r ON l.id=r.id COLLATE callback_collation;") diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index cc51598f..20ca8d71 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -165,9 +165,6 @@ turso/mvcc_sequence.sqltest::mvcc-wal-mvcc-roundtrip-autoincrement | intentional # entries below are the differences that refresh exposed, pending closure. alter_table.sqltest::alter-table-add-column-preserve-schema-collate-locale | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected 'CREATE TABLE tbl (c TEXT COLLATE "fr-FR", d)' but was 'CREATE TABLE tbl (c TEXT COLLATE 'fr-FR', d)' integrity_check/memory.sqltest::integrity-check-qualified-corrupt-attached-database | intentional managed-ATTACH policy: a file ATTACH needs a file-backed managed primary whose IFileSystem the attachment shares (a :memory: primary has none), and the corrupt fixture is independently rejected at open (see the parity_corrupt_index entries) -join/hash.sqltest::hash-join-result-matrix-full-join-uses-a-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN matrix_right AS r' -join/hash.sqltest::hash-join-result-matrix-left-join-uses-a-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN matrix_right AS r' -join/hash.sqltest::hash-join-result-matrix-uses-a-hash-join | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN matrix_right AS r.*SCAN matrix_left AS l' join/hash_join_order_by.sqltest::hash-join-keeps-outer-table-order-elision | database variant 1/1 (:memory:): expected 3 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN c' but was '0/0/0/MANAGED EVALUATOR FALLBACK' join/hash_join_order_by.sqltest::hash-join-materialized-prefix-uses-sorter | database variant 1/1 (:memory:): output does not match pattern '(?s)MATERIALIZE hash build input for a.*HASH JOIN b.*USE SORTER FOR ORDER BY' join/hash_join_order_by.sqltest::hash-join-order-by-build-table-uses-sorter | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN b.*SCAN a.*USE SORTER FOR ORDER BY' @@ -176,10 +173,7 @@ join/memory.sqltest::expression-index-after-earlier-join-reads-source-table | da join/memory.sqltest::expression-index-hash-build-reads-source-table | database variant 1/1 (:memory:): output does not match pattern '(?s)MATERIALIZE hash build input for expression_build_orders AS orders.*HASH JOIN expression_build_inventory AS inventory.*SEARCH orders join/memory.sqltest::future-table-cannot-make-index-usable | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN future_index_child AS child.*SEARCH group_table USING INTEGER PRIMARY KEY.*SCAN future_index_parent AS parent' join/memory.sqltest::group-sort-cost-keeps-cheaper-correlated-seeks | database variant 1/1 (:memory:): output does not match pattern '(?s)CORRELATED SCALAR SUBQUERY.*SEARCH child USING INDEX sqlite_autoindex_sort_cost_child_1 \(parent_id=\?\)' -join/memory.sqltest::hash-build-row-limit-counts-prefix-rows | database variant 1/1 (:memory:): output does not match pattern '"detail":"SEARCH probe_rows USING COVERING INDEX ephemeral_prefix_row_limit_probe_t3' join/memory.sqltest::hash-join-cost-includes-rows-visited | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN hash_cost_right".*"access_cost":300110000' -join/memory.sqltest::index-suffix-does-not-prevent-hash-join | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN suffix_hash_child AS child.*SCAN suffix_hash_parent AS parent' -join/memory.sqltest::indexed-build-hashes-an-unfiltered-large-probe | database variant 1/1 (:memory:): output does not match pattern '(?s)HASH JOIN indexed_hash_child AS child.*SCAN indexed_hash_parent AS parent' join/memory.sqltest::materialized-hash-build-keeps-index-search | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"MATERIALIZE hash build input for selected_hash_build AS build_rows".*"detail":"SEARCH build_rows USING INDEX selected_hash join/memory.sqltest::no-stats-hash-join-keeps-unindexed-probe-fanout | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN c".*"rows_per_input":100000.*"detail":"SEARCH d USING INTEGER PRIMARY KEY' join/memory.sqltest::partial-stats-hash-join-uses-known-unique-key-count | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN c".*"rows_per_input":5.*"detail":"SEARCH d USING INTEGER PRIMARY KEY' diff --git a/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs b/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs index 8a0c2b80..64f9a078 100644 --- a/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs +++ b/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs @@ -163,6 +163,9 @@ WHERE u.age > 21 """); rows.Should().ContainSingle().Which.Should().Equal(SqlValue.Text("Ada"), SqlValue.Real(12.5)); + // A cancellation-capable statement stays on the evaluator, whose LEFT JOIN seeks the + // declared indexes; EXPLAIN stepped the same way describes those paths. + using var cancellation = new CancellationTokenSource(); using var document = JsonDocument.Parse( ReadAll( connection, @@ -173,7 +176,8 @@ FROM users u LEFT JOIN orders o ON o.user_id = u.id WHERE u.age > 21 ORDER BY o.amount; - """).Single()); + """, + cancellation.Token).Single()); var nodes = document.RootElement.GetProperty("nodes"); nodes[0].GetProperty("op").GetProperty("index").GetProperty("name").GetString().Should().Be("idx_users_age"); nodes[0].GetProperty("op").GetProperty("constraints")[0].GetString().Should().Be("age>?"); @@ -714,12 +718,13 @@ public void CompiledSelfJoinSearchRetainsItsChosenIndexAndRightAlias() """; ReadValues(connection, query).Should().HaveCount(2); var details = ReadPlanDetails(connection, "EXPLAIN QUERY PLAN " + query); - details.Should().ContainSingle().Which.Should().Be("SEARCH items USING INDEX items_k (k=?)"); + // Turso's SEARCH names the alias alone (core/translate/eqp.rs Display). + details.Should().Equal("SCAN items AS a", "SEARCH b USING INDEX items_k (k=?)"); using var document = JsonDocument.Parse( ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON " + query).Single()); - var node = document.RootElement.GetProperty("nodes")[0]; - node.GetProperty("detail").GetString().Should().Be(details[0]); + var node = document.RootElement.GetProperty("nodes")[1]; + node.GetProperty("detail").GetString().Should().Be(details[1]); var op = node.GetProperty("op"); op.GetProperty("type").GetString().Should().Be("search"); op.GetProperty("table").GetString().Should().Be("items"); @@ -731,7 +736,7 @@ public void CompiledSelfJoinSearchRetainsItsChosenIndexAndRightAlias() op.GetProperty("index").GetProperty("ephemeral").GetBoolean().Should().BeFalse(); op.GetProperty("constraints").EnumerateArray().Select(static value => value.GetString()) .Should().Equal("k=?"); - var outerScan = document.RootElement.GetProperty("nodes")[1]; + var outerScan = document.RootElement.GetProperty("nodes")[0]; outerScan.GetProperty("detail").GetString().Should().Be("SCAN items AS a"); outerScan.GetProperty("op").GetProperty("type").GetString().Should().Be("scan"); outerScan.GetProperty("op").GetProperty("alias").GetString().Should().Be("a"); @@ -800,15 +805,15 @@ FROM l AS a LEFT JOIN r AS b ON a.k < b.k ReadValues(connection, "EXPLAIN " + query) .Should().Contain(row => row[1].AsText() == "OpenJoinCursor"); ReadPlanDetails(connection, "EXPLAIN QUERY PLAN " + query) - .Should().Equal("MANAGED COMPILED VDBE"); + .Should().Equal("SCAN l AS a", "SCAN r AS b LEFT-JOIN", "USE SORTER FOR ORDER BY"); using var document = JsonDocument.Parse( ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON " + query).Single()); var nodes = document.RootElement.GetProperty("nodes"); - nodes.GetArrayLength().Should().Be(2); + nodes.GetArrayLength().Should().Be(3); nodes[0].GetProperty("detail").GetString().Should().Be("SCAN l AS a"); nodes[0].GetProperty("op").GetProperty("type").GetString().Should().Be("scan"); - nodes[1].GetProperty("detail").GetString().Should().Be("SCAN r AS b"); + nodes[1].GetProperty("detail").GetString().Should().Be("SCAN r AS b LEFT-JOIN"); nodes[1].GetProperty("op").GetProperty("type").GetString().Should().Be("scan"); nodes[1].GetProperty("op").GetProperty("join").GetString().Should().Be("left"); } @@ -847,7 +852,7 @@ public void CompiledSelfJoinWithoutIndexKeepsProbeAndBuildAliasesDistinct() } [Test] - public void CompiledHashBuildLeftUsesTheChosenSmallTable() + public void CompiledHashBuildLeftHashesTheTableTheCostModelPlacesFirst() { using var embedded = new EmbeddedDatabase(); using var connection = embedded.Connect(); @@ -875,20 +880,24 @@ public void CompiledHashBuildLeftUsesTheChosenSmallTable() ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON " + query).Single()); var nodes = document.RootElement.GetProperty("nodes"); nodes.GetArrayLength().Should().Be(2); + // Turso's hash cost charges a build row (hash_cpu_cost + hash_insert_cost) less than a + // probe row (hash_cpu_cost + hash_lookup_cost), so the 40-row table is scanned first and + // hashed, and the 3-row table streams as the probe (access_method.rs + // estimate_hash_join_cost, cost_params.rs). nodes[0].GetProperty("op").GetProperty("type").GetString().Should().Be("hash_join"); - nodes[0].GetProperty("op").GetProperty("table").GetString().Should().Be("big"); - nodes[0].GetProperty("op").GetProperty("alias").GetString().Should().Be("b"); + nodes[0].GetProperty("op").GetProperty("table").GetString().Should().Be("small"); + nodes[0].GetProperty("op").GetProperty("alias").GetString().Should().Be("s"); nodes[0].GetProperty("op").GetProperty("join").GetString().Should().Be("inner"); nodes[1].GetProperty("op").GetProperty("type").GetString().Should().Be("scan"); - nodes[1].GetProperty("op").GetProperty("table").GetString().Should().Be("small"); - nodes[1].GetProperty("op").GetProperty("alias").GetString().Should().Be("s"); + nodes[1].GetProperty("op").GetProperty("table").GetString().Should().Be("big"); + nodes[1].GetProperty("op").GetProperty("alias").GetString().Should().Be("b"); nodes[1].GetProperty("op").TryGetProperty("join", out _).Should().BeFalse(); nodes.EnumerateArray().Should().OnlyContain(static node => node.GetProperty("parent").ValueKind == JsonValueKind.Null); } [Test] - public void CompiledThreeTableHashBuildLeftNestsTheActualJoinedPrefix() + public void CompiledThreeTableRowidEquijoinsSearchEachIntegerPrimaryKey() { using var embedded = new EmbeddedDatabase(); using var connection = embedded.Connect(); @@ -902,6 +911,9 @@ public void CompiledThreeTableHashBuildLeftNestsTheActualJoinedPrefix() string.Join(", ", Enumerable.Range(1, 40).Select(value => $"({value}, 'b{value}')")) + "; ANALYZE;"); + // NOT INDEXED keeps the rowid candidate (optimizer/mod.rs enforce_indexed_by_hints), and a + // unique rowid seek per outer row beats any hash join (access_method.rs), so each later + // table is a rowid lookup keyed by the preceding row. const string query = """ SELECT s.label, b.label FROM small AS s NOT INDEXED JOIN seed AS e NOT INDEXED ON s.id = e.x @@ -910,43 +922,80 @@ public void CompiledThreeTableHashBuildLeftNestsTheActualJoinedPrefix() ReadValues(connection, query).Should().HaveCount(3); var compiled = ReadValues(connection, "EXPLAIN " + query) .Single(row => row[1].AsText() == "OpenJoinCursor")[5].AsText(); - compiled.Should().Contain("hash-build left"); + compiled.Should().Contain("hash-build right"); compiled.Should().Contain("scan order: s, e, b"); - ReadPlanDetails(connection, "EXPLAIN QUERY PLAN " + query) - .Should().Equal("MANAGED COMPILED VDBE"); + ReadPlanDetails(connection, "EXPLAIN QUERY PLAN " + query).Should().Equal( + "SCAN small AS s", + "SEARCH e USING INTEGER PRIMARY KEY (rowid=?)", + "SEARCH b USING INTEGER PRIMARY KEY (rowid=?)"); using var document = JsonDocument.Parse( ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON " + query).Single()); var nodes = document.RootElement.GetProperty("nodes"); - nodes.GetArrayLength().Should().Be(4); - nodes.EnumerateArray().Select(static node => node.GetProperty("id").GetInt32()) - .Should().Equal(1, 2, 3, 4); + nodes.GetArrayLength().Should().Be(3); + nodes.EnumerateArray().Should().OnlyContain(static node => + node.GetProperty("parent").ValueKind == JsonValueKind.Null); + nodes[0].GetProperty("op").GetProperty("type").GetString().Should().Be("scan"); + nodes[0].GetProperty("op").GetProperty("alias").GetString().Should().Be("s"); + foreach (var (node, alias) in new[] { (nodes[1], "e"), (nodes[2], "b") }) + { + node.GetProperty("op").GetProperty("type").GetString().Should().Be("search"); + node.GetProperty("op").GetProperty("alias").GetString().Should().Be(alias); + node.GetProperty("op").GetProperty("join").GetString().Should().Be("inner"); + node.GetProperty("op").GetProperty("constraints")[0].GetString().Should().Be("rowid=?"); + } + + document.RootElement.GetRawText().Should().NotContain("hash_build"); + } + + [Test] + public void CompiledThreeTableHashBuildLeftMaterializesTheJoinedPrefix() + { + using var embedded = new EmbeddedDatabase(); + using var connection = embedded.Connect(); + Execute(connection, + "CREATE TABLE c(id INTEGER PRIMARY KEY); " + + "INSERT INTO c VALUES (1), (2), (3); " + + "CREATE TABLE a(id INTEGER PRIMARY KEY, c_id INTEGER, k INTEGER); " + + "INSERT INTO a SELECT value, ((value - 1) % 3) + 1, (((value - 1) % 3) + 1) * 10 " + + "FROM generate_series(1, 30); " + + "CREATE INDEX a_c_id ON a(c_id); " + + "CREATE TABLE b(k INTEGER); " + + "INSERT INTO b VALUES (30), (10), (20); " + + "INSERT INTO b SELECT 1000 + value FROM generate_series(1, 997); " + + "ANALYZE;"); + + // join/hash_join_order_by.sqltest hash-join-materialized-prefix-uses-sorter: the c ⋈ a + // prefix is the hash build input for the probe of b, so EQP nests the prefix under + // Turso's MATERIALIZE node (EqpDetail::HashBuild). + const string query = """ + SELECT c.id FROM c JOIN a ON a.c_id = c.id JOIN b ON b.k = a.k + WHERE c.id > 0 ORDER BY c.id; + """; + ReadValues(connection, query).Should().HaveCount(30); + ReadValues(connection, "EXPLAIN " + query) + .Single(row => row[1].AsText() == "OpenJoinCursor")[5].AsText() + .Should().Contain("hash-build left"); + using var document = JsonDocument.Parse( + ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON " + query).Single()); + var nodes = document.RootElement.GetProperty("nodes"); + nodes.EnumerateArray().Select(static node => node.GetProperty("id").GetInt32()) + .Should().Equal(1, 2, 3, 4, 5); nodes[0].GetProperty("parent").ValueKind.Should().Be(JsonValueKind.Null); - nodes[0].GetProperty("detail").GetString().Should() - .Be("MATERIALIZE hash build input for small AS s"); + nodes[0].GetProperty("detail").GetString().Should().Be("MATERIALIZE hash build input for a"); nodes[0].GetProperty("op").GetProperty("type").GetString().Should().Be("hash_build"); - nodes[0].GetProperty("op").GetProperty("table").GetString().Should().Be("small"); - nodes[0].GetProperty("op").GetProperty("alias").GetString().Should().Be("s"); - + nodes[0].GetProperty("op").GetProperty("table").GetString().Should().Be("a"); nodes[1].GetProperty("parent").GetInt32().Should().Be(1); - nodes[1].GetProperty("detail").GetString().Should().Be("HASH JOIN small AS s"); - nodes[1].GetProperty("op").GetProperty("type").GetString().Should().Be("hash_join"); - nodes[1].GetProperty("op").GetProperty("alias").GetString().Should().Be("s"); - nodes[1].GetProperty("op").TryGetProperty("join", out _).Should().BeFalse(); - + nodes[1].GetProperty("detail").GetString().Should().Be("SCAN c"); nodes[2].GetProperty("parent").GetInt32().Should().Be(1); - nodes[2].GetProperty("detail").GetString().Should().Be("SCAN seed AS e"); - nodes[2].GetProperty("op").GetProperty("type").GetString().Should().Be("scan"); - nodes[2].GetProperty("op").GetProperty("alias").GetString().Should().Be("e"); - nodes[2].GetProperty("op").GetProperty("join").GetString().Should().Be("inner"); - + nodes[2].GetProperty("op").GetProperty("table").GetString().Should().Be("a"); nodes[3].GetProperty("parent").ValueKind.Should().Be(JsonValueKind.Null); - nodes[3].GetProperty("detail").GetString().Should().Be("HASH JOIN big AS b"); - nodes[3].GetProperty("op").GetProperty("table").GetString().Should().Be("big"); + nodes[3].GetProperty("detail").GetString().Should().Be("HASH JOIN b"); + nodes[3].GetProperty("op").GetProperty("type").GetString().Should().Be("hash_join"); nodes[3].GetProperty("op").GetProperty("join").GetString().Should().Be("inner"); + nodes[4].GetProperty("detail").GetString().Should().Be("USE SORTER FOR ORDER BY"); } - [Test] public void CompiledThreeTableHashBuildRightDoesNotInventPrefixMaterialization() { @@ -977,7 +1026,7 @@ public void CompiledThreeTableHashBuildRightDoesNotInventPrefixMaterialization() } [Test] - public void CompiledJoinAutomaticIndexIsAnEphemeralCoveringSearch() + public void CompiledAnalyzedEquijoinWithoutIndexesIsReportedAsItsHashJoin() { using var embedded = new EmbeddedDatabase(); using var connection = embedded.Connect(); @@ -989,30 +1038,35 @@ public void CompiledJoinAutomaticIndexIsAnEphemeralCoveringSearch() "INSERT INTO inner_items VALUES " + string.Join(", ", Enumerable.Range(1, 100).Select(value => $"({value}, 'i{value}')")) + "; ANALYZE;"); + // With no usable index Turso's cheapest step is the left-deep hash join that builds the + // first table and probes the second (access_method.rs try_hash_join_access_method); the + // compiled operator executes exactly that build-left shape. const string query = """ SELECT o.k, i.payload FROM outer_items AS o JOIN inner_items AS i ON o.k = i.k ORDER BY o.k; """; ReadValues(connection, query).Should().HaveCount(100); - var detail = ReadPlanDetails(connection, "EXPLAIN QUERY PLAN " + query).Single(); - detail.Should().Be("SEARCH inner_items USING AUTOMATIC COVERING INDEX (k=?)"); + ReadValues(connection, "EXPLAIN " + query) + .Single(row => row[1].AsText() == "OpenJoinCursor")[5].AsText() + .Should().Contain("hash-build left"); + var details = ReadPlanDetails(connection, "EXPLAIN QUERY PLAN " + query); + details.Should().Equal("HASH JOIN inner_items AS i", "SCAN outer_items AS o", "USE SORTER FOR ORDER BY"); using var document = JsonDocument.Parse( ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON " + query).Single()); - var node = document.RootElement.GetProperty("nodes")[0]; - node.GetProperty("detail").GetString().Should().Be(detail); - var op = node.GetProperty("op"); - op.GetProperty("type").GetString().Should().Be("search"); - op.GetProperty("table").GetString().Should().Be("inner_items"); - op.GetProperty("alias").GetString().Should().Be("i"); - op.GetProperty("join").GetString().Should().Be("inner"); - op.GetProperty("index").GetProperty("name").GetString().Should().Be("automatic_inner_items"); - op.GetProperty("index").GetProperty("covering").GetBoolean().Should().BeTrue(); - op.GetProperty("index").GetProperty("ephemeral").GetBoolean().Should().BeTrue(); - op.GetProperty("constraints")[0].GetString().Should().Be("k=?"); + var nodes = document.RootElement.GetProperty("nodes"); + nodes.GetArrayLength().Should().Be(3); + nodes[0].GetProperty("detail").GetString().Should().Be(details[0]); + nodes[0].GetProperty("op").GetProperty("type").GetString().Should().Be("hash_join"); + nodes[0].GetProperty("op").GetProperty("table").GetString().Should().Be("inner_items"); + nodes[0].GetProperty("op").GetProperty("alias").GetString().Should().Be("i"); + nodes[0].GetProperty("op").GetProperty("join").GetString().Should().Be("inner"); + nodes[1].GetProperty("detail").GetString().Should().Be(details[1]); + nodes[1].GetProperty("op").GetProperty("type").GetString().Should().Be("scan"); + nodes[1].GetProperty("op").GetProperty("alias").GetString().Should().Be("o"); + nodes[2].GetProperty("op").GetProperty("type").GetString().Should().Be("order_by"); } - [Test] public void CompiledJoinPagerSeekReportsTheDurableIndexRatherThanAnAutomaticIndex() { @@ -1038,11 +1092,15 @@ FROM outer_items o JOIN inner_items i INDEXED BY inner_items_k ON o.k = i.k ORDER BY o.k; """; - var detail = ReadPlanDetails(reopenedConnection, "EXPLAIN QUERY PLAN " + query).Single(); - detail.Should().Be("SEARCH inner_items USING INDEX inner_items_k (k=?)"); + var details = ReadPlanDetails(reopenedConnection, "EXPLAIN QUERY PLAN " + query); + details.Should().Equal( + "SCAN outer_items AS o", + "SEARCH i USING INDEX inner_items_k (k=?)", + "USE SORTER FOR ORDER BY"); + var detail = details[1]; using var document = JsonDocument.Parse( ReadAll(reopenedConnection, "EXPLAIN QUERY PLAN FORMAT=JSON " + query).Single()); - var node = document.RootElement.GetProperty("nodes")[0]; + var node = document.RootElement.GetProperty("nodes")[1]; node.GetProperty("detail").GetString().Should().Be(detail); var op = node.GetProperty("op"); op.GetProperty("type").GetString().Should().Be("search"); @@ -1101,14 +1159,17 @@ private static void Execute(EmbeddedConnection connection, string sql) } } - private static List ReadAll(EmbeddedConnection connection, string sql) + private static List ReadAll( + EmbeddedConnection connection, + string sql, + CancellationToken cancellationToken = default) { var rows = new List(); foreach (var statement in connection.PrepareScript(sql)) { using (statement) { - while (statement.Step(default) == StatementStepResult.Row) + while (statement.Step(cancellationToken) == StatementStepResult.Row) rows.Add(statement.GetValue(0).Kind == SqlValueKind.Text ? statement.GetValue(0).AsText() : statement.GetValue(0).ToString() ?? ""); diff --git a/src/Ahtola.Tests/IndexSeekJoinDirectAccessTests.cs b/src/Ahtola.Tests/IndexSeekJoinDirectAccessTests.cs index 261e27f1..2d7b075c 100644 --- a/src/Ahtola.Tests/IndexSeekJoinDirectAccessTests.cs +++ b/src/Ahtola.Tests/IndexSeekJoinDirectAccessTests.cs @@ -48,7 +48,7 @@ FROM outer_items JOIN inner_items INDEXED BY inner_items_partial using var reopened = EmbeddedDatabase.OpenFile("partial-index-join.db", fileSystem); using var reopenedConnection = reopened.Connect(); - ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Be("SEARCH inner_items USING INDEX inner_items_partial (k=?)"); reopened.ResetJoinOrderDiagnostics(); @@ -81,8 +81,8 @@ FROM outer_items JOIN inner_items ON outer_items.k = inner_items.k ORDER BY inner_items.payload; """; - ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() - .Should().NotContain("inner_items_partial"); + ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Select(static row => row[3].AsText()) + .Should().NotContain(static detail => detail.Contains("inner_items_partial")); database.ResetJoinOrderDiagnostics(); ReadRows(connection, sql).Select(row => row[0].AsText()).Should().Equal("dropped", "kept"); @@ -125,7 +125,7 @@ FROM outer_items JOIN inner_items INDEXED BY inner_items_lower_k using var reopened = EmbeddedDatabase.OpenFile("expression-index-join.db", fileSystem); using var reopenedConnection = reopened.Connect(); - ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Contain("inner_items_lower_k"); reopened.ResetJoinOrderDiagnostics(); diff --git a/src/Ahtola.Tests/IndexSeekJoinTests.cs b/src/Ahtola.Tests/IndexSeekJoinTests.cs index db10c90d..dd976ef1 100644 --- a/src/Ahtola.Tests/IndexSeekJoinTests.cs +++ b/src/Ahtola.Tests/IndexSeekJoinTests.cs @@ -38,8 +38,8 @@ FROM outer_items JOIN inner_items INDEXED BY inner_items_k FROM outer_items JOIN inner_items INDEXED BY inner_items_k ON outer_items.k = inner_items.k; """) - .Should().ContainSingle() - .Which[3].AsText().Should().Be("SEARCH inner_items USING INDEX inner_items_k (k=?)"); + .Select(row => row[3].AsText()) + .Should().Equal("SCAN outer_items", "SEARCH inner_items USING INDEX inner_items_k (k=?)"); database.JoinIndexSeekMetrics.PlansCreated.Should().Be(0); database.JoinIndexSeekMetrics.IndexRowsMaterialized.Should().Be(0); @@ -94,8 +94,9 @@ FROM outer_items JOIN inner_items INDEXED BY inner_items_ab foreach (var statement in setup) Execute(connection, statement); - var detail = ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText(); - detail.Should().Be("SEARCH inner_items USING INDEX inner_items_ab (a=?, b=?)"); + var detail = ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText(); + // Turso joins a search's constraints with " AND " (core/translate/eqp.rs Display). + detail.Should().Be("SEARCH inner_items USING INDEX inner_items_ab (a=? AND b=?)"); database.ResetJoinOrderDiagnostics(); ReadRows(connection, sql).Should().HaveCount(2); database.JoinIndexSeekMetrics.SeeksAttempted.Should().Be(3); @@ -128,7 +129,7 @@ FROM outer_items JOIN inner_items INDEXED BY inner_items_k AssertMatchesSqlite(setup, sql); using var connection = OpenManaged(setup); - ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Contain("SEARCH inner_items USING INDEX inner_items_k"); } @@ -154,7 +155,11 @@ public void UnsupportedIndexShapesFallBackWithoutAdvertisingSearch( var detail = ReadRows( connection, $"EXPLAIN QUERY PLAN SELECT outer_items.a FROM outer_items JOIN inner_items ON {condition};"); - detail.Select(row => row[3].AsText()).Should().NotContain(value => value.StartsWith("SEARCH ", StringComparison.Ordinal)); + // No declared index can drive these equalities; an automatic (ephemeral) index built + // for the join is still allowed, but the unusable declared index is never advertised. + detail.Select(row => row[3].AsText()).Should().NotContain(value => + value.StartsWith("SEARCH ", StringComparison.Ordinal) + && !value.Contains("INDEX ephemeral_", StringComparison.Ordinal)); ReadRows( connection, $"SELECT outer_items.a FROM outer_items JOIN inner_items ON {condition};") @@ -215,7 +220,7 @@ FROM outer_items JOIN inner_items INDEXED BY inner_items_k_desc ON outer_items.k = inner_items.k ORDER BY outer_items.k; """; - ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Be("SEARCH inner_items USING INDEX inner_items_k_desc (k=?)"); database.ResetJoinOrderDiagnostics(); ReadRows(connection, sql).Select(row => row[0].AsText()).Should().Equal("p2", "p250", "p499"); @@ -275,7 +280,7 @@ FROM outer_items JOIN inner_items INDEXED BY inner_items_k ON outer_items.k = inner_items.k ORDER BY outer_items.k; """; - ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Be("SEARCH inner_items USING INDEX inner_items_k (k=?)"); ReadRows(connection, sql).Select(row => row[0].AsText()).Should().Equal("p2", "p40"); } diff --git a/src/Ahtola.Tests/PlannerAccessPathDepthTests.cs b/src/Ahtola.Tests/PlannerAccessPathDepthTests.cs index 099367d4..f8ce0970 100644 --- a/src/Ahtola.Tests/PlannerAccessPathDepthTests.cs +++ b/src/Ahtola.Tests/PlannerAccessPathDepthTests.cs @@ -272,7 +272,7 @@ public void Stat4VectorDerivationScalesByRowsAndColumnsWithoutSampleMultiplier() } [Test] - public void ProfitableInnerJoinBuildsOneAutomaticCoveringIndex() + public void AnalyzedUnindexedInnerJoinHashesItsFirstTableOnce() { using var database = new EmbeddedDatabase(); using var connection = database.Connect(); @@ -296,17 +296,20 @@ public void ProfitableInnerJoinBuildsOneAutomaticCoveringIndex() FROM outer_items JOIN inner_items ON outer_items.k=inner_items.k ORDER BY outer_items.k; """; - PlanDetail(connection, sql).Should().Be( - "SEARCH inner_items USING AUTOMATIC COVERING INDEX (k=?)"); + // Turso's hash join (build the first table, probe the second) is cheaper than building + // a temporary index on inner_items, so no automatic index is built. + PlanDetails(connection, sql).Should().Equal( + "HASH JOIN inner_items", + "SCAN outer_items", + "USE SORTER FOR ORDER BY"); database.ResetJoinOrderDiagnostics(); var rows = ReadRows(connection, sql); rows.Should().HaveCount(100); rows[0].Should().Equal(SqlValue.Integer(1), SqlValue.Text("i1")); rows[^1].Should().Equal(SqlValue.Integer(100), SqlValue.Text("i100")); - database.JoinIndexSeekMetrics.AutomaticIndexesBuilt.Should().Be(1); - database.JoinIndexSeekMetrics.IndexRowsMaterialized.Should().Be(100); - database.JoinIndexSeekMetrics.CandidateRowsVisited.Should().Be(100); + database.JoinIndexSeekMetrics.AutomaticIndexesBuilt.Should().Be(0); + database.JoinIndexSeekMetrics.IndexRowsMaterialized.Should().Be(0); } [Test] @@ -376,8 +379,12 @@ FROM first_keys JOIN second_keys ON target.k2=second_keys.k2 ORDER BY target.k1; """; - PlanDetails(connection, sql).Should().Contain(detail => - detail.StartsWith("SEARCH target USING AUTOMATIC COVERING INDEX", StringComparison.Ordinal)); + // second_keys is joined after target, so no access path for target may constrain k2. + var details = PlanDetails(connection, sql); + details.Should().Contain(detail => detail.Contains("target", StringComparison.Ordinal)); + details.Should().NotContain(detail => + detail.StartsWith("SEARCH target ", StringComparison.Ordinal) + && detail.Contains("k2=?", StringComparison.Ordinal)); ReadRows(connection, sql).Select(row => row[0].AsText()) .Should().Equal(Enumerable.Range(1, 100).Select(value => $"p{value}")); } @@ -411,8 +418,10 @@ FROM outer_items JOIN inner_items INDEXED BY inner_items_k ON outer_items.k=inner_items.k ORDER BY outer_items.k; """; - PlanDetail(reopenedConnection, sql).Should().Be( - "SEARCH inner_items USING INDEX inner_items_k (k=?)"); + PlanDetails(reopenedConnection, sql).Should().Equal( + "SCAN outer_items", + "SEARCH inner_items USING INDEX inner_items_k (k=?)", + "USE SORTER FOR ORDER BY"); reopened.JoinIndexSeekMetrics.IndexRowsMaterialized.Should().Be(0); reopened.ResetJoinOrderDiagnostics(); @@ -431,8 +440,10 @@ FROM outer_items JOIN inner_items INDEXED BY inner_items_k_payload ON outer_items.k=inner_items.k ORDER BY outer_items.k; """; - PlanDetail(reopenedConnection, coveringSql).Should().Be( - "SEARCH inner_items USING COVERING INDEX inner_items_k_payload (k=?)"); + PlanDetails(reopenedConnection, coveringSql).Should().Equal( + "SCAN outer_items", + "SEARCH inner_items USING COVERING INDEX inner_items_k_payload (k=?)", + "USE SORTER FOR ORDER BY"); reopened.ResetJoinOrderDiagnostics(); ReadRows(reopenedConnection, coveringSql).Select(row => row[0].AsText()) .Should().Equal("p3", "p197", "p499"); diff --git a/src/Ahtola.Tests/PlannerStat1JoinCostTests.cs b/src/Ahtola.Tests/PlannerStat1JoinCostTests.cs index 3dba05ea..922927b9 100644 --- a/src/Ahtola.Tests/PlannerStat1JoinCostTests.cs +++ b/src/Ahtola.Tests/PlannerStat1JoinCostTests.cs @@ -99,7 +99,7 @@ public void WithoutAnalyzeTwoTableInnerKeepsFromOrder() } [Test] - public void ThreeWayInnerEquijoinHashBuildsSmallerLeftSideAfterAnalyze() + public void ThreeWayInnerRowidEquijoinSeeksEachLaterTableAfterAnalyze() { using var connection = new EmbeddedDatabase().Connect(); Execute(connection, "CREATE TABLE small(id INTEGER PRIMARY KEY, s TEXT);"); @@ -113,7 +113,9 @@ public void ThreeWayInnerEquijoinHashBuildsSmallerLeftSideAfterAnalyze() Execute(connection, $"INSERT INTO seed VALUES ({i});"); Execute(connection, "ANALYZE;"); - // Root is (small ⋈ seed) ⋈ big: left residual ~3, right big=40 → hash-build left. + // Both later tables are joined on their INTEGER PRIMARY KEY, and a unique rowid seek per + // outer row is cheaper than any hash join (Turso v0.8.1 access_method.rs), so every step + // is a rowid lookup keyed by the accumulated left row. var explain = ReadRows( connection, """ @@ -122,9 +124,20 @@ public void ThreeWayInnerEquijoinHashBuildsSmallerLeftSideAfterAnalyze() """); var openJoin = explain.Single(row => row[1].AsText() == "OpenJoinCursor"); // EXPLAIN columns: addr, opcode, p1, p2, p3, p4, comment — p4 is index 5. - openJoin[5].AsText().Should().Contain("hash-build left"); + openJoin[5].AsText().Should().Contain("hash-build right"); // The FROM order is already optimal here, so the cost optimizer keeps it. openJoin[5].AsText().Should().Contain("scan order: small, seed, big"); + ReadRows( + connection, + """ + EXPLAIN QUERY PLAN SELECT small.s, big.b + FROM small JOIN seed ON small.id = seed.x JOIN big ON small.id = big.id; + """) + .Select(row => row[3].AsText()) + .Should().Equal( + "SCAN small", + "SEARCH seed USING INTEGER PRIMARY KEY (rowid=?)", + "SEARCH big USING INTEGER PRIMARY KEY (rowid=?)"); var rows = ReadRows( connection, @@ -152,8 +165,7 @@ public void ThreeWayInnerEquijoinReordersToDriveFromTheSmallestTable() Execute(connection, "ANALYZE;"); // FROM order is big(40), small(3), seed(1). The N-way cost optimizer drives from the - // one-row table instead, and the resulting root builds its hash over that tiny - // accumulated left side rather than over the 3-row right input. + // one-row table instead, then looks up each later table by its INTEGER PRIMARY KEY. var explain = ReadRows( connection, """ @@ -162,7 +174,7 @@ public void ThreeWayInnerEquijoinReordersToDriveFromTheSmallestTable() """); var openJoin = explain.Single(row => row[1].AsText() == "OpenJoinCursor"); openJoin[5].AsText().Should().Contain("scan order: seed, big, small"); - openJoin[5].AsText().Should().Contain("hash-build left"); + openJoin[5].AsText().Should().Contain("hash-build right"); ReadRows( connection, diff --git a/src/Ahtola.Tests/WithoutRowidIndexSeekJoinTests.cs b/src/Ahtola.Tests/WithoutRowidIndexSeekJoinTests.cs index cf38ff03..65c61ea2 100644 --- a/src/Ahtola.Tests/WithoutRowidIndexSeekJoinTests.cs +++ b/src/Ahtola.Tests/WithoutRowidIndexSeekJoinTests.cs @@ -48,7 +48,7 @@ FROM outer_items JOIN entry using var reopened = EmbeddedDatabase.OpenFile("without-rowid-pk-join.db", fileSystem); using var reopenedConnection = reopened.Connect(); - ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Be("SEARCH entry USING COVERING INDEX sqlite_autoindex_entry_1 (code=?)"); reopened.ResetJoinOrderDiagnostics(); @@ -95,7 +95,7 @@ FROM outer_items JOIN entry INDEXED BY entry_tag using var reopened = EmbeddedDatabase.OpenFile("without-rowid-secondary-join.db", fileSystem); using var reopenedConnection = reopened.Connect(); - ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Be("SEARCH entry USING COVERING INDEX entry_tag (tag=?)"); reopened.ResetJoinOrderDiagnostics(); @@ -143,7 +143,7 @@ FROM outer_items JOIN entry INDEXED BY entry_tag using var reopened = EmbeddedDatabase.OpenFile("without-rowid-noncovering-join.db", fileSystem); using var reopenedConnection = reopened.Connect(); - ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Be("SEARCH entry USING INDEX entry_tag (tag=?)"); reopened.ResetJoinOrderDiagnostics(); @@ -191,7 +191,7 @@ FROM outer_items JOIN entry using var reopened = EmbeddedDatabase.OpenFile("without-rowid-composite-pk-join.db", fileSystem); using var reopenedConnection = reopened.Connect(); - ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Be("SEARCH entry USING COVERING INDEX sqlite_autoindex_entry_1 (part=?)"); reopened.ResetJoinOrderDiagnostics(); @@ -240,7 +240,7 @@ FROM outer_items JOIN entry INDEXED BY entry_tag using var reopened = EmbeddedDatabase.OpenFile("without-rowid-duplicate-secondary.db", fileSystem); using var reopenedConnection = reopened.Connect(); - ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(reopenedConnection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Be("SEARCH entry USING INDEX entry_tag (tag=?)"); reopened.ResetJoinOrderDiagnostics(); @@ -274,8 +274,8 @@ FROM outer_items JOIN entry INDEXED BY entry_tag ON outer_items.code = entry.code AND outer_items.tag = entry.tag ORDER BY entry.payload; """; - ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() - .Should().NotContain("sqlite_autoindex_entry_1"); + ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Select(static row => row[3].AsText()) + .Should().NotContain(static detail => detail.Contains("sqlite_autoindex_entry_1")); ReadRows(connection, sql).Select(row => row[0].AsText()).Should().Equal("p1", "p2"); } @@ -299,7 +299,7 @@ public void PrimaryKeyJoinWithoutOrderByStillUsesTheCostedPagerSeek() using var connection = reopened.Connect(); const string sql = "SELECT entry.payload FROM outer_items JOIN entry ON outer_items.code=entry.code;"; - ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single()[3].AsText() + ReadRows(connection, "EXPLAIN QUERY PLAN " + sql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Contain("sqlite_autoindex_entry_1"); reopened.ResetJoinOrderDiagnostics(); ReadRows(connection, sql).Single()[0].AsText().Should().Be("p2"); @@ -379,7 +379,7 @@ FROM outer_items JOIN entry NOT INDEXED // Baseline, no override yet: the implicit primary-key candidate is offered and used, so // the freely-planned query genuinely searches while the NOT INDEXED one does not. - ReadRows(connection, "EXPLAIN QUERY PLAN " + defaultSql).Single()[3].AsText() + ReadRows(connection, "EXPLAIN QUERY PLAN " + defaultSql).Single(static row => row[3].AsText().StartsWith("SEARCH ", StringComparison.Ordinal))[3].AsText() .Should().Be("SEARCH entry USING COVERING INDEX sqlite_autoindex_entry_1 (code=?)"); database.ResetJoinOrderDiagnostics(); ReadRows(connection, defaultSql).Select(row => row[0].AsText()).Should().Equal("p2", "p300"); From 5b4ef8fb12bf48b74b6fd7be07ac280556c3b8b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 19:38:00 -0400 Subject: [PATCH 15/20] Report Turso cost-model estimates in EXPLAIN QUERY PLAN FORMAT=JSON and seek join indexes on constant key prefixes FORMAT=JSON nodes now carry Turso v0.8.1's plan_estimate figures (optimizer/mod.rs) wherever the described access path is one the ported cost model prices exactly: - filtered full-table scans (estimate_scan_cost + cost_with_where_work, with constraint_output_multipliers and the closed-range factor), declining for BETWEEN/LIKE/IN, subqueries, OR-implied IN filters, or a constraint Turso could seek instead; - single-table index searches and ordered index scans (estimate_cost_for_scan_or_seek, with the is_index_ordered exemption from the non-covering full-scan penalty); - the evaluator's LEFT JOIN index plan (the null-supplying seek priced by the same inner-access planner as semi/anti joins, rows_after_join's outer-join Poisson term); - multi-index OR unions (multi_index.rs estimate_multi_index_scan_cost with rowid-only branches); - uncorrelated IN-subquery seeks (choose_best_in_seek_candidate, list rows capped at sqrt(rows)) and the correlated scalar subquery seek priced per outer row; - compound-select arms, FROM-clause coroutine subqueries (scan plus body cost), and recursive CTE scans (1,000,000-row fallback) with their one-row recursive input. Join planning: a `column = literal` term on a join member can now extend an index seek key (constraints.rs constant constraints) when the literal needs no affinity conversion and the column's collation matches the index; the compiled index seek uses the literal verbatim. Co-Authored-By: Claude Opus 5.5 --- .../JoinOrdering/JoinOrderEnumerator.cs | 35 +- .../JoinOrdering/JoinOrderModel.cs | 10 + ...eddedDatabase.ExplainQueryPlanDescriber.cs | 182 +++++++- .../EmbeddedDatabase.JoinOrderRewrites.cs | 113 +++++ .../EmbeddedDatabase.SemiAntiJoinAccess.cs | 409 +++++++++++++++++- src/Ahtola.Core/EmbeddedDatabase.cs | 330 +++++++++++++- .../managed-sqltest-expected-failures.txt | 9 - .../ExplainQueryPlanFormatJsonTests.cs | 24 +- 8 files changed, 1058 insertions(+), 54 deletions(-) diff --git a/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderEnumerator.cs b/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderEnumerator.cs index 04d904cc..e82175b4 100644 --- a/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderEnumerator.cs +++ b/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderEnumerator.cs @@ -786,7 +786,13 @@ internal static StepEvaluation EvaluateTursoStep( // (constraint_output_multipliers, join.rs:94-152). double RemainingSelectivity(IReadOnlyCollection consumed) { - var selectivity = memberLocalSelectivity; + var selectivity = 1.0; + foreach (var index in ready) + { + if (segment.Terms[index].TableMask == memberBit && !consumed.Contains(index)) + selectivity *= segment.Terms[index].TursoLocalSelectivity; + } + foreach (var equality in joinEqualities) { if (!consumed.Contains(equality.Term)) @@ -854,13 +860,38 @@ double OuterRows(double rowsPerOuterRow, double remainingSelectivity) } } + // A constant equality on the member binds the column as well + // (constraints.rs: a constraint whose other side reads no table). + if (bound < 0 && column.IndexExpression is null) + { + foreach (var index in ready) + { + var term = segment.Terms[index]; + if (term.TableMask == memberBit + && !term.CostOnly + && term.ConstantEqualityOrdinal == column.ColumnOrdinal + && string.Equals(term.ConstantEqualityCollation, column.Collation, StringComparison.OrdinalIgnoreCase) + && !termIndices.Contains(index)) + { + bound = index; + selectivityProduct *= term.TursoLocalSelectivity; + break; + } + } + } + if (bound < 0) break; termIndices.Add(bound); } - if (termIndices.Count == 0) + // The managed seek is keyed by the outer row, so at least one bound column must come + // from a join equality; constants may only extend that key. + if (termIndices.Count == 0 + || !termIndices.Any(index => joinEqualities.Any(equality => equality.Term == index))) + { continue; + } var unique = candidate.Unique && termIndices.Count >= candidate.Columns.Count; var rowsPerSeek = unique diff --git a/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderModel.cs b/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderModel.cs index afd45b27..36504ed6 100644 --- a/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderModel.cs +++ b/src/Ahtola.Core/Compilation/JoinOrdering/JoinOrderModel.cs @@ -161,6 +161,16 @@ internal sealed record JoinPredicateTerm( /// synthesized join node, because the surviving WHERE evaluates it. /// public bool CostOnly { get; init; } + + /// + /// For column = literal on one member (a constant constraint Turso lets an index seek + /// consume, constraints.rs), the column's table ordinal; otherwise -1. Such a term is never + /// an join key and is never hashed. + /// + public int ConstantEqualityOrdinal { get; init; } = -1; + + /// The collation a comparison uses. + public string? ConstantEqualityCollation { get; init; } } /// diff --git a/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs b/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs index de15523c..4cecde3f 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.ExplainQueryPlanDescriber.cs @@ -43,6 +43,88 @@ private static bool IsPlainEqpBaseTable(NamedTableSource source, QueryContext co && context.Views?.ContainsKey(source.Name) != true && context.VirtualTables?.ContainsKey(source.Name) != true; + /// + /// Turso's estimates for the two searches of the evaluator's LEFT JOIN plan: the preserved + /// side's first-loop index search and the null-supplying side's per-row index seek + /// (optimizer/mod.rs plan_estimate), each only when the ported cost model independently + /// selects the very search being described, so no number describes a plan Turso would not + /// run. An outer join keeps each outer row at least once (join.rs:192-248 rows_after_join). + /// + private (EqpJsonEstimate? Left, EqpJsonEstimate? Right) EstimateEvaluatorLeftJoinSearches( + SelectStatement statement, + JoinTableSource join, + Expression leftPredicate, + EmbeddedIndex leftIndex, + string leftConstraint, + EmbeddedIndex rightIndex, + string rightConstraint, + QueryContext context) + { + if (join is not { Left: NamedTableSource left, Right: NamedTableSource right, Condition: { } condition } + || !context.Tables.TryGetValue(left.Name, out var leftTable) + || !context.Tables.TryGetValue(right.Name, out var rightTable) + || !TryCollectReferencedTableColumns(statement, left, leftTable, out var leftColumns) + || !TryCollectReferencedTableColumns(statement, right, rightTable, out var rightColumns) + || EstimateTursoFirstLoopIndexSearch( + left, + leftIndex, + leftPredicate, + leftColumns, + leftConstraint.Split(" AND ", StringSplitOptions.TrimEntries), + context) is not { } leftEstimate) + { + return (null, null); + } + + if (PlanSemiOrAntiInnerAccess(join, leftPredicate, context, rightColumns) is not + { + Kind: SemiAntiInnerAccessKind.DeclaredIndexSearch, + } access + || !ReferenceEquals(access.Index, rightIndex) + || !access.ConstraintTexts.SequenceEqual([rightConstraint], StringComparer.OrdinalIgnoreCase) + || access.SeekTerms.Count != IndexExpressionSemantics.SplitConjuncts(condition).Count + || Math.Abs(access.InputRows - leftEstimate.OutputRows) > 1e-9 * Math.Max(1.0, leftEstimate.OutputRows)) + { + return (leftEstimate, null); + } + + var rowsPerInput = access.RowsPerOuterRow + Math.Exp(-access.RowsPerOuterRow); + var output = access.InputRows * rowsPerInput; + return (leftEstimate, new EqpJsonEstimate( + access.InputRows, + rowsPerInput, + output, + access.Cost, + leftEstimate.TotalCost + access.Cost)); + } + + /// + /// The estimate of an uncorrelated IN-list body that is a plain full scan of one table + /// (its rows become the list's value estimate), or when the body + /// groups, de-duplicates, orders, limits or aggregates, or filters in an unmodeled way. + /// + private EqpJsonEstimate? EstimateListSubqueryBody( + SelectStatement body, + NamedTableSource source, + QueryContext context) + { + if (body.GroupBy.Count != 0 + || body.Distinct + || body.Having is not null + || body.OrderBy.Count != 0 + || body.Limit is not null + || body.Offset is not null + || IsAggregateSelect(body) + || source.IndexDirective is not null) + { + return null; + } + + return body.Where is null + ? EstimateUnfilteredTableScan(source.Name, context) + : EstimateFilteredTableScan(source, body.Where, context); + } + private bool TryDescribeCorrelatedAggregateSubqueryPlan( SelectStatement statement, SqlValue[] parameters, @@ -98,6 +180,15 @@ private bool TryDescribeCorrelatedAggregateSubqueryPlan( PlanRow(1, 0, leftDetail), PlanRow(2, 0, rightDetail), }; + var (leftEstimate, rightEstimate) = EstimateEvaluatorLeftJoinSearches( + statement, + join, + leftPredicate, + leftPlan.Index, + leftConstraint, + rightPlan.Index, + $"{rightPlan.Index.Columns[0].Name}=?", + context); var rowOps = new List { new EqpJsonSearchOp( @@ -105,14 +196,16 @@ private bool TryDescribeCorrelatedAggregateSubqueryPlan( left.Alias, leftPlan.Index.Name, Covering: false, - [$"{leftConstraint}"]), + [$"{leftConstraint}"], + Estimate: leftEstimate), new EqpJsonSearchOp( right.Name, right.Alias, rightPlan.Index.Name, Covering: false, [$"{rightPlan.Index.Columns[0].Name}=?"], - Join: "left"), + Join: "left", + Estimate: rightEstimate), }; if (statement.OrderBy.Count != 0) { @@ -169,6 +262,36 @@ private bool TryDescribeCorrelatedAggregateSubqueryPlan( var scalarOp = scalarPlan is null ? (EqpJsonOp)new EqpJsonScanOp(scalarInner.Name, scalarInner.Alias, IndexName: null, Covering: false) : BuildIndexScanOp(scalarPlan, scalarQuery); + var listCovering = IndexCoversSelect(statement, outerTable, inIndex); + var listBodyEstimate = EstimateListSubqueryBody(inListQuery, listInner, context); + var inSeekEstimate = EstimateTursoInSubquerySeek( + listOuter, + outerTable, + inIndex, + listBodyEstimate?.OutputRows, + listCovering, + context); + if (scalarPlan is { Search: true } && scalarQuery.Where is { } scalarWhere) + { + // The correlated subquery runs once per outer row: its seek is priced with the + // outer loop's rows as input (optimizer/mod.rs plan_correlated_subqueries). + var correlation = new JoinTableSource(listOuter, scalarInner, scalarWhere, JoinKind.Semi); + if (PlanSemiOrAntiInnerAccess(correlation, leftPredicate: null, context, inputRowsOverride: inSeekEstimate.OutputRows) is + { + Kind: SemiAntiInnerAccessKind.DeclaredIndexSearch, + } scalarAccess + && ReferenceEquals(scalarAccess.Index, scalarPlan.Index) + && scalarAccess.SeekTerms.Count == IndexExpressionSemantics.SplitConjuncts(scalarWhere).Count + && scalarAccess.Covering == IndexCoversSelect(scalarQuery, scalarPlan.Table, scalarPlan.Index)) + { + scalarOp = WithEqpEstimate(scalarOp, new EqpJsonEstimate( + scalarAccess.InputRows, + scalarAccess.RowsPerOuterRow, + scalarAccess.InputRows * scalarAccess.RowsPerOuterRow, + scalarAccess.Cost, + scalarAccess.Cost)); + } + } result = new ExecutionResult( ExplainQueryPlanColumns(), @@ -183,14 +306,15 @@ private bool TryDescribeCorrelatedAggregateSubqueryPlan( ops = [ new EqpJsonListSubqueryOp(1, Correlated: false), - new EqpJsonScanOp(listInner.Name, listInner.Alias, IndexName: null, Covering: false), + new EqpJsonScanOp(listInner.Name, listInner.Alias, IndexName: null, Covering: false, Estimate: listBodyEstimate), new EqpJsonSearchOp( listOuter.Name, listOuter.Alias, inIndex.Name, - Covering: IndexCoversSelect(statement, outerTable, inIndex), + Covering: listCovering, [$"{inValue.UnqualifiedName ?? inValue.Name}=?"], - SearchKind: "in_seek"), + SearchKind: "in_seek", + Estimate: inSeekEstimate), new EqpJsonScalarSubqueryOp(2, Correlated: true), scalarOp, ]; @@ -222,6 +346,8 @@ private bool TryDescribeCorrelatedAggregateSubqueryPlan( var outerAlias = inOnlyOuter.Alias ?? inOnlyOuter.Name; var listAlias = inOnlyInner.Alias ?? inOnlyInner.Name; var hasOrderBy = statement.OrderBy.Count > 0; + var inOnlyCovering = IndexCoversSelect(statement, inOnlyOuterTable, inOnlyIndex); + var inOnlyBodyEstimate = EstimateListSubqueryBody(inOnlyListQuery, inOnlyInner, context); var rows = new List { PlanRow(1, 0, "LIST SUBQUERY 1"), @@ -231,14 +357,21 @@ private bool TryDescribeCorrelatedAggregateSubqueryPlan( var rowOps = new List { new EqpJsonListSubqueryOp(1, Correlated: false), - new EqpJsonScanOp(inOnlyInner.Name, inOnlyInner.Alias, IndexName: null, Covering: false), + new EqpJsonScanOp(inOnlyInner.Name, inOnlyInner.Alias, IndexName: null, Covering: false, Estimate: inOnlyBodyEstimate), new EqpJsonSearchOp( inOnlyOuter.Name, inOnlyOuter.Alias, inOnlyIndex.Name, - Covering: IndexCoversSelect(statement, inOnlyOuterTable, inOnlyIndex), + Covering: inOnlyCovering, [$"{inOnlyValue.UnqualifiedName ?? inOnlyValue.Name}=?"], - SearchKind: "in_seek"), + SearchKind: "in_seek", + Estimate: EstimateTursoInSubquerySeek( + inOnlyOuter, + inOnlyOuterTable, + inOnlyIndex, + inOnlyBodyEstimate?.OutputRows, + inOnlyCovering, + context)), }; if (hasOrderBy) { @@ -1059,10 +1192,15 @@ public override string ToJson() => } /// A coroutine-style read of a FROM-clause derived subquery. -internal sealed record EqpJsonSubqueryScanOp(int SubqueryId) : EqpJsonOp +internal sealed record EqpJsonSubqueryScanOp(int SubqueryId, EqpJsonEstimate? Estimate = null) : EqpJsonOp { - public override string ToJson() => - $"{{\"type\":\"scan\",\"table\":\"(subquery-{SubqueryId})\",\"subquery\":{{\"execution\":\"coroutine\"}},\"source\":\"subquery\"}}"; + public override string ToJson() + { + var json = new System.Text.StringBuilder( + $"{{\"type\":\"scan\",\"table\":\"(subquery-{SubqueryId})\",\"subquery\":{{\"execution\":\"coroutine\"}},\"source\":\"subquery\""); + Estimate?.AppendTo(json); + return json.Append('}').ToString(); + } } /// A compound query and one of its set-operation arms. @@ -1089,16 +1227,26 @@ internal sealed record EqpJsonRecursiveStepOp : EqpJsonOp public override string ToJson() => "{\"type\":\"recursive_step\"}"; } -internal sealed record EqpJsonRecursiveCteScanOp(string Table) : EqpJsonOp +internal sealed record EqpJsonRecursiveCteScanOp(string Table, EqpJsonEstimate? Estimate = null) : EqpJsonOp { - public override string ToJson() => - $"{{\"type\":\"scan\",\"table\":{EmbeddedDatabase.JsonEscape(Table)},\"subquery\":{{\"execution\":\"coroutine\",\"cte_id\":0,\"recursive\":true}},\"source\":\"subquery\"}}"; + public override string ToJson() + { + var json = new System.Text.StringBuilder( + $"{{\"type\":\"scan\",\"table\":{EmbeddedDatabase.JsonEscape(Table)},\"subquery\":{{\"execution\":\"coroutine\",\"cte_id\":0,\"recursive\":true}},\"source\":\"subquery\""); + Estimate?.AppendTo(json); + return json.Append('}').ToString(); + } } -internal sealed record EqpJsonRecursiveCteInputScanOp(string Table) : EqpJsonOp +internal sealed record EqpJsonRecursiveCteInputScanOp(string Table, EqpJsonEstimate? Estimate = null) : EqpJsonOp { - public override string ToJson() => - $"{{\"type\":\"scan\",\"table\":{EmbeddedDatabase.JsonEscape(Table)},\"source\":\"recursive_cte_input\"}}"; + public override string ToJson() + { + var json = new System.Text.StringBuilder( + $"{{\"type\":\"scan\",\"table\":{EmbeddedDatabase.JsonEscape(Table)},\"source\":\"recursive_cte_input\""); + Estimate?.AppendTo(json); + return json.Append('}').ToString(); + } } /// A join seek into a once-materialized common table expression. diff --git a/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs b/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs index 738ab23d..d14b8f0d 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.JoinOrderRewrites.cs @@ -1797,12 +1797,26 @@ private JoinPredicateTerm WithTursoTermEstimates( localSelectivity = EstimateTursoLocalSelectivity(conjunct, named, table, context); } + var constantOrdinal = -1; + string? constantCollation = null; + if (!term.IsEquality + && !term.CostOnly + && System.Numerics.BitOperations.PopCount(term.TableMask) == 1 + && TryDescribeConstantEquality(conjunct, infos, context, out var constantMember, out var ordinal, out var collation) + && term.TableMask == 1UL << constantMember) + { + constantOrdinal = ordinal; + constantCollation = collation; + } + return term with { TursoLeftSelectivity = leftSelectivity, TursoRightSelectivity = rightSelectivity, TursoLocalSelectivity = localSelectivity, WhereExtraSteps = CountTursoWhereSteps(conjunct) - 1, + ConstantEqualityOrdinal = constantOrdinal, + ConstantEqualityCollation = constantCollation, }; double MemberEqualitySelectivity(ulong mask, int ordinal) @@ -1816,6 +1830,105 @@ double MemberEqualitySelectivity(ulong mask, int ordinal) } } + /// + /// Recognizes column = literal (either side; a numeric literal may carry a sign) on + /// one join member whose comparison needs no affinity conversion of the literal — a + /// numeric literal against a numeric-affinity column, a text literal against a TEXT column, + /// or any literal against a BLOB-affinity column — so an index seek can use the literal as + /// its key verbatim. The comparison collation is the column's declared collation. + /// + private static bool TryDescribeConstantEquality( + Expression conjunct, + JoinOrderMemberInfo[] infos, + QueryContext context, + out int member, + out int ordinal, + out string collation) + { + member = -1; + ordinal = -1; + collation = "BINARY"; + if (!TryGetConstantEqualityOperands(conjunct, out var column, out var value) + || !TryResolveJoinOrderMember(column, infos, out member) + || infos[member].Table is not { } table + || TryResolveJoinOrderColumnDefinition(infos[member], column, context) is not { } definition + || !table.TryGetColumnIndex(definition.Name, out ordinal)) + { + return false; + } + + if (!IsVerbatimConstantSeekKey(definition, value)) + return false; + + collation = NormalizeDeclaredCollation(definition.Collation).ToUpperInvariant(); + return true; + } + + /// + /// Whether comparing 's column with the literal + /// applies no affinity conversion to the literal, so the literal + /// can be an index seek key as written. + /// + private static bool IsVerbatimConstantSeekKey(EmbeddedColumn definition, SqlValue value) + { + var affinity = GetJoinKeyAffinity(definition); + return affinity is null or ColumnAffinity.Blob + || IsNumericAffinity(affinity) && value.Kind is SqlValueKind.Integer or SqlValueKind.Real + || affinity == ColumnAffinity.Text && value.Kind == SqlValueKind.Text; + } + + /// + /// The bare column and literal value of column = literal / literal = column + /// with no explicit COLLATE; NULL and BLOB literals are not constant seek keys. + /// + private static bool TryGetConstantEqualityOperands( + Expression conjunct, + out ColumnExpression column, + out SqlValue value) + { + column = null!; + value = SqlValue.Null; + if (conjunct is not BinaryExpression { Operator: BinaryOperator.Equal } binary) + return false; + + if (binary.Left is ColumnExpression { BooleanKeyword: null } leftColumn && TryGetLiteral(binary.Right, out value)) + { + column = leftColumn; + return true; + } + + if (binary.Right is ColumnExpression { BooleanKeyword: null } rightColumn && TryGetLiteral(binary.Left, out value)) + { + column = rightColumn; + return true; + } + + return false; + + static bool TryGetLiteral(Expression expression, out SqlValue literal) + { + literal = SqlValue.Null; + switch (expression) + { + case LiteralExpression { Value.Kind: SqlValueKind.Integer or SqlValueKind.Real or SqlValueKind.Text } plain: + literal = plain.Value; + return true; + case UnaryExpression { Operator: UnaryOperator.Plus, Operand: LiteralExpression { Value.Kind: SqlValueKind.Integer or SqlValueKind.Real } positive }: + literal = positive.Value; + return true; + case UnaryExpression { Operator: UnaryOperator.Negate, Operand: LiteralExpression { Value.Kind: SqlValueKind.Real } negativeReal }: + literal = SqlValue.Real(-negativeReal.Value.AsReal()); + return true; + case UnaryExpression { Operator: UnaryOperator.Negate, Operand: LiteralExpression { Value.Kind: SqlValueKind.Integer } negativeInteger } + when negativeInteger.Value.AsInteger() != long.MinValue: + literal = SqlValue.Integer(-negativeInteger.Value.AsInteger()); + return true; + default: + return false; + } + } + } + private readonly record struct JoinOrderTermPlacement(Expression Expression, ulong Mask); private sealed record CompiledJoinIndexSelection( diff --git a/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs b/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs index ab90fdef..772837ef 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.SemiAntiJoinAccess.cs @@ -134,14 +134,20 @@ private static IReadOnlyList BuildConstraintTexts(IReadOnlyList /// Plans the inner access of one semi/anti join. is the /// part of the outer WHERE pushed onto the outer table (the same predicate - /// GetSemiOrAntiJoinRows receives); it only feeds the outer row estimate. + /// GetSemiOrAntiJoinRows receives); it only feeds the outer row estimate. The same + /// B-tree/temporary-index choice prices the null-supplying side of a two-table LEFT JOIN, + /// whose projection may read further inner columns (, + /// which decide whether an index covers them); no hash join is considered for it. /// private SemiAntiInnerAccess? PlanSemiOrAntiInnerAccess( JoinTableSource join, Expression? leftPredicate, - QueryContext context) + QueryContext context, + IReadOnlyCollection? otherInnerColumns = null, + double? inputRowsOverride = null) { - if (join.Kind is not (JoinKind.Semi or JoinKind.Anti) + if (join.Kind is not (JoinKind.Semi or JoinKind.Anti or JoinKind.Left) + || join.Kind == JoinKind.Left && join.Left is not NamedTableSource || join.Condition is null || context.ConcurrentMvStore is not null || !TryGetDecorrelationBaseTable(join.Right, context, out var inner, out var innerQualifier) @@ -205,12 +211,16 @@ private static IReadOnlyList BuildConstraintTexts(IReadOnlyList IsTursoEqualityOperator(c.Operator)), .. constraints.Where(static c => !IsTursoEqualityOperator(c.Operator))]; - var inputRows = EstimateTursoTableRows(outerLeaf.Name, context) - * EstimateTursoLocalSelectivity(leftPredicate, outerLeaf, outerTable, context); + var inputRows = inputRowsOverride + ?? EstimateTursoTableRows(outerLeaf.Name, context) + * EstimateTursoLocalSelectivity(leftPredicate, outerLeaf, outerTable, context); var innerRows = EstimateTursoTableRows(inner.Name, context); var covering = usedColumns.Count > 0; @@ -388,6 +398,374 @@ or TursoConstraintOperator.Greater return (baseRows * EstimateTursoLocalSelectivity(predicate, outer, table, context), cost); } + /// + /// The estimate of a first loop that searches a declared index with the table's own + /// constant constraints (): estimate_cost_for_scan_or_seek with + /// one input row plus the WHERE work of every term (access_method.rs:687-707), producing the + /// rows per seek reduced by the constraints the search leaves as filters. Returns + /// unless the ported seek terms are exactly + /// — the search the caller describes — and every term is + /// a modeled column constraint. + /// + private EqpJsonEstimate? EstimateTursoFirstLoopIndexSearch( + NamedTableSource source, + EmbeddedIndex index, + Expression predicate, + IReadOnlyCollection usedColumns, + IReadOnlyList constraintTexts, + QueryContext context) + { + if (!context.Tables.TryGetValue(source.Name, out var table) || index.IsPartial || index.IsMethodIndex) + return null; + + var qualifier = source.Alias ?? source.Name; + var columns = new HashSet(table.Columns, StringComparer.OrdinalIgnoreCase); + var conjuncts = IndexExpressionSemantics.SplitConjuncts(predicate); + var steps = new int[conjuncts.Count]; + var constraints = new List(); + for (var position = 0; position < conjuncts.Count; position++) + { + steps[position] = CountTursoWhereSteps(conjuncts[position]) - 1; + if (!TryDescribeSemiAntiConstraint(position, conjuncts[position], table, qualifier, columns, source, table, context, out var constraint) + || constraint.DependsOnOuter) + { + return null; + } + + constraints.Add(constraint); + } + + constraints = [.. constraints.Where(static c => IsTursoEqualityOperator(c.Operator)), + .. constraints.Where(static c => !IsTursoEqualityOperator(c.Operator))]; + var terms = BuildSemiAntiIndexSeekTerms(index, table, constraints); + if (terms.Count == 0) + return null; + + var covering = index.Columns.All(static column => !column.IsExpression) + && usedColumns.All(ordinal => ordinal == table.RowidAliasColumnIndex + || index.Columns.Any(column => column.ColumnIndex == ordinal)); + var info = new TursoIndexInfo( + index.Unique, + index.Columns.Count, + covering, + TursoCostModel.IndexLeafRowsPerPage(index.Columns.Count, table.Columns.Length, table.HasRowidAlias)); + var rows = EstimateTursoTableRows(source.Name, context); + var cost = EstimateTursoScanOrSeekCost(info, terms, 1.0, rows, table, index, context); + var rowsPerSeek = EstimateTursoRowsPerSeek(info, terms, rows, table, index, context); + var access = new SemiAntiInnerAccess( + SemiAntiInnerAccessKind.DeclaredIndexSearch, + source, + table, + index, + terms, + HashKeys: [], + covering, + InputRows: 1.0, + rowsPerSeek, + cost); + if (!access.ConstraintTexts.SequenceEqual(constraintTexts, StringComparer.OrdinalIgnoreCase)) + return null; + + // constraint_output_multipliers: the constraints the search does not consume still + // filter its rows, a column bounded on both sides as one closed range. + var remaining = 1.0; + var bounds = new Dictionary(); + foreach (var constraint in constraints) + { + if (terms.Any(term => ReferenceEquals(term.Equality, constraint) + || ReferenceEquals(term.Lower, constraint) + || ReferenceEquals(term.Upper, constraint))) + { + continue; + } + + remaining *= constraint.Selectivity; + var isLower = constraint.Operator is TursoConstraintOperator.Greater or TursoConstraintOperator.GreaterOrEqual; + var isUpper = constraint.Operator is TursoConstraintOperator.Less or TursoConstraintOperator.LessOrEqual; + if (isLower || isUpper) + { + bounds.TryGetValue(constraint.ColumnOrdinal, out var bound); + bounds[constraint.ColumnOrdinal] = (bound.Lower || isLower, bound.Upper || isUpper); + } + } + + foreach (var bound in bounds.Values) + { + if (bound.Lower && bound.Upper) + remaining *= TursoCostParams.ClosedRangeSelectivityFactor; + } + + var accessCost = cost + EstimateSemiAntiWhereWork(access, steps, 1.0); + var output = rowsPerSeek * remaining; + return new EqpJsonEstimate(1, output, output, accessCost, accessCost); + } + + /// + /// The estimate of a first loop that reads a whole declared index in key order (a SCAN … + /// USING INDEX serving ORDER BY or GROUP BY): estimate_cost_for_scan_or_seek with no seek + /// terms plus the WHERE work of every filter over each visited row, producing the rows the + /// table's own column constraints keep. Returns for a filter shape + /// the ported constraint model does not describe. + /// + private EqpJsonEstimate? EstimateTursoFirstLoopIndexScan( + NamedTableSource source, + EmbeddedIndex index, + Expression? predicate, + IReadOnlyCollection usedColumns, + bool isIndexOrdered, + QueryContext context) + { + if (!context.Tables.TryGetValue(source.Name, out var table) || index.IsPartial || index.IsMethodIndex) + return null; + + var steps = 0; + if (predicate is not null) + { + if (ContainsSubqueryExpression(predicate) || GetImpliedOrInFilters(predicate, source, table).Count != 0) + return null; + var qualifier = source.Alias ?? source.Name; + var columns = new HashSet(table.Columns, StringComparer.OrdinalIgnoreCase); + foreach (var conjunct in IndexExpressionSemantics.SplitConjuncts(predicate)) + { + steps += CountTursoWhereSteps(conjunct) - 1; + if (conjunct is BinaryExpression { Operator: BinaryOperator.Or }) + continue; + if (!TryDescribeSemiAntiConstraint(0, conjunct, table, qualifier, columns, source, table, context, out var constraint) + || constraint.DependsOnOuter) + { + return null; + } + } + } + + var covering = index.Columns.All(static column => !column.IsExpression) + && usedColumns.All(ordinal => ordinal == table.RowidAliasColumnIndex + || index.Columns.Any(column => column.ColumnIndex == ordinal)); + var info = new TursoIndexInfo( + index.Unique, + index.Columns.Count, + covering, + TursoCostModel.IndexLeafRowsPerPage(index.Columns.Count, table.Columns.Length, table.HasRowidAlias)); + var rows = EstimateTursoTableRows(source.Name, context); + var cost = EstimateTursoScanOrSeekCost(info, [], 1.0, rows, table, index, context, isIndexOrdered) + + TursoCostModel.EstimateWhereWork(1.0, rows, consumedSteps: 0, steps); + var output = rows * EstimateTursoLocalSelectivity(predicate, source, table, context); + return new EqpJsonEstimate(1, output, output, cost, cost); + } + + /// + /// The estimate of a multi-index OR union that is the whole WHERE clause + /// (multi_index.rs:195-235 estimate_multi_index_scan_cost with its branches costed by + /// choose_multi_index_branch_access): every branch seeks its constraints with a + /// rowid-only (covering) read, the union deduplicates rowids and fetches the distinct + /// rows, and the consumed OR term's WHERE work is charged per output row. A branch with + /// residual conjuncts, or a WHERE with other terms, reports no estimate. + /// + private EqpJsonEstimate? EstimateTursoMultiIndexOrUnion( + NamedTableSource source, + EmbeddedTable table, + Expression where, + IReadOnlyList<(EmbeddedIndex? Index, Expression Predicate)> branches, + QueryContext context) + { + if (IndexExpressionSemantics.SplitConjuncts(where).Count != 1 + || where is not BinaryExpression { Operator: BinaryOperator.Or } + || ContainsSubqueryExpression(where)) + { + return null; + } + + var qualifier = source.Alias ?? source.Name; + var columns = new HashSet(table.Columns, StringComparer.OrdinalIgnoreCase); + var rows = EstimateTursoTableRows(source.Name, context); + var treeDepth = TursoCostModel.EstimateBtreeDepth(rows, JoinCostParams.RowsPerTablePage); + var branchCosts = new List(branches.Count); + var branchRows = new List(branches.Count); + foreach (var (index, predicate) in branches) + { + var constraints = new List(); + var conjuncts = IndexExpressionSemantics.SplitConjuncts(predicate); + for (var position = 0; position < conjuncts.Count; position++) + { + if (!TryDescribeSemiAntiConstraint(position, conjuncts[position], table, qualifier, columns, source, table, context, out var constraint) + || constraint.DependsOnOuter) + { + return null; + } + + constraints.Add(constraint); + } + + if (index is null) + { + // The rowid branch: a unique point lookup on the table b-tree + // (index_info_for_branch with no index). + if (constraints is not [{ Operator: TursoConstraintOperator.Equal } rowid] + || rowid.ColumnOrdinal != table.RowidAliasColumnIndex + || !table.HasRowidAlias) + { + return null; + } + + branchCosts.Add(TursoCostModel.EstimateIndexCost( + rows, + treeDepth, + new TursoIndexInfo(Unique: true, ColumnCount: 1, Covering: true, JoinCostParams.RowsPerTablePage), + 1.0, + 1.0)); + branchRows.Add(1.0); + continue; + } + + constraints = [.. constraints.Where(static c => IsTursoEqualityOperator(c.Operator)), + .. constraints.Where(static c => !IsTursoEqualityOperator(c.Operator))]; + var terms = BuildSemiAntiIndexSeekTerms(index, table, constraints); + var consumed = terms.Sum(static term => term.Equality is not null ? 1 : (term.Lower is null ? 0 : 1) + (term.Upper is null ? 0 : 1)); + if (terms.Count == 0 || consumed != constraints.Count) + return null; + + var info = new TursoIndexInfo( + index.Unique, + index.Columns.Count, + Covering: true, + TursoCostModel.IndexLeafRowsPerPage(index.Columns.Count, table.Columns.Length, table.HasRowidAlias)); + branchCosts.Add(EstimateTursoScanOrSeekCost(info, terms, 1.0, rows, table, index, context)); + branchRows.Add(EstimateTursoRowsPerSeek(info, terms, rows, table, index, context)); + } + + var uniqueRatio = 1.0; + foreach (var branch in branchRows) + uniqueRatio *= 1.0 - Math.Min(branch / rows, 1.0); + var uniqueRows = rows * (1.0 - uniqueRatio); + var rowsetCost = branchRows.Sum() * JoinCostParams.CpuCostPerRow * 2.0; + var tablePages = Math.Max(rows / JoinCostParams.RowsPerTablePage, 1.0); + var fetchCost = uniqueRows / Math.Max(rows, 1.0) * tablePages; + var cost = branchCosts.Sum() + rowsetCost + fetchCost + + TursoCostModel.EstimateWhereWork(1.0, uniqueRows, CountTursoWhereSteps(where) - 1, remainingSteps: 0); + return new EqpJsonEstimate(1, uniqueRows, uniqueRows, cost, cost); + } + + /// + /// The estimate of a first loop driven by an uncorrelated column IN (SELECT …) seek + /// (access_method.rs:505-645 choose_best_in_seek_candidate, constraints.rs:1068-1160): the + /// list contributes its body's estimated rows, capped at the square root of the table's + /// rows (Turso's in_subquery_rows when the body has no estimate), one index seek per value, + /// and the rows that selectivity keeps. + /// + private static EqpJsonEstimate EstimateTursoInSubquerySeek( + NamedTableSource source, + EmbeddedTable table, + EmbeddedIndex index, + double? listRows, + bool covering, + QueryContext context) + { + var rows = EstimateTursoTableRows(source.Name, context); + var values = listRows is { } planned + ? Math.Clamp(planned, 0.0, Math.Max(Math.Sqrt(rows), 1.0)) + : Math.Min(TursoCostParams.InSubqueryRows, rows); + var selectivity = Math.Min(values / rows, 1.0); + var info = new TursoIndexInfo( + index.Unique, + index.Columns.Count, + covering, + TursoCostModel.IndexLeafRowsPerPage(index.Columns.Count, table.Columns.Length, table.HasRowidAlias)); + var rowsPerSeek = index.Unique && index.Columns.Count == 1 + ? 1.0 + : Math.Max(Math.Sqrt(rows * JoinCostParams.SelectivityEqualityIndexed), 1.0); + var cost = TursoCostModel.EstimateIndexCost( + rows, + TursoCostModel.EstimateBtreeDepth(rows, JoinCostParams.RowsPerTablePage), + info, + values, + rowsPerSeek); + var output = Math.Max(selectivity * rows, 1.0); + return new EqpJsonEstimate(1, output, output, cost, cost); + } + + /// + /// The ordinals of every column of the statement reads, the input + /// to Turso's covering-index test. Returns for a * + /// projection, a subquery, or an unqualified reference that could name this table. + /// + private static bool TryCollectReferencedTableColumns( + SelectStatement select, + NamedTableSource source, + EmbeddedTable table, + out HashSet ordinals) + { + var found = new HashSet(); + ordinals = found; + var qualifier = source.Alias ?? source.Name; + foreach (var projection in select.Projections) + { + // * reads every column of every source; q.* every column of q. + if (projection.Expression is StarExpression + || projection.Expression is QualifiedStarExpression qualified + && string.Equals(qualified.Qualifier, qualifier, StringComparison.OrdinalIgnoreCase)) + { + for (var ordinal = 0; ordinal < table.Columns.Length; ordinal++) + found.Add(ordinal); + } + } + + var roots = new List(); + roots.AddRange(select.Projections.Select(static projection => projection.Expression)); + if (select.Where is not null) + roots.Add(select.Where); + roots.AddRange(select.GroupBy); + if (select.Having is not null) + roots.Add(select.Having); + roots.AddRange(select.OrderBy.Select(static term => term.Expression)); + var pendingSources = new Stack(); + pendingSources.Push(select.Source); + while (pendingSources.Count > 0) + { + if (pendingSources.Pop() is JoinTableSource join) + { + if (join.Condition is not null) + roots.Add(join.Condition); + pendingSources.Push(join.Left); + pendingSources.Push(join.Right); + } + } + + // In a single-table statement an unqualified name can only mean this table. + var soleSource = ReferenceEquals(select.Source, source); + foreach (var root in roots) + { + if (ContainsSubqueryExpression(root)) + return false; + if (!ForEachColumnReference(root, column => + { + var name = column.UnqualifiedName ?? column.Name; + if (column.Qualifier is null && soleSource) + { + if (table.TryGetColumnIndex(name, out var soleOrdinal)) + found.Add(soleOrdinal); + return true; + } + + if (column.Qualifier is null) + return !table.TryGetColumnIndex(name, out _) && !IsRowIdAlias(name); + if (!string.Equals(column.Qualifier, qualifier, StringComparison.OrdinalIgnoreCase)) + return true; + if (table.TryGetColumnIndex(name, out var ordinal)) + { + found.Add(ordinal); + return true; + } + + return IsRowIdAlias(name); + })) + { + return false; + } + } + + return true; + } + /// /// The left anti hash join alternative (join.rs:586-1000 with access_method.rs:1294-1537), /// restricted to what the managed executor runs: the build side is the plain outer table @@ -945,7 +1323,8 @@ private static double EstimateTursoScanOrSeekCost( double baseRows, EmbeddedTable table, EmbeddedIndex index, - QueryContext context) + QueryContext context, + bool isIndexOrdered = false) { var treeDepth = TursoCostModel.EstimateBtreeDepth(baseRows, JoinCostParams.RowsPerTablePage); if (IsTursoUniquePointLookup(info, terms)) @@ -953,7 +1332,8 @@ private static double EstimateTursoScanOrSeekCost( var rowsPerSeek = EstimateTursoRowsPerSeek(info, terms, baseRows, table, index, context); var cost = TursoCostModel.EstimateIndexCost(baseRows, treeDepth, info, inputRows, rowsPerSeek); - return !info.Covering && terms.Count == 0 ? cost * 2.0 : cost; + // A non-covering full index scan that no ORDER BY needs pays for its table lookups. + return !info.Covering && terms.Count == 0 && !isIndexOrdered ? cost * 2.0 : cost; } /// cost.rs:252-264 is_unique_point_lookup. @@ -1108,6 +1488,7 @@ private double EstimateTursoLocalSelectivity( var qualifier = source.Alias ?? source.Name; var columns = new HashSet(table.Columns, StringComparer.OrdinalIgnoreCase); var selectivity = 1.0; + var bounds = new Dictionary(); foreach (var conjunct in IndexExpressionSemantics.SplitConjuncts(predicate)) { if (TryDescribeSemiAntiConstraint( @@ -1123,9 +1504,23 @@ private double EstimateTursoLocalSelectivity( && !constraint.DependsOnOuter) { selectivity *= constraint.Selectivity; + var isLower = constraint.Operator is TursoConstraintOperator.Greater or TursoConstraintOperator.GreaterOrEqual; + var isUpper = constraint.Operator is TursoConstraintOperator.Less or TursoConstraintOperator.LessOrEqual; + if (isLower || isUpper) + { + bounds.TryGetValue(constraint.ColumnOrdinal, out var bound); + bounds[constraint.ColumnOrdinal] = (bound.Lower || isLower, bound.Upper || isUpper); + } } } + // A column bounded from both sides is a closed range (join.rs:141-147). + foreach (var bound in bounds.Values) + { + if (bound.Lower && bound.Upper) + selectivity *= TursoCostParams.ClosedRangeSelectivityFactor; + } + return Math.Clamp(selectivity, 0.0, 1.0); } } diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index fe7f51e1..cc9758c9 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -22740,9 +22740,34 @@ private bool TryCreateCompiledJoinIndexScanPlan( // matched against the persisted index expression back in TryCreateJoinOrderTerm. Exactly // one of keys[position]/expressionOuterOrdinal[position] is populated per position. var expressionOuterOrdinal = new int?[selection.EqualityTerms.Count]; + // Set only for a position bound by a column = literal constant of the right table: + // the literal, used verbatim as that key column's seek value. + var constantKeys = new SqlValue?[selection.EqualityTerms.Count]; for (var position = 0; position < keys.Length; position++) { var candidateColumn = selection.Candidate.Columns[position]; + if (candidateColumn.IndexExpression is null + && !selection.Candidate.Automatic + && TryGetConstantEqualityOperands(selection.EqualityTerms[position], out var constantColumn, out var constantValue) + && ResolveJoinSideColumn(constantColumn, right.OutputColumns) is { } constantRightColumn + && ResolveJoinSideColumn(constantColumn, left.OutputColumns) is null) + { + var constantDefinition = GetOutputColumnDefinition(join.Right, constantRightColumn, context); + if (constantRightColumn.Index != candidateColumn.ColumnOrdinal + || constantDefinition is null + || !IsVerbatimConstantSeekKey(constantDefinition, constantValue) + || !string.Equals( + NormalizeDeclaredCollation(constantDefinition.Collation), + candidateColumn.Collation, + StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + constantKeys[position] = constantValue; + continue; + } + if (candidateColumn.IndexExpression is { } indexExpression) { // The enumerator only binds this candidate column to a term shaped as an equality @@ -22868,6 +22893,12 @@ private bool TryCreateCompiledJoinIndexScanPlan( var result = new SqlValue[keys.Length]; for (var position = 0; position < keys.Length; position++) { + if (constantKeys[position] is { } constantKey) + { + result[position] = constantKey; + continue; + } + if (expressionOuterOrdinal[position] is { } ordinal) { if (ordinal < 0 || ordinal >= outer.Values.Length) @@ -29168,7 +29199,7 @@ private ExecutionResult ExecuteExplainQueryPlan( /// or other named row source as a table. Joins cannot be reconstructed from this syntax; /// supported compiled joins instead use their actual OpenJoinCursor plan tree. /// - private static (string Detail, EqpJsonOp Op)? TryDescribeGenuinePlaceholderAccessPath( + private (string Detail, EqpJsonOp Op)? TryDescribeGenuinePlaceholderAccessPath( SelectStatement select, QueryContext context) => select.Source switch @@ -29185,7 +29216,9 @@ when context.Tables.ContainsKey(source.Name) source.Alias, IndexName: null, Covering: false, - Estimate: select.Where is null ? EstimateUnfilteredTableScan(source.Name, context) : null)), + Estimate: select.Where is null + ? EstimateUnfilteredTableScan(source.Name, context) + : EstimateFilteredTableScan(source, select.Where, context))), _ => null, }; @@ -29193,8 +29226,7 @@ when context.Tables.ContainsKey(source.Name) /// The estimate of a lone, unfiltered full table scan: one input row, every table row out, /// priced by the ported Turso scan formula (optimizer/cost.rs estimate_scan_cost). The row /// count is the planner's current sqlite_stat1 figure, or Turso's 1,000,000-row - /// default for an unanalyzed table. A filtered scan needs Turso's per-constraint selectivity - /// model, which is not ported, so it reports no estimate rather than an invented one. + /// default for an unanalyzed table. /// private static EqpJsonEstimate EstimateUnfilteredTableScan(string tableName, QueryContext context) { @@ -29205,6 +29237,238 @@ private static EqpJsonEstimate EstimateUnfilteredTableScan(string tableName, Que return new EqpJsonEstimate(1, rows, rows, cost, cost); } + /// + /// The estimate of a lone full table scan filtered by : one input + /// row, every table row visited, priced by estimate_scan_cost plus the WHERE work of + /// every ready term (access_method.rs:687-707 cost_with_where_work), producing the rows the + /// table's own column constraints keep (join.rs:72-152 constraint_output_multipliers). An + /// OR, a subquery or any other term that is not a column constraint visits rows without + /// reducing the estimate, as in Turso. Any other term shape (BETWEEN, which Turso first + /// rewrites into two range terms, LIKE, IN, subqueries, …) reports no estimate rather than + /// an approximated one. + /// + private EqpJsonEstimate? EstimateFilteredTableScan( + NamedTableSource source, + Expression where, + QueryContext context) + { + // An OR-implied IN filter (lift_common_subexpressions.rs) would itself be a constraint. + if (ContainsSubqueryExpression(where) + || !context.Tables.TryGetValue(source.Name, out var table) + || GetImpliedOrInFilters(where, source, table).Count != 0) + { + return null; + } + + var qualifier = source.Alias ?? source.Name; + var columns = new HashSet(table.Columns, StringComparer.OrdinalIgnoreCase); + foreach (var conjunct in IndexExpressionSemantics.SplitConjuncts(where)) + { + if (conjunct is BinaryExpression { Operator: BinaryOperator.Or }) + continue; + if (!TryDescribeSemiAntiConstraint(0, conjunct, table, qualifier, columns, source, table, context, out var constraint) + || constraint.DependsOnOuter) + { + return null; + } + + // A constraint Turso could seek (a rowid or a leading index key) means its plan + // would not be this full scan; the scan estimate would describe a plan Turso never + // picks, so none is reported. + if (constraint.Operator is not (TursoConstraintOperator.NotEqual or TursoConstraintOperator.IsNot) + && (constraint.ColumnOrdinal == table.RowidAliasColumnIndex + || table.Indexes.Any(index => !index.IsMethodIndex + && index.Columns.Count > 0 + && !index.Columns[0].IsExpression + && index.Columns[0].ColumnIndex == constraint.ColumnOrdinal))) + { + return null; + } + } + + var (rows, cost) = EstimateSemiAntiOuterScan(source, where, context); + return new EqpJsonEstimate(1, rows, rows, cost, cost); + } + + /// + /// Turso's first-loop estimate for a single-table index plan: the search of its seek + /// constraints, or the ordered scan of the whole index. + /// + private EqpJsonEstimate? EstimateSingleTableIndexPlan( + SelectStatement select, + ManagedIndexScanPlan plan, + string constraint, + QueryContext context) + { + if (select.Source is not NamedTableSource source + || !TryCollectReferencedTableColumns(select, source, plan.Table, out var usedColumns)) + { + return null; + } + + return plan.Search + ? select.Where is { } where + ? EstimateTursoFirstLoopIndexSearch( + source, + plan.Index, + where, + usedColumns, + constraint.Split(" AND ", StringSplitOptions.TrimEntries), + context) + : null + : EstimateTursoFirstLoopIndexScan( + source, + plan.Index, + select.Where, + usedColumns, + IsOrderedByIndexPrefix(select, source, plan.Table, plan.Index), + context); + } + + /// Attaches to a single-table scan or search op. + private static EqpJsonOp WithEqpEstimate(EqpJsonOp op, EqpJsonEstimate? estimate) + => (op, estimate) switch + { + (_, null) => op, + (EqpJsonSearchOp search, _) => search with { Estimate = estimate }, + (EqpJsonScanOp scan, _) => scan with { Estimate = estimate }, + _ => op, + }; + + /// + /// The coroutine scan of a FROM-clause subquery that is the statement's only source + /// (access_method.rs find_best_access_method_for_subquery): estimate_scan_cost over + /// the body's estimated rows plus one run of the body. Only a body whose output rows are + /// its single table loop's rows (no grouping, DISTINCT, ordering, LIMIT or aggregate) and + /// an outer query that adds no filter are described. + /// + private EqpJsonEstimate? EstimateCoroutineSubqueryScan( + SelectStatement? outer, + SelectStatement body, + EqpJsonEstimate? bodyEstimate) + { + if (bodyEstimate is null + || outer is not { Where: null } + || body.GroupBy.Count != 0 + || body.Distinct + || body.Having is not null + || body.OrderBy.Count != 0 + || body.Limit is not null + || body.Offset is not null + || IsAggregateSelect(body)) + { + return null; + } + + var rows = Math.Max(bodyEstimate.OutputRows, 1.0); + var cost = TursoCostModel.EstimateScanCost(rows, 1.0) + bodyEstimate.TotalCost; + return new EqpJsonEstimate(1, rows, rows, cost, cost); + } + + /// + /// The recursive step's read of the previous iteration's rows: a one-row full scan + /// (access_method.rs:790-804) kept by its filters, each a comparison of a CTE column with a + /// constant sized by Turso's column-constraint selectivity (no index, no statistics). + /// Returns for any other step shape. + /// + private static EqpJsonEstimate? EstimateRecursiveCteInputScan(SelectStatement step, string cteName) + { + if (step.Source is not NamedTableSource input + || !string.Equals(input.Name, cteName, StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + var selectivity = 1.0; + var steps = 0; + var bounds = new Dictionary(StringComparer.OrdinalIgnoreCase); + if (step.Where is not null) + { + foreach (var conjunct in IndexExpressionSemantics.SplitConjuncts(step.Where)) + { + if (conjunct is not BinaryExpression binary + || !(binary.Left is ColumnExpression { BooleanKeyword: null } && binary.Right is LiteralExpression + || binary.Right is ColumnExpression { BooleanKeyword: null } && binary.Left is LiteralExpression)) + { + return null; + } + + var column = (ColumnExpression)(binary.Left is ColumnExpression ? binary.Left : binary.Right); + var columnOnLeft = binary.Left is ColumnExpression; + var nullLiteral = (binary.Left is LiteralExpression left ? left : (LiteralExpression)binary.Right).Value.Kind == SqlValueKind.Null; + double termSelectivity; + bool isLower = false, isUpper = false; + switch (binary.Operator) + { + case BinaryOperator.Equal: + termSelectivity = JoinCostParams.SelectivityEqualityUnindexed; + break; + case BinaryOperator.Is: + termSelectivity = nullLiteral ? TursoCostParams.SelectivityIsNull : JoinCostParams.SelectivityEqualityUnindexed; + break; + case BinaryOperator.IsNot: + termSelectivity = TursoCostParams.SelectivityIsNotNull; + break; + case BinaryOperator.NotEqual: + termSelectivity = JoinCostParams.SelectivityOther; + break; + case BinaryOperator.LessThan or BinaryOperator.LessThanOrEqual: + termSelectivity = JoinCostParams.SelectivityRange; + isUpper = columnOnLeft; + isLower = !columnOnLeft; + break; + case BinaryOperator.GreaterThan or BinaryOperator.GreaterThanOrEqual: + termSelectivity = JoinCostParams.SelectivityRange; + isLower = columnOnLeft; + isUpper = !columnOnLeft; + break; + default: + return null; + } + + selectivity *= termSelectivity; + steps += CountTursoWhereSteps(conjunct) - 1; + if (isLower || isUpper) + { + var name = column.UnqualifiedName ?? column.Name; + bounds.TryGetValue(name, out var bound); + bounds[name] = (bound.Lower || isLower, bound.Upper || isUpper); + } + } + } + + foreach (var bound in bounds.Values) + { + if (bound.Lower && bound.Upper) + selectivity *= TursoCostParams.ClosedRangeSelectivityFactor; + } + + var cost = TursoCostModel.EstimateScanCost(1.0, 1.0) + + TursoCostModel.EstimateWhereWork(1.0, 1.0, consumedSteps: 0, steps); + return new EqpJsonEstimate(1, selectivity, selectivity, cost, cost); + } + + /// + /// Whether the statement's ORDER BY (or, without one, its GROUP BY) starts with the index's + /// leading plain column, so reading the index in key order serves it (Turso's + /// is_index_ordered). + /// + private static bool IsOrderedByIndexPrefix( + SelectStatement select, + NamedTableSource source, + EmbeddedTable table, + EmbeddedIndex index) + { + var first = select.OrderBy.Count != 0 + ? select.OrderBy[0].Expression + : select.GroupBy.Count != 0 ? select.GroupBy[0] : null; + return first is not null + && index.Columns.Count > 0 + && !index.Columns[0].IsExpression + && TryResolvePlainTableColumn(first, source, table, out var ordinal) + && ordinal == index.Columns[0].ColumnIndex; + } + internal static string JsonEscape(string value) { var escaped = new System.Text.StringBuilder(value.Length + 2); @@ -29364,6 +29628,8 @@ private ExecutionResult ExecuteExplainQueryPlanText( && TryPlanManagedIndexScan(plannedSelect, compilationContext) is { } indexPlan) { var indexPlanCovering = IndexCoversSelect(plannedSelect, indexPlan.Table, indexPlan.Index); + var indexPlanConstraint = indexPlan.SearchConstraint ?? $"{indexPlan.Index.Columns[0].Name}=?"; + var indexPlanEstimate = EstimateSingleTableIndexPlan(plannedSelect, indexPlan, indexPlanConstraint, compilationContext); // The plan carries the exact leading-key predicate chosen by the planner, so JSON // can share TEXT's equality/range constraint without re-parsing rendered detail. ops = indexPlan.Search @@ -29373,14 +29639,16 @@ private ExecutionResult ExecuteExplainQueryPlanText( indexPlan.Source.Alias, indexPlan.Index.Name, indexPlanCovering, - [indexPlan.SearchConstraint ?? $"{indexPlan.Index.Columns[0].Name}=?"]), + [indexPlanConstraint], + Estimate: indexPlanEstimate), ] : [ new EqpJsonScanOp( indexPlan.Table.Name, indexPlan.Source.Alias, indexPlan.Index.Name, - indexPlanCovering), + indexPlanCovering, + Estimate: indexPlanEstimate), ]; return new ExecutionResult( ExplainQueryPlanColumns(), @@ -29409,10 +29677,15 @@ private ExecutionResult ExecuteExplainQueryPlanText( } && TryPlanManagedIndexScan(derivedSelect, compilationContext) is { } derivedIndexPlan) { + var derivedEstimate = EstimateSingleTableIndexPlan( + derivedSelect, + derivedIndexPlan, + derivedIndexPlan.SearchConstraint ?? $"{derivedIndexPlan.Index.Columns[0].Name}=?", + compilationContext); ops = [ - new EqpJsonSubqueryScanOp(0), - BuildIndexScanOp(derivedIndexPlan, derivedSelect), + new EqpJsonSubqueryScanOp(0, EstimateCoroutineSubqueryScan(statement.Inner as SelectStatement, derivedSelect, derivedEstimate)), + WithEqpEstimate(BuildIndexScanOp(derivedIndexPlan, derivedSelect), derivedEstimate), new EqpJsonGroupByOp(), new EqpJsonOrderByOp(), ]; @@ -29458,13 +29731,22 @@ private ExecutionResult ExecuteExplainQueryPlanText( } && TryPlanManagedIndexScan(firstTerm, compilationContext) is { } firstIndexPlan) { + // Each arm is planned on its own, from one input row (optimizer/mod.rs). + var firstArmEstimate = EstimateSingleTableIndexPlan( + firstTerm, + firstIndexPlan, + firstIndexPlan.SearchConstraint ?? $"{firstIndexPlan.Index.Columns[0].Name}=?", + compilationContext); + var secondArmEstimate = secondTerm.Where is null + ? EstimateUnfilteredTableScan(secondSource.Name, compilationContext) + : EstimateFilteredTableScan(secondSource, secondTerm.Where, compilationContext); ops = [ new EqpJsonCompoundOp(), new EqpJsonCompoundArmOp("left_most", TempBtree: false), - BuildIndexScanOp(firstIndexPlan, firstTerm), + WithEqpEstimate(BuildIndexScanOp(firstIndexPlan, firstTerm), firstArmEstimate), new EqpJsonCompoundArmOp("union", TempBtree: true), - new EqpJsonScanOp(secondSource.Name, secondSource.Alias, IndexName: null, Covering: false), + new EqpJsonScanOp(secondSource.Name, secondSource.Alias, IndexName: null, Covering: false, Estimate: secondArmEstimate), new EqpJsonOrderByOp(), ]; return new ExecutionResult( @@ -29555,19 +29837,31 @@ private ExecutionResult ExecuteExplainQueryPlanText( }, }, ], - Query: SelectStatement { Source: NamedTableSource { Name: var outerName } }, + Query: SelectStatement { Source: NamedTableSource { Name: var outerName } } recursiveOuter, } && string.Equals(recursiveName, outerName, StringComparison.OrdinalIgnoreCase) && !SelectContainsRegisteredScalarFunction(anchor) && !SelectContainsRegisteredScalarFunction(recursiveStep)) { + // A recursive CTE has no row estimate, so its coroutine scan is sized by Turso's + // 1,000,000-row fallback with no body cost (access_method.rs + // find_best_access_method_for_subquery); the recursive step reads a one-row input + // (access_method.rs:790-804). + var recursiveScanEstimate = recursiveOuter.Where is null + ? new EqpJsonEstimate( + 1, + JoinCostParams.RowsPerTableFallback, + JoinCostParams.RowsPerTableFallback, + TursoCostModel.EstimateScanCost(JoinCostParams.RowsPerTableFallback, 1.0), + TursoCostModel.EstimateScanCost(JoinCostParams.RowsPerTableFallback, 1.0)) + : null; ops = [ - new EqpJsonRecursiveCteScanOp(recursiveName), + new EqpJsonRecursiveCteScanOp(recursiveName, recursiveScanEstimate), new EqpJsonRecursiveSetupOp(), new EqpJsonConstantRowOp(), new EqpJsonRecursiveStepOp(), - new EqpJsonRecursiveCteInputScanOp(recursiveName), + new EqpJsonRecursiveCteInputScanOp(recursiveName, EstimateRecursiveCteInputScan(recursiveStep, recursiveName)), ]; return new ExecutionResult( ExplainQueryPlanColumns(), @@ -29631,7 +29925,15 @@ private ExecutionResult ExecuteExplainQueryPlanText( new EqpJsonMultiIndexOp( orUnionPlan.Table.Name, orUnionPlan.Branches.Select(static branch => branch.Name).ToArray(), - Alias: orUnionPlan.Source.Alias), + Alias: orUnionPlan.Source.Alias, + Estimate: orUnionSelect.Where is { } orUnionWhere + ? EstimateTursoMultiIndexOrUnion( + orUnionPlan.Source, + orUnionPlan.Table, + orUnionWhere, + orUnionPlan.Branches.Select(static branch => (branch.Index, branch.Predicate)).ToArray(), + compilationContext) + : null), ]; return new ExecutionResult( ExplainQueryPlanColumns(), diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index 20ca8d71..54603267 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -180,7 +180,6 @@ join/memory.sqltest::partial-stats-hash-join-uses-known-unique-key-count | datab join/memory.sqltest::range-prefix-still-allows-hash-join | database variant 1/1 (:memory:): output does not match pattern 'HASH JOIN prefix_child AS child' join/memory.sqltest::unique-composite-hash-key-uses-one-key-estimate | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN composite_hash_child AS child".*"rows_per_input":5' join/memory.sqltest::unique-hash-key-does-not-apply-to-a-different-collation | database variant 1/1 (:memory:): output does not match pattern '(?s)"detail":"HASH JOIN collation_unique_probe".*"rows_per_input":0.1' -multi_index_or_compound.sqltest::derived-or-filter-skips-correlated-columns | database variant 1/1 (:memory:): output does not match pattern '"rows_per_input":1000' multi_index_or_compound.sqltest::implied-or-filter-accepts-negative-numbers | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' multi_index_or_compound.sqltest::implied-or-filter-accepts-parenthesized-equalities | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' multi_index_or_compound.sqltest::implied-or-filter-keeps-compound-index-searches | database variant 1/1 (:memory:): expected 2 row(s), got 1 row(s); row 0 expected '1/0/0/SCAN b' but was '0/0/0/MANAGED EVALUATOR FALLBACK' @@ -196,13 +195,5 @@ turso/alter_column.sqltest::alter-column-rewrites-indexed-virtual-expression-cha turso/alter_column.sqltest::alter-column-rewrites-indexed-virtual-to-regular | database variant 1/1 (:memory:): expected 4 row(s), got 4 row(s); row 0 expected 'wal' but was 'memory' turso/alter_column.sqltest::alter-column-rewrites-partial-index-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' turso/alter_column.sqltest::alter-column-rewrites-partial-index-virtual-expression-change | database variant 1/1 (:memory:): expected 4 row(s), got 4 row(s); row 0 expected 'wal' but was 'memory' -turso/explain-query-plan-json.sqltest::a-compound-select-lists-its-arms | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT name FROM users WHERE age > 30\n UNION\n SELECT n -turso/explain-query-plan-json.sqltest::a-from-clause-subquery-nests-its-body-under-its-own-scan | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT city_id, avg(age)\n FROM (SELECT * FROM users WHERE -turso/explain-query-plan-json.sqltest::a-recursive-cte-reports-its-setup-and-step-phases | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH RECURSIVE cnt(x) AS (\n SELECT 1\n UNION AL -turso/explain-query-plan-json.sqltest::an-or-over-two-indexes-lists-both-branches | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT *\n FROM users\n WHERE id = 5 OR age = 30;","resu turso/explain-query-plan-json.sqltest::both-readers-of-a-shared-cte-point-at-one-materialization | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH spenders AS (\n SELECT user_id, sum(amount) AS tot -turso/explain-query-plan-json.sqltest::list-and-scalar-subqueries-report-whether-they-are-correlated | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, (SELECT count(*) FROM orders o WHERE o.user_id -turso/explain-query-plan-json.sqltest::quotes-in-the-statement-are-escaped-in-the-echo | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.name = 'a \"quote -turso/explain-query-plan-json.sqltest::search-nodes-name-the-index-the-constraints-and-the-join | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name, o.amount\n FROM users u\n LEFT JOIN order -turso/explain-query-plan-json.sqltest::the-json-tree-matches-the-text-rows | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 4 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT u.name\n FROM users u\n WHERE u.age IN (SELECT ag turso/vacuum.sqltest::vacuum-rejects-query-only | database variant 1/1 (:temp:): error 'attempt to write a readonly database' does not match pattern 'Cannot execute VACUUM in query_only mode' diff --git a/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs b/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs index 64f9a078..ea453a15 100644 --- a/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs +++ b/src/Ahtola.Tests/ExplainQueryPlanFormatJsonTests.cs @@ -331,7 +331,7 @@ public void PlainTableFallbackStillDescribesItsActualBaseScan() } [Test] - public void UnfilteredTableScanReportsThePlannersCurrentRowEstimate() + public void TableScanReportsThePlannersCurrentRowEstimate() { using var embedded = new EmbeddedDatabase(); using var connection = embedded.Connect(); @@ -364,10 +364,24 @@ JsonElement Estimate(string query) ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON SELECT * FROM t;").Single() .Should().Contain("\"rows_per_input\":100,\"output_rows\":100,"); - // A filtered scan needs the unported selectivity model, so it reports no estimate. - using var filtered = JsonDocument.Parse( - ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON SELECT * FROM t WHERE value > 3;").Single()); - foreach (var node in filtered.RootElement.GetProperty("nodes").EnumerateArray()) + // A filtered scan keeps the rows its column constraints select (sel_range 0.4) and pays + // the WHERE work of its terms beyond their first step (none for one comparison). + var filtered = Estimate("SELECT * FROM t WHERE value > 3;"); + filtered.GetProperty("rows_per_input").GetDouble().Should().BeApproximately(40, 1e-9); + filtered.GetProperty("output_rows").GetDouble().Should().BeApproximately(40, 1e-9); + filtered.GetProperty("access_cost").GetDouble().Should().BeApproximately(2.3, 1e-12); + + // An OR is not a column constraint: every row survives the estimate, and its extra + // comparison steps are charged per visited row (100 rows x 2 steps x 0.003). + var disjunction = Estimate("SELECT * FROM t WHERE value = 1 OR value > 50;"); + disjunction.GetProperty("output_rows").GetDouble().Should().Be(100); + disjunction.GetProperty("access_cost").GetDouble().Should().BeApproximately(2.9, 1e-12); + + // BETWEEN is rewritten by Turso into two range terms before costing; that shape is not + // modeled, so it reports no estimate rather than an approximated one. + using var between = JsonDocument.Parse( + ReadAll(connection, "EXPLAIN QUERY PLAN FORMAT=JSON SELECT * FROM t WHERE value BETWEEN 3 AND 9;").Single()); + foreach (var node in between.RootElement.GetProperty("nodes").EnumerateArray()) node.GetProperty("op").TryGetProperty("estimate", out _).Should().BeFalse(); } From 818d02a15cb986a30659cbdd459c74d36e7e5d60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 21:57:00 -0400 Subject: [PATCH 16/20] Relabel verified journal-mode policy ledger entries; note v0.8.1 corpus in AGENTS.md Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 5 ++++- .../managed-sqltest-expected-failures.txt | 16 ++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e437e026..82a4cae0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,10 @@ including the tagged browser WASM binding, without cloning ad hoc or guessing at API shape. Historical gap/MVCC contracts that explicitly name `v0.7.2` retain that version as their behavioral analysis baseline, and the gap inventory and audits that name `v0.8.0-pre.7` (commit `277ddd050`) keep that -tag as theirs until they are refreshed against the current pin. +tag as theirs until they are refreshed against the current pin. The vendored +sqltest corpus (`conformance/sqlite-sqltests`) and its expected-failures +ledger are refreshed to `v0.8.1`; see the "v0.8.1 refresh" section of +`docs/turso-remaining-gap-plan.md`. ```powershell git submodule update --init --recursive # first checkout / fresh clone diff --git a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt index 2e48f3d6..5ba8902c 100644 --- a/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt +++ b/src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt @@ -195,14 +195,14 @@ multi_index_or_compound.sqltest::nested-and-branches-use-one-index-search | data multi_index_or_compound.sqltest::nested-or-branches-use-one-index-search | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '1/0/0/SEARCH t USING COVERING INDEX tx (x=?)' but was '0/0/0/MANAGED EVALUATOR FALLBACK' turso/alter_column.sqltest::alter-column-rejects-index-only-rewrite-mvcc | error message lacks the CLI "Parse error:" prefix the Turso corpus pins (the MVCC index-rebuild rejection text is otherwise identical) turso/alter_column.sqltest::alter-column-rejects-index-rewrite-mvcc | error message lacks the CLI "Parse error:" prefix the Turso corpus pins (the MVCC index-rebuild rejection text is otherwise identical) -turso/alter_column.sqltest::alter-column-rewrites-expression-index-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' -turso/alter_column.sqltest::alter-column-rewrites-generated-index-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' -turso/alter_column.sqltest::alter-column-rewrites-indexed-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' -turso/alter_column.sqltest::alter-column-rewrites-indexed-regular-to-virtual | database variant 1/1 (:memory:): expected 4 row(s), got 4 row(s); row 0 expected 'wal' but was 'memory' -turso/alter_column.sqltest::alter-column-rewrites-indexed-virtual-expression-change | database variant 1/1 (:memory:): expected 4 row(s), got 4 row(s); row 0 expected 'wal' but was 'memory' -turso/alter_column.sqltest::alter-column-rewrites-indexed-virtual-to-regular | database variant 1/1 (:memory:): expected 4 row(s), got 4 row(s); row 0 expected 'wal' but was 'memory' -turso/alter_column.sqltest::alter-column-rewrites-partial-index-affinity-change | database variant 1/1 (:memory:): expected 5 row(s), got 5 row(s); row 0 expected 'wal' but was 'memory' -turso/alter_column.sqltest::alter-column-rewrites-partial-index-virtual-expression-change | database variant 1/1 (:memory:): expected 4 row(s), got 4 row(s); row 0 expected 'wal' but was 'memory' +turso/alter_column.sqltest::alter-column-rewrites-expression-index-affinity-change | intentional storage-mode policy: a :memory: database keeps journal_mode=memory where Turso reports wal; every later row (rewritten values, index reads, integrity_check) matches +turso/alter_column.sqltest::alter-column-rewrites-generated-index-affinity-change | intentional storage-mode policy: a :memory: database keeps journal_mode=memory where Turso reports wal; every later row (rewritten values, index reads, integrity_check) matches +turso/alter_column.sqltest::alter-column-rewrites-indexed-affinity-change | intentional storage-mode policy: a :memory: database keeps journal_mode=memory where Turso reports wal; every later row (rewritten values, index reads, integrity_check) matches +turso/alter_column.sqltest::alter-column-rewrites-indexed-regular-to-virtual | intentional storage-mode policy: a :memory: database keeps journal_mode=memory where Turso reports wal; every later row (rewritten values, index reads, integrity_check) matches +turso/alter_column.sqltest::alter-column-rewrites-indexed-virtual-expression-change | intentional storage-mode policy: a :memory: database keeps journal_mode=memory where Turso reports wal; every later row (rewritten values, index reads, integrity_check) matches +turso/alter_column.sqltest::alter-column-rewrites-indexed-virtual-to-regular | intentional storage-mode policy: a :memory: database keeps journal_mode=memory where Turso reports wal; every later row (rewritten values, index reads, integrity_check) matches +turso/alter_column.sqltest::alter-column-rewrites-partial-index-affinity-change | intentional storage-mode policy: a :memory: database keeps journal_mode=memory where Turso reports wal; every later row (rewritten values, index reads, integrity_check) matches +turso/alter_column.sqltest::alter-column-rewrites-partial-index-virtual-expression-change | intentional storage-mode policy: a :memory: database keeps journal_mode=memory where Turso reports wal; every later row (rewritten values, index reads, integrity_check) matches turso/explain-query-plan-json.sqltest::a-compound-select-lists-its-arms | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT name FROM users WHERE age > 30\n UNION\n SELECT n turso/explain-query-plan-json.sqltest::a-from-clause-subquery-nests-its-body-under-its-own-scan | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n SELECT city_id, avg(age)\n FROM (SELECT * FROM users WHERE turso/explain-query-plan-json.sqltest::a-recursive-cte-reports-its-setup-and-step-phases | database variant 1/1 (:memory:): expected 1 row(s), got 1 row(s); row 0 expected '{"version":1,"sql":"EXPLAIN QUERY PLAN FORMAT=JSON\n WITH RECURSIVE cnt(x) AS (\n SELECT 1\n UNION AL From 8d600d9a3df93595a831f036dfe0cc92be5b7560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Wed, 30 Sep 2026 23:05:34 -0400 Subject: [PATCH 17/20] Document the Turso v0.8.1 parity refresh and reconcile the ledger count Record what the v0.8.1 refresh delivered, the 33 remaining v0.8.1 differences (evaluator-route planner shapes and documented policies), what stays unported, and known follow-ups. The inventory's live expected-failure count is now 109. Co-Authored-By: Claude Opus 5.5 --- docs/turso-gap-inventory.json | 4 +- docs/turso-remaining-gap-plan.md | 117 +++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/docs/turso-gap-inventory.json b/docs/turso-gap-inventory.json index 5c5dcc6e..95ef5523 100644 --- a/docs/turso-gap-inventory.json +++ b/docs/turso-gap-inventory.json @@ -14,8 +14,8 @@ "commit_at_analysis": "7b62a3f" }, "conformance_ground_truth": "src/Ahtola.Tests/Conformance/managed-sqltest-expected-failures.txt (606 failure lines at analysis time)", - "current_conformance_expected_failures": 191, - "current_conformance_expected_failures_note": "Integrated work closes the original SQL/window/CTE correctness groups, 24 new SQL-extension, 14 generated-column/reserved-name, three legacy-view, safe ATTACH routing/CDC/sequence behavior, and the cycling-sequence commit failure. Non-cycling watermarks are repaired; its private row-count representation remains different. Multiple memory-only databases can commit atomically; any physical database write must be the transaction's sole database mutation. The planner and Plan/EXPLAIN parity work closes 31 additional conformance markers, including JSON EQP, grouped-derived probes, and correlated-IN automatic-index cursor allocation. Other assigned gaps and intentional feature/representation/eager-corruption differences remain. Executing the previously CLI-gated turso/fts.sqltest corpus adds five markers, all the CLI \"Parse error:\" prefix on otherwise identical MATCH-context errors. Refreshing the corpus to the pinned v0.8.1 content (33 new and 41 refreshed files) adds 113 markers pending closure. These are not 191 independent bugs. The remaining-gap plan tracks current work; the 217 closed records below are historical dispositions.", + "current_conformance_expected_failures": 109, + "current_conformance_expected_failures_note": "Integrated work closes the original SQL/window/CTE correctness groups, 24 new SQL-extension, 14 generated-column/reserved-name, three legacy-view, safe ATTACH routing/CDC/sequence behavior, and the cycling-sequence commit failure. Non-cycling watermarks are repaired; its private row-count representation remains different. Multiple memory-only databases can commit atomically; any physical database write must be the transaction's sole database mutation. The planner and Plan/EXPLAIN parity work closes 31 additional conformance markers, including JSON EQP, grouped-derived probes, and correlated-IN automatic-index cursor allocation. Other assigned gaps and intentional feature/representation/eager-corruption differences remain. Executing the previously CLI-gated turso/fts.sqltest corpus adds five markers, all the CLI \"Parse error:\" prefix on otherwise identical MATCH-context errors. Refreshing the corpus to the pinned v0.8.1 content (33 new and 41 refreshed files) exposed 113 markers; the v0.8.1 parity work closed 80 of them (and two older generated-column integrity markers), and the remaining 33 are planner/EQP shapes the evaluator route does not execute plus documented storage-mode and message policies. These are not 109 independent bugs. The remaining-gap plan tracks current work; the 217 closed records below are historical dispositions.", "current_execution_plan": "docs/turso-remaining-gap-plan.md", "last_reconciled": "2026-09-23", "entry_count": 217, diff --git a/docs/turso-remaining-gap-plan.md b/docs/turso-remaining-gap-plan.md index 29938036..89ef4e39 100644 --- a/docs/turso-remaining-gap-plan.md +++ b/docs/turso-remaining-gap-plan.md @@ -1,5 +1,122 @@ # Remaining Turso gap closure plan +## v0.8.1 refresh (2026-09-30) + +The `turso-src` pin moved to `v0.8.1` (`8549c1659`), and the vendored corpus +was refreshed to match: 33 new and 41 changed sqltest files. The refresh +exposed 113 new case-level differences. The 0.8 parity work closed 80 of them, +plus two older generated-column integrity markers. The ledger now holds +**109** entries: 76 from earlier baselines and 33 from this refresh. + +### Delivered + +- **MVCC correctness.** A `BEGIN CONCURRENT` writer whose snapshot predates a + peer's committed update or delete now gets a write-write conflict. Before + this, it could overwrite the peer or bring back a deleted row (Turso + `0f7f30eac`). Explicit rowids and INTEGER PRIMARY KEY values are no longer + rewritten by the store-global allocator. Only rowids the statement chose + itself are promoted, and two concurrent explicit inserts of one key conflict. +- **SQL surface.** + - `CROSS JOIN ... ON/USING`, and a comma join followed by `ON`, now parse. + CROSS JOIN no longer allows the joins to be reordered. + - Correlated IN/EXISTS after a FULL JOIN now runs instead of being rejected. + - Generated columns: REPLACE and DEFAULT values are applied before virtual + columns are computed, and invalid clauses are rejected. + - UPSERT honours the target alias. A BEFORE UPDATE trigger now refreshes the + columns the UPSERT does not SET. + - Internal tables are protected whatever their letter case. + - The 2,000-column limit is enforced. + - Parameter names follow SQLite's rules: a bare `$` is rejected, and + TCL-style `$::v` and `$a(b)` names are accepted. + - `PRAGMA count_changes` behaves like SQLite. + - Smaller fixes: renaming a table onto an existing view, the + temp.synchronous value, and NULL arguments to `generate_series`. +- **Functions.** + - `json_group_object` skips NULL labels, and `jsonb(NULL)` returns NULL. + - `json_each` and `json_tree` expose a `rowid`. + - Date/time functions read BLOB arguments as text, and round correctly just + before the Unix epoch. + - `min`/`max` break ties the way SQLite does. + - A COUNT on a NOT NULL column uses the simple-count plan. +- **Planner (Turso 0.8 cost model, `TursoCostModel.cs`).** + - Correlated IN/EXISTS with non-equality comparisons is unnested, and the + inner side is costed with Turso's semi/anti-join model. NOT EXISTS can use + a hash anti-join. + - IN-list and OR-implied IN index searches; multi-index OR with AND branches. + - A disjunct of a partial index's predicate can prove that index usable. + - Compiled joins are costed with Turso's join planner. This includes + left-deep hash joins and materialized build prefixes. + - EQP output describes the join tree that actually executes, and FORMAT=JSON + reports the cost model's real estimates. + - ANALYZE row estimates are reported for unfiltered scans. +- **Storage.** + - `PRAGMA max_page_count` is enforced on file-backed and `:memory:` + databases, failing with `database or disk is full` (code 13). + - `page_count` is exact; for `:memory:` it is computed from a fresh page + image built by the same page builders the engine uses to write the file. + - integrity_check reports STRICT storage-class violations and virtual + generated-column violations. + - Explicit checkpoints are refused while a statement is still running on + the same connection. + - WAL readers can use read-marks 1–4 while a checkpoint holds read-mark 0. + - auto_vacuum reports the mode stored in page 1. + - MVCC ALTER COLUMN rejects changes that would require rebuilding an index. + - FTS converts non-text queries to text. +- **Error codes.** SQLite primary and extended result codes now reach + `SqliteException`, on both local and remote connections: constraint + 19/2067/1299/275/1555/787, full 13, notadb 26, busy 5, readonly 8. +- **Bindings (Turso 0.8 `bindings/dotnet`).** + - 13 replica connection-string keys. + - Explicit `Pull`, `Push`, `Checkpoint` and `GetSyncStatistics` calls. + - An automatic-sync status property and change event, plus an opt-in + `PullOnly` mode (the default still pushes and pulls). + - An auth-token provider hook for rotating tokens. + - Unexpected `replication_index` values are tolerated instead of throwing. + +### Remaining v0.8.1 differences (33) + +- **Planner shapes the evaluator route cannot honestly report (19):** in + join/memory, hash_join_order_by and multi_index_or_compound, plus one + shared-CTE JSON case. The corpus harness runs with a cancellation token, + which sends non-aggregate SELECTs through the evaluator. The evaluator always + hashes the right input. It has no build-left join, no materialized-prefix + join and no reordering. Other remaining cases need Turso's + 1,000,000-row default for tables with no ANALYZE statistics, or per-step + JSON estimates passed through to the executed plan. Printing Turso's plan + for any of these would describe work that does not run. +- **Documented policies (14):** + - Eight `turso/alter_column` cases differ only because a `:memory:` + database keeps journal_mode=memory. Every later row was verified to match. + - Two MVCC ALTER COLUMN rejections differ only by the CLI's `Parse error:` + prefix. + - Schema text keeps SQLite's verbatim `COLLATE 'fr-FR'` quoting. + - VACUUM under query_only reports SQLite's readonly error. + - Managed ATTACH requires a file-backed primary database. + - One planner case would print `COVERING` for a seek that does read table + rows, so it is left as is. + +### Not ported + +- The PostgreSQL frontend, general typed values and incremental materialized + views remain separate product decisions (see below). +- Replica connection pooling, the standalone Serverless/Platform client + packages, `Force Logical MVCC Pull=True`, and per-step batch statistics. + +### Known follow-ups + +- A single-table GROUP BY over the 10,000-row default fixture takes 25–40 s. + It is equally slow on the pre-refresh build `4d17cd2`. That puts + `groupby/default.sqltest` and `offset/default.sqltest` right at the corpus's + 30-second per-case cap, so they intermittently time out. This is a + pre-existing performance problem, not a v0.8.1 parity difference. +- Large plain equi-joins (3,000 × 3,000) take tens of seconds, both before and + after this work. This is likely the hash-join spill path under the 2 MB + execution budget. +- INDEXED BY / NOT INDEXED should rule out hash joins and ephemeral indexes, + as in Turso. +- At page size 1024, large index keys pack less tightly than in SQLite, which + shows in `page_count`. + ## Baseline and scope Prepared 2026-09-06 against Ahtola `83bb892` and the read-only Turso From 6717362f5ba03a0ff002d61e154843b499ef3b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Thu, 1 Oct 2026 09:08:54 -0400 Subject: [PATCH 18/20] Buffer sorter spill I/O so spilled GROUP BY stays fast A GROUP BY whose sort spills under the default 2 MB execution budget took 25-40 s at 10k rows: the spill codec issued two or three unbuffered file operations per value written or read. The sorter's spill file now coalesces writes in a buffer that absorbs the codec's reserved length headers, and each run reader reads ahead in blocks. Both buffers scale with the memory limit (none below 512 bytes, so tiny budgets keep their exact accounting) and are charged to the spill and merge infrastructure. The 10k-row GROUP BY drops from ~28 s to ~0.5 s; groupby/default.sqltest from ~15 minutes to ~80 s, so it no longer hits the 30 s per-case cap. Co-Authored-By: Claude Opus 5.5 --- docs/turso-remaining-gap-plan.md | 14 +- .../Execution/ResumableStatement.cs | 39 +++- .../Execution/VdbeExecutionMemory.cs | 40 +++- src/Ahtola.Core/Execution/VdbeSpillStore.cs | 151 ++++++++++++++ .../VdbeBufferedSpillFileTests.cs | 195 ++++++++++++++++++ 5 files changed, 419 insertions(+), 20 deletions(-) create mode 100644 src/Ahtola.Tests/VdbeBufferedSpillFileTests.cs diff --git a/docs/turso-remaining-gap-plan.md b/docs/turso-remaining-gap-plan.md index 89ef4e39..64cad530 100644 --- a/docs/turso-remaining-gap-plan.md +++ b/docs/turso-remaining-gap-plan.md @@ -104,11 +104,15 @@ plus two older generated-column integrity markers. The ledger now holds ### Known follow-ups -- A single-table GROUP BY over the 10,000-row default fixture takes 25–40 s. - It is equally slow on the pre-refresh build `4d17cd2`. That puts - `groupby/default.sqltest` and `offset/default.sqltest` right at the corpus's - 30-second per-case cap, so they intermittently time out. This is a - pre-existing performance problem, not a v0.8.1 parity difference. +- Fixed: a single-table GROUP BY over the 10,000-row default fixture took + 25–40 s, which also predated the refresh (`4d17cd2`). The sort spills + under the default 2 MB execution budget, and the spill codec issued two or + three unbuffered file operations for every value it wrote or read. The + sorter's spill file now coalesces writes, and each run reader reads ahead. + Both buffers scale with the memory limit and are charged to it. The same + query now takes under a second, and `groupby/default.sqltest` runs in + about 80 s for the whole file, down from about 15 minutes. Hash-join, + window and keyed-row spills still use unbuffered I/O. - Large plain equi-joins (3,000 × 3,000) take tens of seconds, both before and after this work. This is likely the hash-join spill path under the 2 MB execution budget. diff --git a/src/Ahtola.Core/Execution/ResumableStatement.cs b/src/Ahtola.Core/Execution/ResumableStatement.cs index 79f61fdb..ffe96b47 100644 --- a/src/Ahtola.Core/Execution/ResumableStatement.cs +++ b/src/Ahtola.Core/Execution/ResumableStatement.cs @@ -5027,7 +5027,9 @@ private List SortBufferedRows(CancellationToken cancellationToken) private void StartMerge(CancellationToken cancellationToken) { var runCount = _spill!.RunCount; - var infrastructureBytes = VdbeManagedFootprint.EstimateMergeInfrastructure(runCount); + var infrastructureBytes = VdbeManagedFootprint.EstimateMergeInfrastructure( + runCount, + _memory.LimitBytes); _memory.RetainOrThrow(infrastructureBytes, rows: 0); _mergeInfrastructureBytes = infrastructureBytes; try @@ -5128,7 +5130,8 @@ private SorterSpill CreateSpill() VdbeMemoryReservation.Create( _memory, VdbeManagedFootprint.EstimateSorterSpillInfrastructure( - _executionOptions.TemporaryDirectory)); + _executionOptions.TemporaryDirectory, + _memory.LimitBytes)); try { return SorterSpill.Create( @@ -5172,7 +5175,8 @@ private bool TryBuffer(SqlValue[] record, long recordBytes) && _executionOptions.AllowTemporaryFileSpill && retainedBytes > _memory.AvailableBytes - VdbeManagedFootprint.EstimateSorterSpillInfrastructure( - _executionOptions.TemporaryDirectory)) + _executionOptions.TemporaryDirectory, + _memory.LimitBytes)) { return false; } @@ -5345,7 +5349,10 @@ private void Initialize() currentCapacity: 0, requiredCount: 1)); _temporaryFile = VdbeTemporaryFile.Create(_executionOptions, "sorter"); - _file = _temporaryFile.File; + var writeBufferBytes = VdbeManagedFootprint.GetSorterSpillWriteBufferBytes(_memory.LimitBytes); + _file = writeBufferBytes == 0 + ? _temporaryFile.File + : new VdbeBufferedSpillFile(_temporaryFile.File, writeBufferBytes); _writePosition = VdbeSpillRecordCodec.InitializeFile( File, VdbeSpillFileKind.SorterRun, @@ -5583,7 +5590,9 @@ private RunDescriptor MergeRunGroup( RunReader[]? readers = null; RowLease[]? heads = null; RowLease? current = null; - var infrastructureBytes = VdbeManagedFootprint.EstimateMergeInfrastructure(count); + var infrastructureBytes = VdbeManagedFootprint.EstimateMergeInfrastructure( + count, + memory.LimitBytes); memory.RetainOrThrow(infrastructureBytes, rows: 0); try { @@ -5600,7 +5609,8 @@ private RunDescriptor MergeRunGroup( input.Offset, input.RowCount, _columnCount, - _executionOptions.Metrics); + _executionOptions.Metrics, + VdbeManagedFootprint.GetSorterRunReadBufferBytes(memory.LimitBytes)); readers[index] = reader; if (reader.TryReadNext(memory, out var row, cancellationToken)) { @@ -5692,7 +5702,8 @@ public RunReader OpenRunReader(int runIndex) run.Offset, run.RowCount, _columnCount, - _executionOptions.Metrics); + _executionOptions.Metrics, + VdbeManagedFootprint.GetSorterRunReadBufferBytes(_memory.LimitBytes)); } public void Dispose() @@ -5726,7 +5737,7 @@ private int GetEffectiveFanIn(int start, int maximumFanIn, long availableBytes) private long EstimateMergeBytes(int start, int count) { - var total = VdbeManagedFootprint.EstimateMergeInfrastructure(count); + var total = VdbeManagedFootprint.EstimateMergeInfrastructure(count, _memory.LimitBytes); for (var index = 0; index < count; index++) { total = checked( @@ -5833,6 +5844,7 @@ public sealed class RunReader : IDisposable private readonly IFile _file; private readonly int _columnCount; private readonly VdbeExecutionMetrics _metrics; + private readonly bool _ownsView; private long _position; public RunReader( @@ -5840,9 +5852,11 @@ public RunReader( long offset, int rowCount, int columnCount, - VdbeExecutionMetrics metrics) + VdbeExecutionMetrics metrics, + int readBufferBytes) { - _file = file; + _file = readBufferBytes == 0 ? file : new VdbeReadAheadSpillFile(file, readBufferBytes); + _ownsView = readBufferBytes != 0; _columnCount = columnCount; _metrics = metrics; _position = offset; @@ -5899,7 +5913,10 @@ public bool TryReadNext( public void Dispose() { - // The IFile is shared and owned by SorterSpill. + // The underlying IFile is shared and owned by SorterSpill; only the + // read-ahead view belongs to this reader. + if (_ownsView) + _file.Dispose(); } } } diff --git a/src/Ahtola.Core/Execution/VdbeExecutionMemory.cs b/src/Ahtola.Core/Execution/VdbeExecutionMemory.cs index 59e50758..5ffc9e20 100644 --- a/src/Ahtola.Core/Execution/VdbeExecutionMemory.cs +++ b/src/Ahtola.Core/Execution/VdbeExecutionMemory.cs @@ -331,6 +331,34 @@ internal static class VdbeManagedFootprint private const long PriorityQueueObjectBytes = 64; private const long PriorityQueueNodeBytes = 48; private const long RunReaderObjectBytes = 64; + private const long BufferedSpillFileObjectBytes = 48; + + private const int MaximumSorterSpillWriteBufferBytes = 16 * 1024; + private const int MaximumSorterRunReadBufferBytes = 4 * 1024; + private const int MinimumSpillBufferBytes = 512; + + /// + /// Write-coalescing buffer of a sorter spill file: at most 1/64 of the statement's memory + /// limit (capped at 16 KB), and none below 512 bytes, so a small budget keeps unbuffered + /// writes. Charged with the spill infrastructure. + /// + public static int GetSorterSpillWriteBufferBytes(long memoryLimitBytes) + => ScaleSpillBuffer(memoryLimitBytes / 64, MaximumSorterSpillWriteBufferBytes); + + /// + /// Read-ahead block of one sorter run reader: at most 1/512 of the statement's memory limit + /// (capped at 4 KB), and none below 512 bytes. Charged with the merge infrastructure. + /// + public static int GetSorterRunReadBufferBytes(long memoryLimitBytes) + => ScaleSpillBuffer(memoryLimitBytes / 512, MaximumSorterRunReadBufferBytes); + + private static int ScaleSpillBuffer(long share, int maximum) + => share < MinimumSpillBufferBytes ? 0 : (int)Math.Min(share, maximum); + + private static long EstimateSpillBuffer(int bufferBytes) + => bufferBytes == 0 + ? 0 + : checked(BufferedSpillFileObjectBytes + EstimateArray(sizeof(byte), bufferBytes)); private const long RunDescriptorSlotBytes = 32; private const long SorterSpillObjectBytes = 96; private const long HashSpillObjectBytes = 96; @@ -434,7 +462,7 @@ public static long EstimateSortWorkspace(int count) => + ListObjectBytes + EstimateArray(ReferenceBytes, count)); - public static long EstimateMergeInfrastructure(int runCount) + public static long EstimateMergeInfrastructure(int runCount, long memoryLimitBytes = 0) { ArgumentOutOfRangeException.ThrowIfNegative(runCount); if (runCount == 0) @@ -443,7 +471,8 @@ public static long EstimateMergeInfrastructure(int runCount) PriorityQueueObjectBytes + EstimateArray(PriorityQueueNodeBytes, runCount) + (2 * EstimateArray(ReferenceBytes, runCount)) - + (runCount * RunReaderObjectBytes)); + + (runCount * (RunReaderObjectBytes + + EstimateSpillBuffer(GetSorterRunReadBufferBytes(memoryLimitBytes))))); } public static long EstimateHashSpillInfrastructure( @@ -500,7 +529,9 @@ public static long EstimateHashPartitionSplitInfrastructure( + (subPartitionCount * (HashPartitionObjectBytes + subPartitionFileBytes))); } - public static long EstimateSorterSpillInfrastructure(string temporaryDirectory) + public static long EstimateSorterSpillInfrastructure( + string temporaryDirectory, + long memoryLimitBytes = 0) { ArgumentNullException.ThrowIfNull(temporaryDirectory); return checked( @@ -512,7 +543,8 @@ public static long EstimateSorterSpillInfrastructure(string temporaryDirectory) requiredCount: 1)) + EstimateTemporaryFileInfrastructure( temporaryDirectory.Length, - "sorter".Length)); + "sorter".Length) + + EstimateSpillBuffer(GetSorterSpillWriteBufferBytes(memoryLimitBytes))); } public static long EstimateKeyedRowSetSpillInfrastructure(string temporaryDirectory) diff --git a/src/Ahtola.Core/Execution/VdbeSpillStore.cs b/src/Ahtola.Core/Execution/VdbeSpillStore.cs index b832967d..02b8097d 100644 --- a/src/Ahtola.Core/Execution/VdbeSpillStore.cs +++ b/src/Ahtola.Core/Execution/VdbeSpillStore.cs @@ -109,6 +109,157 @@ public void Dispose() } } +/// +/// Coalesces a spill writer's small value writes into one buffered byte range so a record +/// costs a few file writes per buffer instead of several per value. The record codec writes +/// each value at the end of the record and then patches the record's length header at its +/// start, so the buffer accepts writes anywhere inside its capacity (a reserved header gap is +/// filled before the record completes) and a header written just ahead of a fresh buffer. +/// Reads of buffered bytes, length changes and flushes write the buffer out first. The +/// buffer is part of the owner's charged spill infrastructure. +/// +internal sealed class VdbeBufferedSpillFile(IFile inner, int bufferBytes) : IFile +{ + private readonly byte[] _buffer = new byte[bufferBytes]; + private long _bufferStart; + private int _bufferLength; + + public long Length => _bufferLength == 0 + ? inner.Length + : Math.Max(inner.Length, _bufferStart + _bufferLength); + + public bool IsReadOnly => inner.IsReadOnly; + + public int Read(long position, Span destination) + { + if (_bufferLength > 0 + && position < _bufferStart + _bufferLength + && position + destination.Length > _bufferStart) + { + FlushBuffer(); + } + + return inner.Read(position, destination); + } + + public void Write(long position, ReadOnlySpan source) + { + if (_bufferLength > 0) + { + var offset = position - _bufferStart; + // A gap after the buffered bytes may only be zero-filled while it is still unwritten + // file space; otherwise flushing the buffer would overwrite bytes already on disk. + if (offset >= 0 + && offset + source.Length <= _buffer.Length + && (offset <= _bufferLength || _bufferStart + _bufferLength >= inner.Length)) + { + var gapStart = _bufferLength; + var end = checked((int)offset + source.Length); + if (offset > gapStart) + _buffer.AsSpan(gapStart, (int)offset - gapStart).Clear(); + source.CopyTo(_buffer.AsSpan((int)offset)); + _bufferLength = Math.Max(_bufferLength, end); + return; + } + + if (position + source.Length == _bufferStart + && _bufferLength + source.Length <= _buffer.Length) + { + _buffer.AsSpan(0, _bufferLength).CopyTo(_buffer.AsSpan(source.Length)); + source.CopyTo(_buffer); + _bufferStart = position; + _bufferLength += source.Length; + return; + } + + FlushBuffer(); + } + + if (source.Length >= _buffer.Length) + { + inner.Write(position, source); + return; + } + + source.CopyTo(_buffer); + _bufferStart = position; + _bufferLength = source.Length; + } + + public void SetLength(long length) + { + FlushBuffer(); + inner.SetLength(length); + } + + public void FlushToDisk() + { + FlushBuffer(); + inner.FlushToDisk(); + } + + // The temporary file that owns the handle disposes it; buffered bytes of a discarded + // spill are never read again. + public void Dispose() => _bufferLength = 0; + + private void FlushBuffer() + { + if (_bufferLength == 0) + return; + + inner.Write(_bufferStart, _buffer.AsSpan(0, _bufferLength)); + _bufferLength = 0; + } +} + +/// +/// A sequential reader's read-ahead window over an immutable spill run. Short value reads are +/// served from one refilled block instead of one file read each. The block is charged as part +/// of the reader's merge infrastructure. Writes are not supported through this view. +/// +internal sealed class VdbeReadAheadSpillFile(IFile inner, int bufferBytes) : IFile +{ + private readonly byte[] _buffer = new byte[bufferBytes]; + private long _bufferStart; + private int _bufferLength; + + public long Length => inner.Length; + + public bool IsReadOnly => true; + + public int Read(long position, Span destination) + { + var offset = position - _bufferStart; + if (_bufferLength == 0 || offset < 0 || offset >= _bufferLength) + { + if (destination.Length >= _buffer.Length) + return inner.Read(position, destination); + + _bufferLength = Math.Max(0, inner.Read(position, _buffer)); + _bufferStart = position; + offset = 0; + if (_bufferLength == 0) + return 0; + } + + var count = Math.Min(destination.Length, _bufferLength - (int)offset); + _buffer.AsSpan((int)offset, count).CopyTo(destination); + return count; + } + + public void Write(long position, ReadOnlySpan source) => + throw new NotSupportedException("A spill read-ahead view is read-only."); + + public void SetLength(long length) => + throw new NotSupportedException("A spill read-ahead view is read-only."); + + public void FlushToDisk() + { + } + + public void Dispose() => _bufferLength = 0; +} + internal static class VdbeSpillRecordCodec { private static ReadOnlySpan Magic => "AHTSPILL"u8; diff --git a/src/Ahtola.Tests/VdbeBufferedSpillFileTests.cs b/src/Ahtola.Tests/VdbeBufferedSpillFileTests.cs new file mode 100644 index 00000000..19110579 --- /dev/null +++ b/src/Ahtola.Tests/VdbeBufferedSpillFileTests.cs @@ -0,0 +1,195 @@ +using AwesomeAssertions; +using Ahtola.Core; +using Ahtola.Core.Execution; +using Ahtola.Core.Storage; +using Ahtola.Data.Sqlite; + +namespace Ahtola.Tests; + +/// +/// The sorter's spill writer and run readers coalesce the record codec's per-value file +/// operations. Their bytes must be identical to unbuffered writes for every codec write +/// shape (reserved length headers patched afterwards, buffers filling mid-record, records +/// larger than the buffer, reads of not-yet-flushed bytes, rollback truncation), and a large +/// spilled GROUP BY must stay roughly linear instead of issuing several writes per value. +/// +public sealed class VdbeBufferedSpillFileTests +{ + [TestCase(16)] + [TestCase(64)] + [TestCase(4096)] + public void BufferedCodecWritesMatchUnbufferedBytes(int bufferBytes) + { + var random = new Random(4242); + var rows = new List(); + for (var index = 0; index < 400; index++) + { + var width = random.Next(1, 6); + var row = new SqlValue[width]; + for (var column = 0; column < width; column++) + { + row[column] = random.Next(5) switch + { + 0 => SqlValue.Null, + 1 => SqlValue.Integer(random.NextInt64()), + 2 => SqlValue.Real(random.NextDouble()), + 3 => SqlValue.Text(new string('x', random.Next(0, bufferBytes * 2))), + _ => SqlValue.BlobOwned(new byte[random.Next(0, 9)]), + }; + } + + rows.Add(row); + } + + var expected = WriteRecords(file => file, rows); + var actual = WriteRecords(file => new VdbeBufferedSpillFile(file, bufferBytes), rows); + + actual.Should().Equal(expected); + } + + [Test] + public void ReadsOfBufferedBytesSeeThePendingWrites() + { + using var inner = OpenFile(); + var file = new VdbeBufferedSpillFile(inner, 64); + + file.Write(0, [1, 2, 3, 4]); + file.Length.Should().Be(4); + inner.Length.Should().Be(0, "the bytes are still buffered"); + + Span read = stackalloc byte[4]; + file.Read(0, read).Should().Be(4); + read.ToArray().Should().Equal(1, 2, 3, 4); + inner.Length.Should().Be(4, "a read of buffered bytes writes them out first"); + } + + [Test] + public void SetLengthAfterBufferedWritesTruncatesTheFlushedImage() + { + using var inner = OpenFile(); + var file = new VdbeBufferedSpillFile(inner, 64); + + file.Write(0, [1, 2, 3, 4, 5, 6]); + file.SetLength(2); + file.Write(2, [9]); + file.FlushToDisk(); + + var bytes = new byte[inner.Length]; + inner.Read(0, bytes); + bytes.Should().Equal(1, 2, 9); + } + + [Test] + public void ReadAheadViewServesSequentialReadsFromOneBlock() + { + using var inner = OpenFile(); + var payload = Enumerable.Range(0, 300).Select(static value => (byte)value).ToArray(); + inner.Write(0, payload); + var counting = new CountingFile(inner); + using var view = new VdbeReadAheadSpillFile(counting, 128); + + var collected = new List(); + Span one = stackalloc byte[1]; + for (var position = 0L; position < payload.Length; position++) + { + view.Read(position, one).Should().Be(1); + collected.Add(one[0]); + } + + collected.Should().Equal(payload); + counting.Reads.Should().BeLessThanOrEqualTo(3, "300 one-byte reads fit in three 128-byte blocks"); + } + + [Test] + public void SpilledGroupByWritesWholeBuffersNotSingleValues() + { + // The default 2 MB execution budget spills this sort; before buffering, each spilled + // value cost two or three file writes and the statement took tens of seconds at 10k rows. + using var connection = new SqliteConnection("Data Source=:memory:"); + connection.Open(); + Execute(connection, "CREATE TABLE users (id INTEGER PRIMARY KEY, first_name TEXT, last_name TEXT, email TEXT, phone_number TEXT, address TEXT, city TEXT, state TEXT, zipcode TEXT, age INTEGER)"); + Execute(connection, "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i < 10000) INSERT INTO users(first_name,last_name,email,phone_number,address,city,state,zipcode,age) SELECT 'fn'||(i%700), 'ln'||(i%900), 'e'||i, 'p'||i, 'a'||i, 'c'||(i%300), 'S'||(i%50), 'z'||i, i%100 FROM n"); + + var watch = System.Diagnostics.Stopwatch.StartNew(); + using var command = connection.CreateCommand(); + command.CommandText = + "SELECT first_name, sum(age), count(*) FROM users GROUP BY first_name ORDER BY first_name LIMIT 3"; + var rows = new List(); + using (var reader = command.ExecuteReader()) + { + while (reader.Read()) + rows.Add($"{reader.GetString(0)}|{reader.GetInt64(1)}|{reader.GetInt64(2)}"); + } + watch.Stop(); + + rows.Should().Equal("fn0|0|14", "fn1|15|15", "fn10|150|15"); + watch.Elapsed.Should().BeLessThan( + TimeSpan.FromSeconds(10), + "the spilled sort must not degrade to per-value file I/O (it took ~28 s before)"); + } + + private static byte[] WriteRecords(Func wrap, IReadOnlyList rows) + { + using var inner = OpenFile(); + var file = wrap(inner); + var metrics = new VdbeExecutionMetrics(); + var position = VdbeSpillRecordCodec.InitializeFile(file, VdbeSpillFileKind.SorterRun, metrics); + foreach (var row in rows) + { + var recordStart = VdbeSpillRecordCodec.BeginRecord(ref position); + VdbeSpillRecordCodec.WriteValues(file, ref position, row, metrics); + VdbeSpillRecordCodec.CompleteRecord(file, recordStart, position, metrics); + } + + file.FlushToDisk(); + var bytes = new byte[inner.Length]; + inner.Read(0, bytes); + + position = VdbeSpillRecordCodec.FileHeaderSize; + foreach (var row in rows) + { + var recordEnd = VdbeSpillRecordCodec.ReadRecordEnd(file, ref position, metrics); + var values = VdbeSpillRecordCodec.ReadValues( + file, ref position, row.Length, recordEnd, metrics, CancellationToken.None); + VdbeSpillRecordCodec.RequireRecordEnd(position, recordEnd); + values.Length.Should().Be(row.Length); + } + + return bytes; + } + + private static IFile OpenFile() + => new InMemoryFileSystem().OpenFile("spill.bin", FileOpenMode.CreateNew); + + private static void Execute(SqliteConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } + + private sealed class CountingFile(IFile inner) : IFile + { + public int Reads { get; private set; } + + public long Length => inner.Length; + + public bool IsReadOnly => inner.IsReadOnly; + + public int Read(long position, Span destination) + { + Reads++; + return inner.Read(position, destination); + } + + public void Write(long position, ReadOnlySpan source) => inner.Write(position, source); + + public void SetLength(long length) => inner.SetLength(length); + + public void FlushToDisk() => inner.FlushToDisk(); + + public void Dispose() + { + } + } +} From f857af08caac4caf9654db7c73f4370fe7c9585a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Thu, 1 Oct 2026 10:30:10 -0400 Subject: [PATCH 19/20] Keep spilled hash joins batching and read partitions in blocks A 3,000 x 3,000 equi-join whose build side spills under the default 2 MB budget took ~32 s: - Resident build partitions filled the budget, so probe-batch admission (which keeps a quarter of the budget free) never grew a batch past one probe, and each probe reloaded its partition. Batch admission now evicts cached partitions to keep growing, and each batch answers groups whose partition is already resident first, so cyclic batches stop evicting a partition just before reusing it. - About half of the loads ran out of memory part-way and restarted after each eviction. Partitions now record their entries' retention as they are written; a load makes room for that lower bound first and skips a load that cannot fit. - Every partition read went through the spill codec one value at a time. Sequential partition scans now read through a transient read-ahead block charged to the statement's memory. The join drops to ~0.4 s (10,000 x 10,000: ~2.4 s); output keeps probe order. Co-Authored-By: Claude Opus 5.5 --- docs/turso-remaining-gap-plan.md | 17 ++- .../Execution/VdbeExecutionMemory.cs | 10 ++ .../Execution/VdbeHashJoinRuntime.cs | 136 ++++++++++++++++-- src/Ahtola.Core/Execution/VdbeSpillStore.cs | 2 +- .../VdbeBufferedSpillFileTests.cs | 30 ++++ 5 files changed, 178 insertions(+), 17 deletions(-) diff --git a/docs/turso-remaining-gap-plan.md b/docs/turso-remaining-gap-plan.md index 64cad530..163affb7 100644 --- a/docs/turso-remaining-gap-plan.md +++ b/docs/turso-remaining-gap-plan.md @@ -111,11 +111,18 @@ plus two older generated-column integrity markers. The ledger now holds sorter's spill file now coalesces writes, and each run reader reads ahead. Both buffers scale with the memory limit and are charged to it. The same query now takes under a second, and `groupby/default.sqltest` runs in - about 80 s for the whole file, down from about 15 minutes. Hash-join, - window and keyed-row spills still use unbuffered I/O. -- Large plain equi-joins (3,000 × 3,000) take tens of seconds, both before and - after this work. This is likely the hash-join spill path under the 2 MB - execution budget. + about 80 s for the whole file, down from about 15 minutes. +- Fixed: a spilled 3,000 × 3,000 equi-join took about 32 s. Resident build + partitions starved probe-batch admission, so batches held one probe and each + probe reloaded its partition. About half of those loads ran out of memory + part-way and started again, and every read went one value at a time. + Now a probe batch evicts cached partitions to keep growing, and groups are + answered resident-first. A load first makes room for its partition's + recorded entry retention, and skips a load that cannot fit. Each sequential + partition scan reads through a charged read-ahead block. The join now takes + about 0.4 s, and 10,000 × 10,000 takes about 2.4 s. Each probe batch still + re-reads the partitions it touches, because output keeps probe order. + Window and keyed-row spills still use unbuffered I/O. - INDEXED BY / NOT INDEXED should rule out hash joins and ephemeral indexes, as in Turso. - At page size 1024, large index keys pack less tightly than in SQLite, which diff --git a/src/Ahtola.Core/Execution/VdbeExecutionMemory.cs b/src/Ahtola.Core/Execution/VdbeExecutionMemory.cs index 5ffc9e20..023c8853 100644 --- a/src/Ahtola.Core/Execution/VdbeExecutionMemory.cs +++ b/src/Ahtola.Core/Execution/VdbeExecutionMemory.cs @@ -352,6 +352,16 @@ public static int GetSorterSpillWriteBufferBytes(long memoryLimitBytes) public static int GetSorterRunReadBufferBytes(long memoryLimitBytes) => ScaleSpillBuffer(memoryLimitBytes / 512, MaximumSorterRunReadBufferBytes); + /// + /// Transient read-ahead block of one sequential hash-spill scan (partition load, key + /// index build or fallback scan), sized like a sorter run reader's block. + /// + public static int GetSpillScanReadBufferBytes(long memoryLimitBytes) + => GetSorterRunReadBufferBytes(memoryLimitBytes); + + /// The bytes a spill read-ahead view of retains. + public static long EstimateSpillReadBuffer(int bufferBytes) => EstimateSpillBuffer(bufferBytes); + private static int ScaleSpillBuffer(long share, int maximum) => share < MinimumSpillBufferBytes ? 0 : (int)Math.Min(share, maximum); diff --git a/src/Ahtola.Core/Execution/VdbeHashJoinRuntime.cs b/src/Ahtola.Core/Execution/VdbeHashJoinRuntime.cs index 032587f8..9247010a 100644 --- a/src/Ahtola.Core/Execution/VdbeHashJoinRuntime.cs +++ b/src/Ahtola.Core/Execution/VdbeHashJoinRuntime.cs @@ -456,6 +456,11 @@ IEnumerable DrainPendingBatch() // budget in addition keeps that headroom scaling WITH the budget. var residencyReserve = context.Memory.LimitBytes / 4; var reserve = Math.Max(maxBuildEntryBytes, residencyReserve); + while (context.Memory.AvailableBytes - entryBytes < reserve + && residency.EvictForProbeBatch(context)) + { + } + if (context.Memory.AvailableBytes - entryBytes >= reserve && context.Memory.TryRetain(entryBytes)) { @@ -1020,7 +1025,17 @@ void CommitSlot(int localIndex, List? matches, long reserved, List< indices.Add(localIndex); } - foreach (var topIndex in orderedTopLevelIndices) + // Answer groups whose partition is already resident first. Batches tend to touch + // the same partitions in the same order, and when they outnumber what the cache + // can hold, first-appearance order makes LRU evict each partition just before the + // next batch needs it again (a cyclic sweep that never hits). Results are still + // emitted strictly in original probe order through EmitReadyPrefix. + var residentFirst = orderedTopLevelIndices + .Where(residency.HasResident) + .Concat(orderedTopLevelIndices.Where(topIndex => !residency.HasResident(topIndex))) + .ToList(); + + foreach (var topIndex in residentFirst) { context.ThrowIfCancellationRequested(); var topLevelLocalIndices = topLevelGroups[topIndex]; @@ -1684,6 +1699,18 @@ public Resolution Resolve(HashSpill spill, string key, VdbeJoinExecutionContext PartitionKey id, VdbeJoinExecutionContext context) { + // Make room for at least the partition's known entry retention before reading it, + // and skip a load that cannot fit even with every other partition evicted: a load + // that runs out of memory part-way rereads the partition from the start after + // each further eviction, which made failed loads half of all partition reads. + var required = spill.GetLoadLowerBoundBytes(id); + while (context.Memory.AvailableBytes < required && EvictOneLeastRecentlyUsed(id, context)) + { + } + + if (context.Memory.AvailableBytes < required) + return null; + var loaded = spill.TryLoadPartitionByKey(id, context); while (loaded is null && EvictOneLeastRecentlyUsed(id, context)) loaded = spill.TryLoadPartitionByKey(id, context); @@ -1742,14 +1769,36 @@ public bool ReleaseSelf(PartitionKey id, VdbeExecutionMetrics metrics) return true; } + /// + /// Evicts the least-recently-used resident partition so a probe batch can keep growing. + /// Resident partitions are only a cache; a batch that cannot grow flushes one probe at a + /// time and makes every probe re-resolve (and typically reload) its partition, so + /// trading cached partitions for batch headroom is what keeps grouping effective. + /// Returns false once nothing is resident. + /// + public bool HasResident(int topIndex) + { + foreach (var id in _lruNodes.Keys) + { + if (id.TopIndex == topIndex) + return true; + } + return false; + } + + public bool EvictForProbeBatch(VdbeJoinExecutionContext context) => + _lruOrder.First is { } node && Evict(node, context); + private bool EvictOneLeastRecentlyUsed(PartitionKey protect, VdbeJoinExecutionContext context) { var node = _lruOrder.First; while (node is not null && node.Value.Equals(protect)) node = node.Next; - if (node is null) - return false; + return node is not null && Evict(node, context); + } + private bool Evict(LinkedListNode node, VdbeJoinExecutionContext context) + { var victim = node.Value; _lruOrder.Remove(node); _lruNodes.Remove(victim); @@ -1924,6 +1973,7 @@ public void WriteBuild(BuildEntry entry, VdbeJoinExecutionContext context) var partition = _partitions[entry.Key is null ? 0 : GetPartition(entry.Key)]!; var position = partition.Position; WriteEntry(partition.File.File, ref position, entry, context); + partition.RecordEntry(EstimateRetainedEntryBytes(partition.Position, position)); partition.Position = position; partition.Count++; _totalBuildEntries++; @@ -1982,9 +2032,22 @@ public IEnumerable ReadPartitionByKey( partition.File.File, VdbeSpillFileKind.HashPartition, _options.Metrics); - return ReadEntriesWithEviction(partition.File.File, partition.Count, context, id, residency); + return ReadEntriesWithEviction(partition.File.File, partition.Count, context, id, residency, scanView: true); } + /// + /// A lower bound on the memory a full load of the partition retains: the sum of its + /// entries' decode retention (the same estimate TryReadEntry charges), without + /// the bucket dictionary's own overhead. A load cannot succeed with less available. + /// + public long GetLoadLowerBoundBytes(PartitionKey id) => GetPartitionFile(id).EntryRetainedBytes; + + private long EstimateRetainedEntryBytes(long recordStart, long recordEnd) => + VdbeManagedFootprint.EstimateHashBuildEntryFromEncodedLength( + recordEnd - recordStart - VdbeSpillRecordCodec.RecordLengthSize, + _columnCount, + _rowIdCount); + public LoadedPartition? TryLoadPartitionByKey( PartitionKey id, VdbeJoinExecutionContext context) => @@ -2010,11 +2073,12 @@ public IEnumerable ReadPartitionByKey( partition.File.File, VdbeSpillFileKind.HashPartition, _options.Metrics); + using var scan = OpenScanView(partition.File.File, context); long position = VdbeSpillRecordCodec.FileHeaderSize; for (var index = 0; index < partition.Count; index++) { using var lease = TryReadEntry( - partition.File.File, + scan.File, ref position, context, requireAvailable: false); @@ -2153,24 +2217,25 @@ public IEnumerable ReadPartitionByKey( partition.File.File, VdbeSpillFileKind.HashPartition, _options.Metrics); + using var scan = OpenScanView(partition.File.File, context); long position = VdbeSpillRecordCodec.FileHeaderSize; for (var entryIndex = 0; entryIndex < partition.Count; entryIndex++) { context.ThrowIfCancellationRequested(); var recordStart = position; var recordEnd = VdbeSpillRecordCodec.ReadRecordEnd( - partition.File.File, + scan.File, ref position, _options.Metrics); // Skip the ordinal - the index only needs to locate a record, not its // build-side ordinal, which is re-read from the record itself on a hit. - VdbeSpillRecordCodec.ReadInt64(partition.File.File, ref position, _options.Metrics); - var hasKey = VdbeSpillRecordCodec.ReadByte(partition.File.File, ref position, _options.Metrics); + VdbeSpillRecordCodec.ReadInt64(scan.File, ref position, _options.Metrics); + var hasKey = VdbeSpillRecordCodec.ReadByte(scan.File, ref position, _options.Metrics); string? key = hasKey switch { 0 => null, 1 => VdbeSpillRecordCodec.ReadString( - partition.File.File, + scan.File, ref position, recordEnd, _options.Metrics), @@ -2367,6 +2432,7 @@ public bool TrySplitPartition(int topIndex, VdbeJoinExecutionContext context) var sub = subPartitions[subIndex]; var position = sub.Position; WriteEntry(sub.File.File, ref position, entry, context); + sub.RecordEntry(EstimateRetainedEntryBytes(sub.Position, position)); sub.Position = position; sub.Count++; } @@ -2516,16 +2582,55 @@ private void WriteEntry( } } + /// + /// Opens a transient read-ahead view for one sequential scan of a spill file, so a + /// partition load, key-index build or fallback scan reads whole blocks instead of + /// issuing a file read per decoded value. The block is charged to the statement's + /// memory for the scan's duration; when it does not fit, the scan reads directly. + /// + private static SpillScanView OpenScanView(IFile file, VdbeJoinExecutionContext context) + { + var bufferBytes = VdbeManagedFootprint.GetSpillScanReadBufferBytes(context.Memory.LimitBytes); + var chargedBytes = VdbeManagedFootprint.EstimateSpillReadBuffer(bufferBytes); + if (bufferBytes == 0 || !context.Memory.TryRetain(chargedBytes, rows: 0)) + return SpillScanView.Direct(file); + + return new SpillScanView( + new VdbeReadAheadSpillFile(file, bufferBytes), + context.Memory, + chargedBytes); + } + + private sealed class SpillScanView(IFile file, VdbeExecutionMemory? memory, long chargedBytes) + : IDisposable + { + private long _chargedBytes = chargedBytes; + + public IFile File { get; } = file; + + public static SpillScanView Direct(IFile file) => new(file, memory: null, chargedBytes: 0); + + public void Dispose() + { + if (_chargedBytes == 0) + return; + + memory!.Release(_chargedBytes, rows: 0); + _chargedBytes = 0; + } + } + private IEnumerable ReadEntries( IFile file, int count, VdbeJoinExecutionContext context) { + using var scan = OpenScanView(file, context); long position = VdbeSpillRecordCodec.FileHeaderSize; for (var index = 0; index < count; index++) { yield return TryReadEntry( - file, + scan.File, ref position, context, requireAvailable: true)!; @@ -2548,8 +2653,11 @@ private IEnumerable ReadEntriesWithEviction( int count, VdbeJoinExecutionContext context, PartitionKey protectId, - PartitionResidencyCache residency) + PartitionResidencyCache residency, + bool scanView) { + using var scan = scanView ? OpenScanView(file, context) : SpillScanView.Direct(file); + file = scan.File; long position = VdbeSpillRecordCodec.FileHeaderSize; for (var index = 0; index < count; index++) { @@ -2731,6 +2839,12 @@ private sealed class Partition : IDisposable public int Count { get; set; } + /// Sum of the written entries' decode retention estimates. + public long EntryRetainedBytes { get; private set; } + + public void RecordEntry(long retainedBytes) => + EntryRetainedBytes = checked(EntryRetainedBytes + retainedBytes); + public void SetFile(VdbeTemporaryFile file) => _file = file; public void Dispose() => _file?.Dispose(); diff --git a/src/Ahtola.Core/Execution/VdbeSpillStore.cs b/src/Ahtola.Core/Execution/VdbeSpillStore.cs index 02b8097d..3ee6fe9f 100644 --- a/src/Ahtola.Core/Execution/VdbeSpillStore.cs +++ b/src/Ahtola.Core/Execution/VdbeSpillStore.cs @@ -265,7 +265,7 @@ internal static class VdbeSpillRecordCodec private static ReadOnlySpan Magic => "AHTSPILL"u8; private const byte FormatVersion = 1; public const int FileHeaderSize = 10; - private const int RecordLengthSize = 5; + public const int RecordLengthSize = 5; public static long InitializeFile( IFile file, diff --git a/src/Ahtola.Tests/VdbeBufferedSpillFileTests.cs b/src/Ahtola.Tests/VdbeBufferedSpillFileTests.cs index 19110579..6856471d 100644 --- a/src/Ahtola.Tests/VdbeBufferedSpillFileTests.cs +++ b/src/Ahtola.Tests/VdbeBufferedSpillFileTests.cs @@ -128,6 +128,36 @@ public void SpilledGroupByWritesWholeBuffersNotSingleValues() "the spilled sort must not degrade to per-value file I/O (it took ~28 s before)"); } + [Test] + public void SpilledHashJoinKeepsProbeBatchingAndReadsPartitionsInBlocks() + { + // Under the default 2 MB budget this join spills its build side. Resident partitions + // used to starve probe-batch admission (one probe per batch, a partition reload per + // probe, half of them failing part-way), all read one value at a time: ~32 s at 3k rows. + using var connection = new SqliteConnection("Data Source=:memory:"); + connection.Open(); + Execute(connection, "CREATE TABLE a(id INTEGER PRIMARY KEY, k INTEGER, pad TEXT)"); + Execute(connection, "CREATE TABLE b(id INTEGER PRIMARY KEY, k INTEGER, pad TEXT)"); + Execute(connection, "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i < 3000) INSERT INTO a(k, pad) SELECT i, printf('%050d', i) FROM n"); + Execute(connection, "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i < 3000) INSERT INTO b(k, pad) SELECT i % 1500, printf('%050d', i) FROM n"); + + var watch = System.Diagnostics.Stopwatch.StartNew(); + using var command = connection.CreateCommand(); + command.CommandText = "SELECT count(*), sum(a.k) FROM a JOIN b ON a.k = b.k"; + using var reader = command.ExecuteReader(); + reader.Read().Should().BeTrue(); + var matches = reader.GetInt64(0); + var keySum = reader.GetInt64(1); + watch.Stop(); + + // b.k cycles 0..1499 twice; a.k 1..1499 match two rows each (k = 0 matches nothing). + matches.Should().Be(2998); + keySum.Should().Be(2L * (1499L * 1500L / 2)); + watch.Elapsed.Should().BeLessThan( + TimeSpan.FromSeconds(10), + "a spilled hash join must not degrade to a partition reload per probe"); + } + private static byte[] WriteRecords(Func wrap, IReadOnlyList rows) { using var inner = OpenFile(); From daf60419646d2e84a1a828402bbc2416ce8fb1e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Andr=C3=A9=20Moreau?= Date: Thu, 1 Oct 2026 11:44:07 -0400 Subject: [PATCH 20/20] Hash-bucket evaluator DISTINCT, compound set operators and DISTINCT aggregates Evaluator DISTINCT, UNION/INTERSECT/EXCEPT, DISTINCT aggregates and recursive-CTE UNION compared every row with every kept row, whatever the memory budget: count(DISTINCT pad) over 20k rows took ~16 s and a self-UNION ~42 s. They now share a DistinctRowSet that buckets rows by a hash consistent with DISTINCT equality (integer/real numerically, BINARY, NOCASE and RTRIM text by their built-in rules; any other collation hashes text as a constant, so it stays correct). RowsEqual still decides equality within a bucket; UNION keeps the later row of an equal group as the pinned corpus expects. The same queries now take ~0.1-0.25 s. Differential tests against bundled SQLite cover value classes, collations, compound operators, DISTINCT aggregates and recursive UNION. Co-Authored-By: Claude Opus 5.5 --- docs/turso-remaining-gap-plan.md | 15 +- src/Ahtola.Core/EmbeddedDatabase.cs | 249 ++++++++++++++++-- .../DistinctHashBucketParityTests.cs | 186 +++++++++++++ 3 files changed, 421 insertions(+), 29 deletions(-) create mode 100644 src/Ahtola.Tests/DistinctHashBucketParityTests.cs diff --git a/docs/turso-remaining-gap-plan.md b/docs/turso-remaining-gap-plan.md index 163affb7..196f8291 100644 --- a/docs/turso-remaining-gap-plan.md +++ b/docs/turso-remaining-gap-plan.md @@ -122,7 +122,20 @@ plus two older generated-column integrity markers. The ledger now holds partition scan reads through a charged read-ahead block. The join now takes about 0.4 s, and 10,000 × 10,000 takes about 2.4 s. Each probe batch still re-reads the partitions it touches, because output keeps probe order. - Window and keyed-row spills still use unbuffered I/O. + Window and keyed-row spills still use unbuffered I/O, but no SQL shape + measured here reached them: window, DISTINCT, compound and recursive + queries over 20,000 rows created no spill files even at a 200 KB budget. +- Fixed: evaluator DISTINCT, UNION/INTERSECT/EXCEPT, DISTINCT aggregates + and recursive-CTE UNION compared each row with every kept row, whatever + the memory budget. At 20,000 rows `count(DISTINCT pad)` took about 16 s + and a self-UNION about 42 s. They now share a set that buckets rows by a + hash consistent with DISTINCT equality: integer and real compare + numerically, and BINARY, NOCASE and RTRIM text get their own hashes. Any + other collation hashes text as a constant, so it stays correct, just + slower. The same queries now take about 0.1–0.25 s. UNION still keeps the + later row of an equal group, as the pinned corpus expects. +- Pre-existing, noticed while testing: a stored `-0.0` loses its sign, and + `quote()` renders 9007199254740992.0 as `9.007199254740992e+15`. - INDEXED BY / NOT INDEXED should rule out hash joins and ephemeral indexes, as in Turso. - At page size 1024, large index keys pack less tightly than in SQLite, which diff --git a/src/Ahtola.Core/EmbeddedDatabase.cs b/src/Ahtola.Core/EmbeddedDatabase.cs index ac8ebf3b..46c19a92 100644 --- a/src/Ahtola.Core/EmbeddedDatabase.cs +++ b/src/Ahtola.Core/EmbeddedDatabase.cs @@ -35757,7 +35757,7 @@ private SourceData EvaluateRecursiveCte( } var result = new List(); - var seen = deduplicate ? new List() : null; + var seen = deduplicate ? new DistinctRowSet(this, collations) : null; // The budget caps how many EMITTED rows the expansion may produce: the // compound's own LIMIT (the OFFSET is consumed inside this loop), or the // outer query's row budget when the compound is unlimited. With both, the @@ -36110,16 +36110,19 @@ IReadOnlyList Transform(IReadOnlyList frontierRows) anchorRows, Transform, RecursiveCteRowLimit, RecursiveCteRowLimit); } - private bool TryAddRecursiveDistinctRow( - List seen, + private static bool TryAddRecursiveDistinctRow( + DistinctRowSet seen, SqlValue[] candidate, IReadOnlyList collations) { - if (seen.Any(existing => RecursiveRowsEqual(existing, candidate, collations))) - return false; + // A row whose width does not match the collation list never equals another row. + if (candidate.Length != collations.Count) + { + seen.Add(candidate); + return true; + } - seen.Add(candidate); - return true; + return seen.TryAdd(candidate); } private bool RecursiveRowsEqual( @@ -37007,18 +37010,18 @@ private List ApplyIntersect( IReadOnlyList right, IReadOnlyList collations) { - var result = new List(); + var rightSet = new DistinctRowSet(this, collations, right); + var result = new DistinctRowSet(this, collations); foreach (var row in left) { - if (result.Any(candidate => RowsEqual(candidate, row, collations)) - || !right.Any(candidate => RowsEqual(candidate, row, collations))) + if (!rightSet.Contains(row) || result.Contains(row)) continue; result.Add(row.ToArray()); } - SortCompoundSetRows(result, collations); - return result; + SortCompoundSetRows(result.Rows, collations); + return result.Rows; } private List ApplyExcept( @@ -37026,18 +37029,18 @@ private List ApplyExcept( IReadOnlyList right, IReadOnlyList collations) { - var result = new List(); + var rightSet = new DistinctRowSet(this, collations, right); + var result = new DistinctRowSet(this, collations); foreach (var row in left) { - if (result.Any(candidate => RowsEqual(candidate, row, collations)) - || right.Any(candidate => RowsEqual(candidate, row, collations))) + if (rightSet.Contains(row) || result.Contains(row)) continue; result.Add(row.ToArray()); } - SortCompoundSetRows(result, collations); - return result; + SortCompoundSetRows(result.Rows, collations); + return result.Rows; } // SQLite/Turso materialize UNION, INTERSECT, and EXCEPT sets in a temporary B-tree. The traversal of @@ -37078,11 +37081,14 @@ private void AddDistinctRows( IEnumerable source, IReadOnlyList collations) { + // A later row replaces the kept row of its equal group, as the pinned corpus expects + // (select/memory.sqltest collate-compound-1). + var set = new DistinctRowSet(this, collations, backing: destination); foreach (var row in source) { - var existingIndex = destination.FindIndex(candidate => RowsEqual(candidate, row, collations)); + var existingIndex = set.IndexOf(row); if (existingIndex < 0) - destination.Add(row.ToArray()); + set.Add(row.ToArray()); else destination[existingIndex] = row.ToArray(); } @@ -37413,14 +37419,14 @@ private SqlValue[][] ApplyDistinctLimit( var rows = source.ToList(); if (distinct) { - var distinctRows = new List(rows.Count); + // Bucket kept rows by a hash that agrees with RowsEqual, so each row is compared only + // with the kept rows that could equal it instead of with every kept row (quadratic). + // RowsEqual still decides equality inside a bucket, and first occurrences keep order. + var distinctRows = new DistinctRowSet(this, collations); foreach (var row in rows) - { - if (!distinctRows.Any(candidate => RowsEqual(candidate, row, collations))) - distinctRows.Add(row); - } + distinctRows.TryAdd(row); - rows = distinctRows; + rows = distinctRows.Rows; } if (offset >= rows.Count) @@ -37433,6 +37439,194 @@ private SqlValue[][] ApplyDistinctLimit( return rows.ToArray(); } + /// + /// Insertion-ordered set of rows under DISTINCT equality (): rows are + /// bucketed by and compared only within their bucket, so + /// UNION/INTERSECT/EXCEPT, DISTINCT and DISTINCT aggregates stay linear instead of comparing + /// each row with every kept row. + /// + private sealed class DistinctRowSet + { + private readonly EmbeddedDatabase _owner; + private readonly IReadOnlyList? _collations; + private readonly Dictionary> _buckets = []; + private DistinctTextHash?[] _textHashes = []; + + /// Supplies DISTINCT equality and collation resolution. + /// Per-column collations, or null for BINARY. + /// Rows added (without deduplication) to a new backing list. + /// + /// A caller-owned list to index and append to in place, instead of a new list. The caller + /// may replace a kept row with an equal one; equal rows share a bucket. + /// + public DistinctRowSet( + EmbeddedDatabase owner, + IReadOnlyList? collations, + IEnumerable? initial = null, + List? backing = null) + { + _owner = owner; + _collations = collations; + Rows = backing ?? []; + for (var index = 0; index < Rows.Count; index++) + Index(Rows[index], index); + + if (initial is null) + return; + + foreach (var row in initial) + Add(row); + } + + /// The kept rows, in insertion order. + public List Rows { get; } + + public bool Contains(IReadOnlyList row) => IndexOf(row) >= 0; + + public int IndexOf(IReadOnlyList row) + { + if (!_buckets.TryGetValue(Hash(row), out var candidates)) + return -1; + + foreach (var candidate in candidates) + { + if (_owner.RowsEqual(Rows[candidate], row, _collations)) + return candidate; + } + + return -1; + } + + /// Adds unless an equal row is already kept. + public bool TryAdd(SqlValue[] row) + { + if (Contains(row)) + return false; + + Add(row); + return true; + } + + /// Appends ; the caller has established it is new. + public void Add(SqlValue[] row) + { + Index(row, Rows.Count); + Rows.Add(row); + } + + private void Index(IReadOnlyList row, int position) + { + var hash = Hash(row); + if (!_buckets.TryGetValue(hash, out var candidates)) + { + candidates = []; + _buckets.Add(hash, candidates); + } + + candidates.Add(position); + } + + private int Hash(IReadOnlyList row) + { + if (_textHashes.Length < row.Count) + Array.Resize(ref _textHashes, row.Count); + return _owner.HashDistinctRow(row, _collations, _textHashes); + } + } + + /// How a DISTINCT column's text values hash consistently with . + private enum DistinctTextHash + { + /// The collation's equality is not modeled; text contributes a constant. + Opaque, + Binary, + NoCase, + RTrim, + } + + /// + /// Hashes a row so values RowsEqual treats as equal hash equally: NULLs alike, integers and + /// reals through their numeric value, blobs by bytes, and text by the column collation's + /// built-in equality. A registered, external or locale collation is opaque. + /// + private int HashDistinctRow( + IReadOnlyList row, + IReadOnlyList? collations, + DistinctTextHash?[] textHashes) + { + var hash = new HashCode(); + for (var index = 0; index < row.Count; index++) + { + var value = row[index]; + switch (value.Kind) + { + case SqlValueKind.Null: + hash.Add(0); + break; + case SqlValueKind.Integer: + hash.Add(HashDistinctNumber(value.AsInteger())); + break; + case SqlValueKind.Real: + hash.Add(HashDistinctNumber(value.AsReal())); + break; + case SqlValueKind.Blob: + var blobHash = new HashCode(); + blobHash.AddBytes(value.AsBlobSpan()); + hash.Add(blobHash.ToHashCode()); + break; + case SqlValueKind.Text: + var mode = index < textHashes.Length + ? textHashes[index] ??= ResolveDistinctTextHash(collations?[index]) + : DistinctTextHash.Opaque; + hash.Add(HashDistinctText(value.AsText(), mode)); + break; + } + } + + return hash.ToHashCode(); + } + + // An integer and a real compare equal only when the real is exactly that integer, which is + // then exactly representable, so hashing both as doubles keeps equal values together. + private static int HashDistinctNumber(double value) + => value == 0 ? 0 : value.GetHashCode(); + + private DistinctTextHash ResolveDistinctTextHash(string? collation) + { + var name = collation ?? "BINARY"; + if (_collations.ContainsKey(name) || _externalCollationResolver is not null) + return DistinctTextHash.Opaque; + if (string.Equals(name, "BINARY", StringComparison.OrdinalIgnoreCase)) + return DistinctTextHash.Binary; + if (string.Equals(name, "NOCASE", StringComparison.OrdinalIgnoreCase)) + return DistinctTextHash.NoCase; + if (string.Equals(name, "RTRIM", StringComparison.OrdinalIgnoreCase)) + return DistinctTextHash.RTrim; + return DistinctTextHash.Opaque; + } + + private static int HashDistinctText(string text, DistinctTextHash mode) + { + switch (mode) + { + case DistinctTextHash.Binary: + return string.GetHashCode(text, StringComparison.Ordinal); + case DistinctTextHash.RTrim: + return string.GetHashCode(text.AsSpan().TrimEnd(' '), StringComparison.Ordinal); + case DistinctTextHash.NoCase: + // NOCASE folds only ASCII letters; past an embedded NUL it compares lengths + // alone, so such text is left opaque rather than modeled here. + if (text.Contains('\0')) + return 1; + var hash = new HashCode(); + foreach (var character in text) + hash.Add(character is >= 'A' and <= 'Z' ? (char)(character + ('a' - 'A')) : character); + return hash.ToHashCode(); + default: + return 1; + } + } + private SqlValue[] EvaluateGroupKey( IReadOnlyList groupBy, SqlValue[] parameters, @@ -45858,16 +46052,15 @@ private IReadOnlyList ApplyAggregateModifiers( throw new EmbeddedSqlException("DISTINCT aggregates must have exactly one argument."); var collation = GetEffectiveCollation(function.Arguments[0], context); - var seen = new List(); + var seen = new DistinctRowSet(this, [collation]); var deduplicated = new List(); foreach (var row in result) { context.CheckInterrupt(); var value = Evaluate(function.Arguments[0], parameters, row, context); - if (seen.Any(existing => DistinctValuesEqual(existing, value, collation))) + if (!seen.TryAdd([value])) continue; - seen.Add(value); deduplicated.Add(row); } diff --git a/src/Ahtola.Tests/DistinctHashBucketParityTests.cs b/src/Ahtola.Tests/DistinctHashBucketParityTests.cs new file mode 100644 index 00000000..015f50a0 --- /dev/null +++ b/src/Ahtola.Tests/DistinctHashBucketParityTests.cs @@ -0,0 +1,186 @@ +using AwesomeAssertions; +using Ahtola.Data.Sqlite; +using MsData = Microsoft.Data.Sqlite; + +namespace Ahtola.Tests; + +/// +/// The evaluator's DISTINCT buckets kept rows by a hash before comparing them, instead of +/// comparing every row with every kept row. The hash must agree with DISTINCT equality for +/// every value class and collation, or duplicates would survive; these cases compare the kept +/// rows (including which representative of an equal group survives) with bundled SQLite. +/// +public sealed class DistinctHashBucketParityTests +{ + private const string Setup = """ + CREATE TABLE v(x, t TEXT, n TEXT COLLATE NOCASE, r TEXT COLLATE RTRIM, c TEXT COLLATE reverse_text); + INSERT INTO v VALUES + (1, 'a', 'Abc', 'pad', 'k'), + (1.0, 'a', 'aBC', 'pad ', 'k'), + (0.0, 'A', 'abc', 'pad ', 'K'), + (-0.0, 'a ', 'ABD', 'Pad', 'k'), + (NULL, NULL, NULL, NULL, NULL), + (NULL, NULL, NULL, NULL, NULL), + (x'0102', 'é', 'É', 'é ', 'é'), + (x'0102', 'é', 'é', 'é', 'é'), + (9007199254740993, 'b', 'a' || char(0) || 'x', 'b', 'b'), + (9007199254740992.0, 'b', 'a' || char(0) || 'y', 'b ', 'b'), + ('1', 'c', 'mixed Case', 'c', 'c'), + (2, 'c', 'MIXED case', 'c', 'c'); + """; + + [TestCase("x")] + [TestCase("t")] + [TestCase("n")] + [TestCase("r")] + [TestCase("c")] + [TestCase("x, t")] + [TestCase("n, r")] + [TestCase("t COLLATE NOCASE")] + [TestCase("t COLLATE RTRIM")] + [TestCase("x, n, c")] + public void DerivedDistinctKeepsTheSameRowsAsSqlite(string columns) + { + var sql = $"SELECT * FROM (SELECT DISTINCT {columns} FROM v)"; + + Run(sql).Should().Equal(RunSqlite(sql)); + } + + [TestCase("x")] + [TestCase("n")] + [TestCase("x, t")] + public void TopLevelDistinctKeepsTheSameRowsAsSqlite(string columns) + { + var sql = $"SELECT DISTINCT {columns} FROM v"; + + Run(sql).Should().Equal(RunSqlite(sql)); + } + + // Which row represents an equal UNION group depends on the plan (SQLite keeps the first + // here but the pinned corpus expects the later row), so groups are compared by count. + [TestCase("SELECT x FROM v UNION SELECT x FROM v WHERE x IS NOT NULL")] + [TestCase("SELECT n FROM v UNION SELECT t FROM v")] + [TestCase("SELECT t FROM v UNION SELECT n FROM v")] + [TestCase("SELECT r, x FROM v UNION SELECT r, x FROM v")] + [TestCase("SELECT c FROM v UNION SELECT t FROM v")] + [TestCase("SELECT x, t FROM v INTERSECT SELECT x, t FROM v WHERE x > 0")] + [TestCase("SELECT n FROM v INTERSECT SELECT t FROM v")] + [TestCase("SELECT x FROM v EXCEPT SELECT x FROM v WHERE typeof(x) = 'integer'")] + [TestCase("SELECT r FROM v EXCEPT SELECT t FROM v")] + [TestCase("SELECT n FROM v UNION SELECT r FROM v")] + public void CompoundSetOperatorsKeepTheSameGroupsAsSqlite(string compound) + { + var sql = $"SELECT count(*) FROM ({compound})"; + + Run(sql).Should().Equal(RunSqlite(sql)); + } + + [TestCase("SELECT t FROM v UNION SELECT t FROM v")] + [TestCase("SELECT x, t FROM v WHERE typeof(x) = 'integer' UNION SELECT x, t FROM v WHERE typeof(x) = 'integer'")] + [TestCase("SELECT t FROM v INTERSECT SELECT t FROM v WHERE t > 'a'")] + [TestCase("SELECT t FROM v EXCEPT SELECT t FROM v WHERE t > 'b'")] + public void CompoundSetOperatorsWithSingleSpellingGroupsMatchSqlite(string sql) + => Run(sql).Should().Equal(RunSqlite(sql)); + + [TestCase("SELECT count(DISTINCT x), count(DISTINCT t), count(DISTINCT n), count(DISTINCT r), count(DISTINCT c) FROM v")] + [TestCase("SELECT sum(DISTINCT x), group_concat(DISTINCT n) FROM (SELECT * FROM v ORDER BY rowid)")] + [TestCase("SELECT count(DISTINCT t COLLATE NOCASE), count(DISTINCT t COLLATE RTRIM) FROM v")] + public void DistinctAggregatesMatchSqlite(string sql) + => Run(sql).Should().Equal(RunSqlite(sql)); + + [TestCase("WITH RECURSIVE r(k) AS (SELECT n FROM v UNION SELECT upper(k) FROM r WHERE length(k) < 4) SELECT k FROM r")] + [TestCase("WITH RECURSIVE r(i, k) AS (SELECT 1, 1 UNION SELECT i + 1, (i + 1) % 3 * 1.0 FROM r WHERE i < 30) SELECT k FROM r")] + public void RecursiveUnionKeepsTheSameRowsAsSqlite(string sql) + => Run(sql).Should().Equal(RunSqlite(sql)); + + [Test] + public void LargeDerivedDistinctIsNotQuadratic() + { + using var connection = Open(); + Execute(connection, "CREATE TABLE big(pad TEXT)"); + Execute(connection, "WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM n WHERE i < 20000) INSERT INTO big SELECT printf('%0200d', i % 15000) FROM n"); + + var watch = System.Diagnostics.Stopwatch.StartNew(); + using var command = connection.CreateCommand(); + command.CommandText = "SELECT count(*) FROM (SELECT DISTINCT pad FROM big)"; + var count = Convert.ToInt64(command.ExecuteScalar()); + watch.Stop(); + + count.Should().Be(15000); + watch.Elapsed.Should().BeLessThan( + TimeSpan.FromSeconds(10), + "DISTINCT compared every row with every kept row and took ~17 s at 20k rows"); + } + + private static List Run(string sql) + { + using var connection = Open(); + foreach (var statement in Setup.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + Execute(connection, statement); + return Read(connection.CreateCommand(), sql); + } + + private static List RunSqlite(string sql) + { + using var connection = new MsData.SqliteConnection("Data Source=:memory:"); + connection.Open(); + connection.CreateCollation("reverse_text", static (left, right) => string.CompareOrdinal(right, left)); + foreach (var statement in Setup.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + using var command = connection.CreateCommand(); + command.CommandText = statement; + command.ExecuteNonQuery(); + } + + return Read(connection.CreateCommand(), sql); + } + + private static List Read(System.Data.Common.DbCommand command, string sql) + { + using (command) + { + command.CommandText = sql; + using var reader = command.ExecuteReader(); + var rows = new List(); + while (reader.Read()) + { + var values = new string[reader.FieldCount]; + for (var index = 0; index < values.Length; index++) + values[index] = Describe(reader.GetValue(index)); + rows.Add(string.Join("|", values)); + } + + rows.Sort(StringComparer.Ordinal); + return rows; + } + } + + // Type-tagged, round-trippable rendering so the surviving representative of each equal + // group is compared (1 vs 1.0, text vs blob), not just the number of groups. + private static string Describe(object value) => value switch + { + DBNull => "null", + long integer => $"i:{integer}", + // The managed engine does not preserve a stored -0.0's sign (a separate, pre-existing + // representation difference); DISTINCT treats both zeros as equal either way. + double real => $"r:{(real == 0 ? 0d : real).ToString("R", System.Globalization.CultureInfo.InvariantCulture)}", + string text => $"t:{text}", + byte[] blob => $"b:{Convert.ToHexString(blob)}", + _ => $"?:{value}", + }; + + private static SqliteConnection Open() + { + var connection = new SqliteConnection("Data Source=:memory:"); + connection.Open(); + connection.CreateCollation("reverse_text", static (left, right) => string.CompareOrdinal(right, left)); + return connection; + } + + private static void Execute(SqliteConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + command.ExecuteNonQuery(); + } +}