diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 0de8a646cfe2..b8c2ed479fd6 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -1247,6 +1247,7 @@ static bool shall_skip_gtids(const Log_event *ev) { case mysql::binlog::event::FORMAT_DESCRIPTION_EVENT: case mysql::binlog::event::ROTATE_EVENT: case mysql::binlog::event::IGNORABLE_LOG_EVENT: + case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT: case mysql::binlog::event::INCIDENT_EVENT: filtered = false; break; diff --git a/design/binary-log/683-binlog-large-transaction-optimization/high-level-design.md b/design/binary-log/683-binlog-large-transaction-optimization/high-level-design.md new file mode 100644 index 000000000000..88b8fbb9b766 --- /dev/null +++ b/design/binary-log/683-binlog-large-transaction-optimization/high-level-design.md @@ -0,0 +1,230 @@ +# Binlog Optimization for Large Transaction - Proposal + +**Main Contributor:** Xueting(Hillary) Wu\ +**Secondary Contributors:** Omar Farhat, Ali Bhagat, Michael Dowling, Bob Yang, Kevin Yang, Derrick Yang, Zongkai Xia + +## 1. High-Level Description + +### Executive summary + +After this feature is implemented, MySQL server will stop exhibiting commit stalls when committing large transactions and will also achieve constant time recovery for large transactions. This allows applications with workloads containing large transactions to have an improved MySQL experience. + +Large transactions present a significant performance challenge for MySQL's binary logging subsystem. When a transaction exceeds the in-memory binlog cache (controlled by `binlog-cache-size`), events spill to a local temp file. At commit time, copying the entire file contents to the binlog file while holding `LOCK_log` takes longer as the transaction size increases. This inflates commit latency, stalls concurrent commits, and widens the crash window. If the server crashes while copying the transaction, recovery must scan the entire binlog file to identify incomplete transactions, a process that can take hours for multi-gigabyte transactions.\ +\ +We propose an optimization where large transactions spill binlog events directly to a dedicated temp file that is structurally identical to a binlog file, and finalize the commit by promoting that file into the binlog sequence, eliminating the bottleneck entirely. Crash recovery also becomes constant time regardless of the last binlog file's size, since recovery skips scanning the large transaction's payload.\ +\ +This optimization has been running in production in Amazon Aurora MySQL since 2020, where it reduced large transaction commit latency by 6x and brought P99 binlog crash recovery time to under one minute. MariaDB, inspired by Aurora MySQL, implemented a similar optimization ([MDEV-32014](https://github.com/MariaDB/server/commit/fba09f8ccb9ad55a86349bc9586b048c775d5205)), validating the design.\ + + +### User / developer stories + +As a MySQL user performing workloads that produce large transactions, such as bulk data loads or large batch updates, I want binary logging to not impose an additional performance penalty on large transaction commits, so that a large transaction's commit does not delay the commits of other concurrent transactions. + +As a MySQL user performing workloads that produce large transactions, such as bulk data loads or large batch updates, I want binlog recovery of a large transaction to stay fast, so that my downtime stays short and predictable regardless of transaction size. + +As a MySQL user running a mixed workload, I want large transactions to benefit from the optimization while non large transactions continue committing with the same latency and behavior, so that I can adopt this feature without risk to my everyday workload. + +As a MySQL user, I want my applications to remain unchanged when this feature is enabled. The binary log produced by this optimization must remain fully compatible with my replicas, point in time recovery, and binlog based tools, so that I can adopt it without changing my replication topology or existing workflows. + +As a MySQL user, I want observability into the optimization, including whether it is active and how often it is triggered, so that I can confirm it is behaving as expected and correlate it with the performance I am seeing. + +### In scope + +- Introduce a new commit code path for large transactions, which can execute concurrently alongside transactions using the standard commit path. +- Improve recovery performance through the introduction of a new event, `Large_transaction_header_event`, that allows recovery to skip scanning the transaction payload. +- Provide a knob to enable and disable the optimization. +- Expose observability metrics for the feature. +- Emit error logs when the optimization encounters failures. + +### Out of scope/Limitations + +Setting `binlog_format` to `STATEMENT` or `MIXED`, enabling binlog encryption, enabling group replication or enabling binlog transaction compression is not supported by the optimization. When a transaction that would otherwise qualify for the optimization (its spilled size exceeds `binlog_large_transaction_optimization_threshold`) runs under any of these configurations, it falls back to the standard commit path, the server emits a warning to the error log indicating that the optimization is disabled for that transaction, and the `binlog_large_transaction_optimization_missed_count` status variable is incremented. + +## 2. High-Level Design + +The idea behind this optimization is simple: when the in-memory binlog cache spills to a temp file on disk, instead of copying that temp file back into the binlog at commit, we make the temp file look like a real binlog file from the start and, at commit, promote it directly into the binlog sequence. + +The expensive copy under `LOCK_log` disappears. The standard code path also has a crash-recovery cost: its lengthy copy widens the window in which a restart or crash can occur during commit, and when a crash does happen, recovery must scan the entire transaction to locate its end, making recovery time proportional to transaction size. The optimization does the opposite. It shrinks that window, and even when a large transaction is present, recovery can skip scanning its payload entirely, keeping recovery time constant regardless of transaction size. + +### Write and Commit Code Path + +As a transaction runs, its binlog events are generated and held in the binlog cache. When the cache grows past `binlog_cache_size`, it spills to a temp file on disk. This temp file contains one or more binlog events belonging to that transaction. This already happens today. Our approach reuses that spilled file directly: instead of copying it into the binlog at commit, we treat the spilled file itself as a binlog file and promote it into the sequence.\ +\ +There is a problem with treating it as-is. Every binlog event header carries a `log_pos`, the offset of the end of that event within its binlog file. Today, when events are written to the cache and spilled to the temp file, `log_pos` is not filled in, because an event's final position is not known until commit, when the cache is copied into the active binlog file and each event's `log_pos` is computed against its destination offset. The copy is where positions get assigned.\ +\ +So the spilled events are not finalized until the positions are filled-in. To promote it as-is, the optimization would have to revisit every event in the file and rewrite its `log_pos`. This is an expensive operation and will take O(n). This will likely cause the transaction execution to experience a stall when `binlog_large_transaction_optimization_threshold` is exceeded.\ +\ +Instead, because the promoted binlog file would always have the large transaction as the first transaction, we can assign each event its `log_pos` as it spills. The first event has to begin right after the file's header events: the `Format_description_log_event`, the `Previous_gtids_log_event`, and the transaction's `Gtid_log_event`. We do not know their exact combined size when spilling starts, so we reserve an estimate (the current `Previous_gtids_log_event` size plus about 32 KB of headroom) and assume the first event begins at that offset.\ +\ +Later on, at commit, the real header events are written into the reserved region. If the estimate was exact, they fill it. If we overestimated, the headers are smaller than the reserved region and leave a gap. To keep the file contiguous and preserve the `log_pos` values we already assigned, we fill that gap with a `Large_transaction_header_event` sized to occupy exactly the remaining bytes, placed so that the `Gtid_log_event` ends right at the reserved offset and the first transaction event begins there.\ +\ +When a large transaction reaches commit, its events are already in the temp file in final binlog form, so the commit stage copies no data. It performs a fixed amount of work regardless of transaction size. + +``` +1. Sync the temp file to durable storage (body). +2. Acquire LOCK_log. +3. Compute the binlog sequence number for the temp file. +4. Write the header events into the reserved region of the temp file. +5. Sync the temp file to durable storage. +6. Record the computed binlog sequence number in the purge_index_file. +7. Promote the temp file into the binlog sequence +8. Append a Rotate event to the currently active binlog file, pointing to the promoted file. +9. Add the promoted file to the index file. +10. Delete promoted file entry from purge_index_file. +11. Release LOCK_log. +12. Commit the large transaction in InnoDB. +13. If the promoted binlog file exceeds max_binlog_size, rotate to a new binlog file. +``` + +Steps 6 through 10 are the same sequence the server already performs for an ordinary binlog rotation. The optimization code path deliberately reuses that existing rotation mechanism so the change stays small and builds on well-tested code. + +#### Anatomy of a promoted binlog file + +After promotion, the new binlog file is a normal binlog file. Its only distinguishing feature is the `Large_transaction_header_event` at the front and the reserved region it sits in:\ + + +``` ++---------------------------------------------------------+ +| Binlog magic number | +| Format_description_log_event | +| Previous_gtids_log_event | +| Large_transaction_header_event (ignorable) | # new event +| Gtid_log_event | +| Begin | +| ... | +| Terminating event (internal or external XID) | ++---------------------------------------------------------+ +``` + +The fixed header events occupy the front of the reserved region, the `Large_transaction_header_event` consumes whatever they did not use, and the `Gtid_log_event` follows it so that the transaction body begins at exactly the offset assumed while the events were spilling. The wire format of `Large_transaction_header_event` is shown below: [LTO-V3-R2] + +``` + +----------------------------------------------------------------+ + | Common Binlog Event Header (19 bytes) | + +----------------------------------------------------------------+ + | timestamp (4) | type_code (1) | server_id (4) | + | event_length (4) | next_position (4) | flags (2) | + | | 0x80 = IGNORABLE_F | + +----------------------------------------------------------------+ + | Post-header (0 bytes) | + +----------------------------------------------------------------+ + | Event Data Body (variable) | + +---------+------------------+------------------------+----------+ + | Version | XID Event Offset | Terminating Event Type | Padding | + | (1 byte)| (8 bytes) | (1 byte) | (var) | + +---------+------------------+------------------------+----------+ +``` + +Term Event Type (1 byte): The `type_code` of the transaction's terminating event located at XID Event Offset. Valid values are: + +- 0x02(2) - `QUERY_EVENT`: DDL transaction. +- 0x10(16) - `XID_EVENT`: DML transaction committed via implicit XA. +- 0x26(38) - `XA_PREPARE_LOG_EVENT`: XA PREPARE or XA COMMIT ONE PHASE transaction. + +During crash recovery, the engine seeks to XID Event Offset and compares the event header's `type_code` against the stored Terminating Event Type. A match confirms the transaction is complete and its GTID is added to `gtid_executed`; a mismatch indicates corruption or incomplete write, and the transaction is rolled back. + +#### Reserved bytes + +Because we assign each event its `log_pos` as it spills, we must finalize the offset where the transaction's first event begins before we know the exact size of the header events that will precede it. We reserve a padding region at the top of the temp file and place the first event just past it. The reservation must hold everything a binlog file begins with: the magic number (4 bytes), the `Format_description_log_event` (about 120 bytes, effectively fixed) and the `Previous_gtids_log_event`, which is the only part that grows with the deployment, `Large_transaction_header_event` and `Gtid_log_event`.\ +\ +The `Previous_gtids_log_event` encodes the set of GTIDs already present in the binlog. Each GTID records each source UUID with a single contiguous range costs 40 bytes (16 for the UUID, 8 for the range count, 16 for one range), plus 8 bytes for the overall count. A deployment that has seen 10 distinct sources therefore encodes to about 408 bytes, growing with additional ranges per source or with tagged GTIDs.\ +\ +Because this size is deployment-dependent and changes over time, the reservation is not a static number. We size it as the current `Previous_gtids_log_event` size plus 32 KB of headroom. The current size captures the actual GTID state, and the 32 KB absorbs any growth between the moment the reservation is sized and the moment the transaction commits, along with the small fixed events. The unused portion is padded into the file by the `Large_transaction_header_event`. This overhead is negligible, since a transaction executing the optimization's code path has by definition exceeded `binlog_large_transaction_optimization_threshold` (at least 10 MB), so a few KB of padding is a tiny fraction of the file. If the header still does not fit the reservation, the transaction falls back to the standard code path. + +#### Integration with Binary Log Group Commit + +When a spilled transaction commits, it takes a dedicated single-commit path that promotes the temp file to a real binlog file. This path acquires and holds `LOCK_log` during promotion, which is the same lock that the standard group commit pipeline acquires during its flush and commit stages, which guarantees that no standard commit, no other spilled commit, and no binlog rotation can interleave with the promotion. + +**GTID and Dependency Tracking.** GTID is assigned during the promotion while `LOCK_log` is held. Furthermore, because promoting a spilled transaction creates a new binlog file, the logical clock (used by the replica's parallel applier to determine which transactions can run concurrently) is reset. The spilled transaction is assigned sequence values that indicate the start of a new dependency group. This tells the replica that all prior transactions must finish before this one can be applied, effectively a serialization point. + +**Fallback**: When the optimization falls back to the standard commit, the already spilled temp file content is copied back into the active binlog file, following the standard code path. + +#### Optimized Recovery + +Crash recovery proceeds exactly as it does today, with one addition. When recovery encounters a `Large_transaction_header_event`, it learns two things at once: (1) the transaction is complete and was committed through the optimization's code path, and (2) the offset of the terminating event. Recovery uses that offset to seek directly to the terminating event, collect its XID, and continue, without reading the transaction body. A multi-gigabyte transaction is skipped in a single seek instead of a full scan, which is what makes recovery time independent of transaction size. Everything else in the file is recovered normally. + +**Validation**: Before seeking, the offset is checked against the file length. If it exceeds the file length, the fast path is abandoned and the reader continues scanning sequentially from its current position. The event at the seeked position is read through the standard reader, which validates the event header, length, and checksum. Any subsequent transactions in the same file are scanned sequentially after the seek, and a partial tail from a crash is handled identically to the non-optimized case. A malformed or truncated target event terminates scanning of that file, and the transaction's GTID is not added to the executed set, as the transaction cannot be confirmed complete. + +#### Crash-Safety Analysis + +The recovery behavior is aligned with all major operations that interact with the index file, including rotate, purge, and resetting binary logs and GTIDs. Note that the complete binlog content, header and body, is durable before writing the file sequence number into the `purge_index_file` (step 6). Every case below therefore rolls forward or back against a fully durable file. Below is the crash-safety analysis based on the commit code path steps above. + +- If MySQL restarts or crashes before the new file name is recorded in `purge_index_file` (steps 1 - 5), then nothing was written to `purge_index_file`, no file exists under the new name, and nothing is in the index. The temp file is cleaned from `#binlog_temp_files`, and InnoDB finds no XID in binlog and rolls the transaction back. +- If MySQL restarts or crashes after recording the file in `purge_index_file` but before adding it to the index (steps 6 - 8), then the file is deleted on recovery following the mechanism that exists today; the trailing Rotate event in the previously active file is handled exactly as it is for an interrupted ordinary rotation. InnoDB finds no XID in binlog and rolls the transaction back. +- If MySQL restarts or crashes after adding the file to the binlog index (step 9) but before completing the remaining steps, the file is already discoverable in the index. Recovery scans the binlog, finds the XID, and InnoDB rolls the transaction forward. The `purge_index_file` entry, if still present, is cleaned up as a no-op since the file is already indexed. +- If MySQL restarts or crashes after the InnoDB commit while the promoted file is the active binlog file (not yet rotated away by step 13), then the transaction is durable in both the binlog and InnoDB. Recovery encounters the promoted file during its normal scan of the last binlog file and collects the transaction's XID using the optimized recovery algorithm, seeking straight to the terminating event through the `Large_transaction_header_event` instead of reading the transaction body. This lets ordinary crash recovery skip a multi-gigabyte transaction in a single seek, keeping recovery of the last file constant-time regardless of transaction size. +- If MySQL restarts or crashes after the promoted file has already been rotated away, the promoted file is an earlier, fully durable binlog file that recovery does not need to reprocess. The transaction is committed. + +#### Filesystem Durability and Error Handling + +The promotion reuses the same filesystem durability mechanisms as standard binlog rotation (index file sync, purge index sync, directory fsync). No new durability ordering is introduced. If any step fails, the commit is aborted with a flush error, the cache is reset, and the server either logs an error and continues (if `binlog_error_action = IGNORE_ERROR`) or aborts the server process (if `binlog_error_action = ABORT_SERVER`). No partial state is left visible to clients because the `LOCK_log` is held throughout and the engine commit has not yet occurred. + +#### Performance Advantage + +The optimization improves the two costs that scale with transaction size today: commit latency and crash recovery time.\ +\ +At commit, the standard path does O(n) work under `LOCK_log`, because it copies the entire binlog cache into the active file. The promotion path does a fixed amount of work, since the data was written to the temp file during the transaction's lifetime. The large transaction's own commit latency stops growing with its size, and because the commit stage no longer holds `LOCK_log` for the duration of a multi-gigabyte copy, the transactions queued behind it are no longer delayed proportionally to its size.\ +\ +At recovery, the standard path may have to scan an entire multi-gigabyte transaction to find where it ends, which is an O(n) operation. The optimization's code path seeks past it using the recorded offset, making recovery constant-time regardless of transaction size.\ + +## 3. User Interface + +### New system variables + +#### `binlog_large_transaction_optimization_enabled` + +| Property | Value | +| --- | --- | +| **Values** | boolean (ON \| OFF) | +| **Default** | ON | +| **Scope** | GLOBAL | +| **Dynamic** | Yes. Takes effect for transactions that begin after the change; in-flight transactions are unaffected. | +| **Replicated** (written to the binary log) | No | +| **Persist** | PERSIST, PERSIST_ONLY | +| **Command line** | Yes | +| **Privileges required** | `SYSTEM_VARIABLES_ADMIN` | + +**Description**: Enables the large transaction optimization, which keeps large-transaction commit latency low, avoids stalling concurrent commits, and keeps crash recovery fast regardless of transaction size. When ON (the default), a transaction whose spilled size exceeds `binlog_large_transaction_optimization_threshold` is committed through the optimized path. When OFF, all transactions commit through the standard code path and `binlog_large_transaction_optimization_threshold` is ignored. + +#### `binlog_large_transaction_optimization_threshold` + +| Property | Value | +| --- | --- | +| **Values** | unsigned integer, bytes. Minimum 10485760 (10 MB); values below the minimum are rejected. Range [10485760 - 2^64-1]. | +| **Default** | 134217728 (128 MB) | +| **Scope** | GLOBAL | +| **Dynamic** | Yes. Takes effect for transactions that begin after the change. | +| **Replicated** (written to the binary log) | No | +| **Persist** | PERSIST, PERSIST_ONLY | +| **Command line** | Yes | +| **Privileges required** | `SYSTEM_VARIABLES_ADMIN` | + +**Description**: The spilled size above which a transaction qualifies for the large transaction optimization. Used together with `binlog_large_transaction_optimization_enabled`: the optimization must be enabled for this threshold to take effect, and it has no effect while `binlog_large_transaction_optimization_enabled` is OFF. + +### New status variables (Observability) + +#### `binlog_large_transaction_optimization_count` + +| Property | Value | +| --- | --- | +| **Values** | unsigned integer (monotonic counter) | +| **Scope** | GLOBAL | + +**Description**: The number of transactions committed through the large transaction optimization since server startup. + +#### `binlog_large_transaction_optimization_missed_count` + +| Property | Value | +| --- | --- | +| **Values** | unsigned integer (monotonic counter) | +| **Scope** | GLOBAL | + +**Description**: The number of transactions that exceeded `binlog_large_transaction_optimization_threshold` but could not be optimized because they were incompatible with the optimization, since server startup. + +### Security Context + +The `#binlog_temp_files` directory is server managed storage of uncommitted binlog data. Files within it contain binlog equivalent data and use the permissions required by NFR12. Cleanup at startup follows the existing prefix filtered deletion pattern to ensure only files managed by the optimization are removed. + +No new SQL privilege is introduced. Both new system variables `binlog_large_transaction_optimization_enabled` and `binlog_large_transaction_optimization_threshold` are controlled by the existing `SYSTEM_VARIABLES_ADMIN` privilege. + +--- \ No newline at end of file diff --git a/design/binary-log/683-binlog-large-transaction-optimization/low-level-design.md b/design/binary-log/683-binlog-large-transaction-optimization/low-level-design.md new file mode 100644 index 000000000000..7372062009cf --- /dev/null +++ b/design/binary-log/683-binlog-large-transaction-optimization/low-level-design.md @@ -0,0 +1,338 @@ +# Low-Level Design + +The low-level design describes how the large transaction optimization (BOLT) is implemented in the server, complementing the high-level design, which covers the mechanism and rationale. It is organized by implementation area: the infrastructure the optimization builds on (including its system and status variables), followed by the commit and recovery paths, and finally testing. Throughout, it references the requirements (FR/NFR). + +## Section 1: Infra Setup + +This section covers the infrastructure the optimization is built on, including the new binary log event `Large_transaction_header_event`, the changes that let a binlog cache's spill file become a binary log file, the dedicated directory those files live in, and the system and status variables that control and observe it. This infra provides the setup needed to implement the commit and recovery paths, which are covered in Section 2. + +### System and Status Variables + +Two global, dynamic system variables control the optimization, namely `binlog_large_transaction_optimization_enabled` (boolean, default ON) and `binlog_large_transaction_optimization_threshold` (bytes, default 128 MB, minimum 10 MB). Their full specification (default, scope, privileges, persistence) is in the User Interface section of the HLD, so only the implementation is described here. + +Both are defined in `sys_vars.cc` as GLOBAL, `NOT_IN_BINLOG` variables. The 10 MB minimum (FR8) is enforced by the threshold's `VALID_RANGE`. The one implementation subtlety is the interaction with `binlog_cache_size` (FR9.3 to FR9.5). While the optimization is enabled, the effective threshold is held at `max(threshold, binlog_cache_size)`. This is enforced in three places: + +- At startup, `update_binlog_large_transaction_optimization_threshold()` (called from `init_common_variables()`). +- When the threshold is set (`check_binlog_large_transaction_optimization_threshold`). +- When `binlog_cache_size` is set (`fix_binlog_cache_size`). + +In all three cases, if the requested threshold is below `binlog_cache_size` it is raised to match, and, when the optimization is enabled, a warning is emitted to both the client and the error log (`ER_BINLOG_BOLT_THRESHOLD_ADJUSTED`, `ER_BINLOG_BOLT_THRESHOLD_ADJUSTED_SQL_WARNING`). + +In addition to the system variables, we also introduce two atomic status counters that report how often the optimization runs: `binlog_large_transaction_optimization_count` is incremented once per successful promotion, and `binlog_large_transaction_optimization_missed_count` is incremented during fallback. Both are exposed through `SHOW GLOBAL STATUS` and `performance_schema.global_status`. + +### `Large_transaction_header_event` + +A new event type, `LARGE_TRANSACTION_HEADER_EVENT = 43`, is added to `Log_event_type`. As described in the high-level design, it serves two purposes: it records where the transaction's terminating event sits, so recovery can seek to it, and its padding fills whatever the fixed header events leave unused in the reserved region. Its body is `version` (1 byte), `terminating_event_offset` (8 bytes), `terminating_event_type` (1 byte), and variable-length padding. + +Following the usual MySQL event split, it is implemented as two classes: + +- `mysql::binlog::event::Large_transaction_header_event`, the binlog-events library class, handles deserialization. It exposes the format constants (`kVersion = 1`, `kFixedBodyLength = 1 + 8 + 1 = 10`), validates the version, reads the terminating event's offset and type, and treats the remainder as padding. +- `Large_transaction_header_log_event`, the server class, inherits from the library event and from `Log_event` and handles serialization (`write_data_body`), `pack_info`/`print`, and `do_apply_event`, which is a no-op since the event carries recovery metadata only. Its constructor always sets `LOG_EVENT_IGNORABLE_F`, so the event is transparent to anything that does not understand it, preserving replication and tooling compatibility (NFR8, NFR10). + +### Promotable Binlog Cache + +During query processing, the server generates the query's binlog events and writes them to the per-thread cache (`binlog_cache_data`). The first write to the transaction cache latches the BOLT decision for the whole transaction (`latch_large_trx_optimization`), which captures the value of the knob and threshold once, so any mid-transaction change to those settings is ignored. At the same time, when the knob is enabled, it marks the cache's future spill file as a named file (it sets `IO_CACHE::named_file`), which is what later allows the file to be renamed on the filesystem and promoted into a binary log file. If that flag is left unset (the knob was off, or this is the statement cache), the spill file is created as an anonymous, unlinked file with no name in the filesystem, so it can never be renamed or promoted. + +The current implementation of binlog cache is layered: `binlog_cache_data` (event buffer) -> `Binlog_cache_storage` (byte-container facade) -> `IO_CACHE_binlog_cache_storage` (mysys-backed implementation over one `IO_CACHE`). BOLT changes three things in this path: + +- **Where the binlog spill file lives.** While the physical file is still created lazily on the first spill by the existing `IO_CACHE`, it is created in this predefined temp directory (`Binlog_cache_storage::open` passes `binlog_temp_files_dir.path()`) rather than the general temp dir. This holds whether or not the BOLT knob is on. Every other `IO_CACHE` user (for example filesort and internal temporary tables) is unchanged and still spills anonymously into `--tmpdir`. +- **A reserved region at the front.** Opening the cache also shifts its write start position past a reserved header region at the front of the file (`IO_CACHE_binlog_cache_storage::open`). Every binlog cache reserves the region even when the BOLT knob is off. A cache that is never promoted simply leaves the region empty and never copies it into the binary log, so it is invisible in the output. + + - Why is the region reserved unconditionally, rather than by checking the knob at cache initialization: the cache is initialized once per session and reset between transactions. If the decision on whether to reserve space at init were based on the BOLT knob, we would need to ensure that reset checks the same value that init checked. Since the knob is a dynamic global that can change at any time, gating on it would effectively fix the knob per connection: enabling the optimization at runtime would have no effect on existing sessions until they reconnect. + + + +- **Naming the file for promotion.** The first write that actually spills to disk renames the file from its generic mkstemp name to the `bolt__` form (`IO_CACHE_binlog_cache_storage::write` -> `rename_spilled_file`), but only for a cache marked as named. A cache that was not marked named (the knob was off, or it is the statement cache) keeps its anonymous, unlinked file and is never promoted. + + - A `bolt_` name only records that the binary log cache created the file; it is not a promise the file will be promoted. Every spilled transaction cache gets the name while the optimization is enabled, including transactions that eventually commit through the standard code path because they fall below the threshold or hit a fallback condition at commit. + +The first event offset written into the spilled file does not start at zero. Instead, the cache reserves a block of bytes at the front of the file when it is opened (`binlog_cache_data::open`), sized as the current `Previous_gtids` size plus 32 KB of headroom, rounded up to a 64 KB quantum (`get_binlog_temp_file_reserved_bytes`). The cache then places all of its data after this reserved region. + +### Temp-files Directory + +Every binlog cache spill file is created in a dedicated directory next to the binary logs, `#binlog_temp_files`. At startup, `Binlog_temp_files_dir::init()` creates the directory, or, if it already exists, clears leftover `bolt_` files from a previous run (executed by `temp_files_dir_clear_files`). The cleanup rejects a symlinked directory and does not delete anything that is not a regular file it recognizes by the `bolt_` naming (which is validated through the function `is_bolt_temp_file`). + +A promoted file becomes a real binary log file, so it must carry the same permissions a normal binary log file would. Normal binlog files are created honoring `my_umask` via `my_open`, but the temp file is created by mkstemp (`mysql_file_create_temp`), which forces 0600 and ignores the umask. BOLT therefore reconstructs the umask derived permission set (`binlog_temp_file_permissions`) and calls `my_chmod` on the promoted file to match. This reproduces the regular binlog file permissions rather than inventing a new scheme. + +## Section 2: Commit and Recovery + +This section walks the three code paths a transaction exercises around commit: the **write code path**, where events are generated and staged in the binlog cache; the **commit code path**, where the transaction is either promoted or committed through the standard path; and the **recovery code path**, where a promoted file is read back after a crash. Each is described below with a diagram of its flow. + +### Write code path + +``` + Event generated for the transaction + | + v + write to binlog_cache_data + (raw event) + | + exceeds binlog_cache_size? + / \ + no yes + | | + v v + stays in memory spill to temp file in + #binlog_temp_files + + log_pos and checksum are attached when the cache is + written out to a binary log file: + standard path -> during the copy at commit + (Binlog_event_writer) + BOLT on -> as content is written to the spill + file, after the reserved region +``` + +As a transaction runs, its events are written into the transaction cache (`binlog_cache_data`) as raw events. `log_pos` and the checksum are attached when the cached bytes are written out to a binary log file, not while they sit in the cache. + +On the standard path this happens at commit: `Binlog_event_writer` copies the cache into the active binary log and stamps each event's `log_pos` (its destination offset) and checksum as it copies. + +With BOLT on, the spilled file itself becomes the binary log file, so there is no later copy step. `log_pos` and checksum are attached as the cache content is written to the spill file, with events placed after the reserved header region (Section 1). Because the reserved region fixes where the first event begins, each `log_pos` is final and the file is already in binary-log shape, so promotion at commit needs no rewrite. + +#### Checksum overhead + +Standord binary logging computes an event's CRC once, when the cache is copied into the active binary log (the same pass that also assigns its final `log_pos`); cached events carry no CRC. BOLT adds a new point at which the CRC is calculated. With BOLT enabled, the CRC is attached when writing to the transaction cache, which shifts the computation earlier and changes the existing behavior, where now the CRC can be computed twice. This happens in three of the four cases: + +- Small transactions that never spill: computed twice, once into the cache and once into the binary log. +- Transactions that spill but stay below the promotion threshold (128 MB by default): computed twice, once at the spill write and again when the standard commit path copies the spilled file into the active binary log and reassigns `log_pos`. +- Large transactions that spills and commits through BOLT computes the CRC exactly once, when writing into the cache. +- Transactions that spill above the threshold but fall back to the standard path: computed twice, for the same reason. + +We chose this implementation because deferring the checksum computation until spill time would require substantial changes. The main blocker is savepoint handling. When a savepoint is set, the current position in the cache is recorded. If the checksum were added later, at spill time, that recorded position would no longer match the file: the position was recorded without the checksum bytes, while the file now contains checksums. This gap is fixable, but the position is stored in two places (the savepoint's slot in the SQL layer, and as a key in the cache's own state map), so correcting it means we first need to determine what the correction should be and then apply it in both places, which is a substantial change. + +**Interaction with compression** + +Compression also decides whether a checksum is written into the cache, because events inside a compressed payload must not carry one. When `binlog_transaction_compression` is on at the transaction's first event, `BINLOG_CHECKSUM_ALG_OFF` is recorded and no CRC enters the cache. Compression and promotion are then mutually exclusive outcomes: a transaction that does get compressed is not promotable, and falls back with `kCompression`. As a defence, `shall_compress()` also declines compression outright if the cache already holds checksums, since that combination would produce a malformed payload; that requires compression to have been enabled after the first event, which the session-variable semantics prevent, so it is unreachable in practice. + +This gives three cases: + +1. **Compression on at the first event, and compression succeeds.** No checksum is in the cache, and BOLT rejects the transaction because it is compressed. +2. **Compression on at the first event, but** `shall_compress()`** declines for another reason.** The cache holds no checksum, and promotion requires the recorded algorithm to match `binlog_checksum`. With `NONE` the two agree, so the transaction can commit through BOLT if the remaining checks pass, and the promoted file is checksum-free, matching the configuration. With `CRC32` they disagree, so BOLT rejects it and it commits through the standard path, where the checksum is computed while copying into the binary log. +3. **Compression off throughout.** `shall_compress()` returns at its first check, and the defensive check above is never reached. + +#### Durability policy + +In the standard commit code path, every transaction prepares with `HA_IGNORE_DURABILITY`, so the engine writes its prepare record to its own log (for example InnoDB's redo log) without persisting it during prepare. That debt is repaid once per group in the standard commit path's binlog flush stage, which calls `ha_flush_logs(true)` before any cache reaches the binary log. + +BOLT bypasses that group flush stage, so it repays the same debt itself, with an unconditional `ha_flush_logs(true)` in `commit_large_transaction()`, before the promotion. + +Unlike the standard path, BOLT takes this flush with no binlog lock (no `LOCK_log` and no queue lock) held. No lock is needed as the flush and the promotion happen in the same thread, so this transaction's prepare record is on disk before the transaction reaches the binary log. + +The durability property is deliberately not chosen per transaction during prepare, because it is not yet known at that point whether the transaction will be promoted. During prepare, the terminal event has not been appended, so the cache is short of its final size, and a transaction just below the threshold at that point can still cross it and be promoted. + +### Commit code path + +``` + COMMIT: get_cache_for_large_trx_commit() + | + +---------------+-----------------+ + | | + spilled, > threshold, otherwise + and eligible (6 checks) | + | v + v ordered_commit() + commit_large_transaction() (standard group commit: + | copy cache into active + | binlog) + v ^ + header fits and | + GTIDs persisted? ------- No ------------+ + | (fallback) + Yes + | + v + Promote: sync body, write header into reserved region, + register in purge index, rename into binlog sequence, + Rotate event on old file, add to main index + | + v + Engine commit (finish_commit); + rotate if > max_binlog_size +``` + +When `get_cache_for_large_trx_commit()` returns a cache (§2.4), `commit_large_transaction()` promotes it using the optimization commit code path. It performs the HLD's 13 commit steps; steps 6 to 10 reuse MySQL's existing crash-safe rotation (register in the purge index -> rename -> Rotate event -> add to the main index -> clear the purge index), so BOLT does not reimplement rotation. The mapping to code: + +``` +// sql/binlog/large_trx_commit.cc commit_large_transaction() +// (comment numbers = the HLD's 13 commit steps) +flush_and_sync_spilled_file(); // 1 make the body durable +init_thd_variables(...); +LOCK_log; wait_for_prep_xids(); LOCK_commit; // 2 acquire locks +generate_new_name(new_name, name); // 3 next binlog file name +persist_gtids_on_rotate(..., &keep_current_binlog); // read-only table -> fallback +write_promoted_binlog_header(..., &fits); // 4,5 write header, sync temp file + if (!fits) goto fallback_to_ordered_commit; +open_purge_index_file / register_create_index_entry + / sync_purge_index_file; // 6 record file in purge index +my_rename(temp_file_name, new_name); + mysql_file_sync(spilled_file()); // 7 promote (rename) + sync +Rotate_log_event r(...); write_event_to_binlog(&r); + m_binlog_file->flush_and_sync(); // 8 chain old file -> new file +open_binlog(..., promoted_file_is_renamed = true); // 9,10 add to index, clear purge index +binlog_large_transaction_optimization_count++; // success counter (under LOCK_log) +unlock LOCK_log; // 11 release log lock +cache_data->reset(true /*preserve_spilled_file*/); +finish_commit(thd); // 12 commit in the engine +unlock LOCK_commit; +if (rotate_if_needed()) ...; // 13 rotate if > max_binlog_size +``` + +Three parts are new or worth calling out: + +Header + LTH sizing (step 4). `write_promoted_binlog_header()` serializes magic + FDE + Previous_gtids, then sizes the `Large_transaction_header_event`'s padding so the following `Gtid_log_event` ends exactly at reserved_bytes, where the first spilled event was placed. If the GTID state grew since spill and the header no longer fits, it sets \*fits = false and BOLT takes a clean fallback (§2.4). + +GTID + dependency reset. A promoted file starts a new binary log file, so the dependency tracker is rotated before the transaction's logical timestamps are generated. This makes the transaction a parallelization barrier (serialization point) for the replica's parallel applier. + +Locking. `LOCK_log` is held from step 2 to 11 and `LOCK_commit` through the engine commit, which is the same locks the standard group-commit flush/commit stages use, so no standard commit, other promotion, or rotation can interleave. `LOCK_log` is released before the engine commit (step 11 before 12) so the next transaction's flush can overlap this one's engine commit, exactly as the standard pipeline does. + +#### Fallback + +BOLT checks whether promotion is possible at two points along the commit path. If any check fails, the transaction falls back to the standard commit path, and for the recorded reasons it increments `binlog_large_transaction_optimization_missed_count`. + +Check 1. Before promotion starts (`get_cache_for_large_trx_commit`, called from `MYSQL_BIN_LOG::commit`). + +First a basic gate decides whether the transaction is a candidate: the knob is on, the cache actually spilled, and the spilled size is above the threshold. For a candidate, all six of the following must hold, otherwise it falls back: + +- the statement cache is empty, +- the transaction has no pending incident, +- it contains only row-format events, +- its spilled file is not encrypted, +- it is not compressed, +- `binlog_checksum` did not change during the transaction. + +Check 2. After promotion has started (inside `commit_large_transaction`), two late conditions can still force a fallback: + +- `gtid_persistence`: persisting the outgoing log's GTIDs to `mysql.gtid_executed` fails because that table is read-only, and +- `reserved_header_space`: the header events no longer fit the reserved region. + +Both jump to `fallback_to_ordered_commit`, which cleanly re-runs the standard commit path; nothing irreversible has happened at either point. + +#### Fallback vs. failure + +All of the above are clean fallbacks: the transaction still commits, just through the standard path. Only genuine I/O errors after the promotion's point of no return fail the transaction; those are handled by `handle_binlog_flush_or_sync_error` per `binlog_error_action` (abort the server, or disable binary logging and let the commit proceed in the engine). + +A fallback re-enters the standard commit path (`MYSQL_BIN_LOG::ordered_commit`), which copies the spilled cache out of the temp file and into the active binary log (`do_write_cache` -> `Binlog_cache_storage::copy_to`). As each event is copied, the writer overrides its `log_pos` for the new position, and the `Gtid_log_event`'s transaction length is recomputed for the destination's checksum setting by `set_trx_length_by_cache_size()` (-> `adjust_trx_length_to_checksum_changes()`). Those are the same functions the normal path, and BOLT's own header write, already use, so there is no BOLT-specific fallback handling. The reserved region at the front of the temp file is skipped by the read cursor, and the temp file is released when the cache is reset. + +### Recovery code path + +``` + Recovery scan of the last binary log file + | + v + Large_transaction_header_event encountered + | + relay-log recovery OR source_verify_checksum ON? + / \ + yes no + | | + v v + ignore the header; validate offset + type, + fall back to ordinary then seek directly to the + sequential scan terminating event + | + v + event at that offset matches + the recorded type? + / \ + yes no + | | + v v + transaction is complete; file marked malformed, + collect its XID / GTID transaction rolled back +``` + +The recovery behavior on a binlog file promoted through the BOLT code path mostly reuses the existing recovery, aside from adding one extra optimization. + +While recovery scans the last binary log file, encountering a `Large_transaction_header_event` lets it skip the transaction body. The header carries the offset and type of the transaction's terminating event. `process_large_trx_header_event()` validates that offset (inside the file, ahead of the current position, with room for an event), seeks to it, and remembers the recorded offset and type. On a later iteration, `validate_large_trx_terminal_event()` (called at the top of the scan loop) confirms that the event now sitting at that offset has the recorded type before the transaction's GTID is added to the executed set. A mismatch marks the file as malformed, and the transaction rolls back. + +Two cases deliberately skip this shortcut and fall back to an ordinary sequential scan: relay log recovery, because the header's offset refers to the source's binary log rather than the relay log (gated by `is_relay_log_recovery()`), and any run with `source_verify_checksum` on. The five crash window cases are covered in the HLD and not repeated here. + +The same validate-then-seek optimization is applied at both places recovery reads a binary log file: when the GTID set is collected (`read_gtids_from_binlog`), and during 2PC crash recovery (`Log_sanitizer`, via the `process_large_trx_header_event()` / `validate_large_trx_terminal_event()` pair named above). + +### Savepoint handling + +BOLT adds no new savepoint logic; it reuses MySQL's existing `ROLLBACK TO SAVEPOINT` handling (`binlog_savepoint_rollback`). If the transaction has touched a non transactional table, the server writes a `ROLLBACK TO SAVEPOINT` event into the cache as it does today; otherwise, it truncates the cache back to the savepoint position (`restore_savepoint`). The only BOLT specific detail is that the truncation is reserved region aware: the storage layer shifts the requested offset past the reserved header bytes (`IO_CACHE_binlog_cache_storage::truncate`), and when the spilled file is later synced (`flush_and_sync_spilled_file`), any stale bytes past the new logical end are cut with `my_chsize`, so a promoted file still ends exactly at its terminating event. Whether the (possibly shrunk) transaction is promoted is decided normally at commit, by comparing its spilled size against the threshold. + +### Refactoring / modularization + +The BOLT commit path shares a lot with the existing group-commit path (THD commit-state setup, GTID-event field derivation) and with the existing binlog cache. Rather than duplicate that logic, and to keep the new code self-contained, the shared and BOLT-specific pieces were split into dedicated files: + +- `transaction_commit_helper.{h,cc}` holds two pieces that the code both commit paths use: `init_thd_variables()` and `Transaction_gtid_header`. + + - `init_thd_variables()` sets up the THD's commit state (commit error, \`next_to_commit\` linkage, and in debug builds the preempt flag). + - \`Transaction_gtid_header\` collects the six values that a transaction's \`Gtid_log_event\` is built from: \`last_committed\`, \`sequence_number\`, the original and immediate commit timestamps, and the original and immediate server versions, which are derived from the THD and the dependency tracker. + + + +- `large_trx_commit.{h,cc}` holds the BOLT-only eligibility and promotion entry points (`get_cache_for_large_trx_commit`, `commit_large_transaction`, `write_promoted_binlog_header`). +- `cache_data.h` holds `binlog_cache_data` and the cache manager, moved out of `binlog.cc` so the separate `large_trx_commit.cc` translation unit can use them. + +## Section 3: Feature compatibility + +### Semi-sync replication (compatible) + +BOLT works with semi synchronous replication: the optimized commit path invokes the same replication hooks as `ordered_commit()`, so a promoted transaction is acknowledged by a replica like any other. Three semi sync hooks sit on the BOLT commit path: `after_flush`, `after_sync`, and `after_commit`. + +- `after_flush`**:** runs inside `promote_spilled_file()`, as the second to last step of the promotion, immediately before the binary log end position is published. This differs slightly from `ordered_commit()`: in `promote_spilled_file`, `after_flush` is not strictly called after the flush stage, but rather after the file has already been promoted, which means it has already been synced. The hook is placed here because: + + - It must run after `open_binlog()` has made the promoted file the active binary log. The hook needs to report the binlog file that contains the transaction, and it reports it through the `log_file_name` variable, which is the server's current binary log, so we must run the hook after the promoted file becomes the active one. + - It must run before the end position is published, as in the standard commit path. +- `after_sync`**:** runs after the `after_flush` observer and before the transaction is committed, with `LOCK_log` released and `LOCK_commit` still held. This is the same lock state and the same relative position as in `ordered_commit()`. +- `after_commit`**:** runs inside `finish_commit()`, under `LOCK_commit`. The standard path releases `LOCK_commit` and acquires `LOCK_after_commit` before running this hook, so BOLT holds the commit lock longer for this hook. This was chosen deliberately, for two reasons: + + - It only extends the lock hold time; it does not introduce any deadlock. The semi sync plugin acquires no server side binlog mutex, so nothing in the acknowledgement path can deadlock against the commit lock. The wait is also bounded: it is a timed wait against `rpl_semi_sync_source_timeout`. + - The extra hold time only materializes when `rpl_semi_sync_source_wait_point` is AFTER_COMMIT, which is the non default setting. + +### Group replication (incompatible) + +BOLT is not compatible with Group Replication and is disabled while Group Replication is running. + +The incompatibility comes from the fact that BOLT writes a checksum into the binlog cache. On the standard path, a transaction's events are serialized into the cache without a checksum, and the checksum is added later, when the cache is copied into the binary log. Unlike the standard path, BOLT promotes the temp file itself into a binary log file, so there is no copy step at which to add the checksum. + +Group Replication reads the same binlog cache through the `before_commit` observer and broadcasts its bytes to the group as is, expecting the cache to have no per event checksum. A cache written with BOLT enabled carries checksums and breaks that contract. + +With Group Replication running, the optimization is off for the transaction from its first event, so no checksum is written into the cache. Two checks guard this: (1) at the first write into the binlog cache, and (2) when deciding whether to commit through the optimized path. The first check ensures that no checksum is written into the cache, while the second catches a transaction that started before Group Replication began. + +There is no inherent challenge in adding BOLT support for Group Replication, but it is out of scope for this PR. + +### Encryption (incompatible) + +BOLT does not work with encryption. A large transaction is promoted only when neither its spilled file is encryption nor the `binlog_encryption` is set to ON: + +| No. | **spilled file** | `binlog_encryption` | **outcome** | +| --- | --- | --- | --- | +| 1 | plaintext | OFF | promote | +| 2 | plaintext | ON | fall back | +| 3 | encrypted | OFF | fall back | +| 4 | encrypted | ON | fall back | + +Two checks guard the correctness of this table. + +The first is in `get_cache_for_large_trx_commit`, during commit, when we decide whether to use BOLT: + +``` +Large_trx_fallback_reason large_trx_commit_blocker() { + if (trx_cache->get_cache()->is_encrypted() || rpl_encryption.is_enabled()) + return Large_trx_fallback_reason::kEncryption; +} +``` + +The second check is in `promote_spilled_file`, which rechecks the system variable again after we acquire `LOCK_log`: + +```cpp +MYSQL_BIN_LOG::Promote_outcome MYSQL_BIN_LOG::promote_spilled_file() { + if (rpl_encryption.is_enabled()) + return fallback_outcome(Large_trx_fallback_reason::kEncryption); +} +``` + +`binlog_encryption` is a dynamic variable, so it can change between `get_cache_for_large_trx_commit` and the actual promotion of the temp file. Rechecking under `LOCK_log` guarantees the correctness of the encryption check, regardless of whether `Rpl_encryption::enable` or BOLT acquires `LOCK_log` first. + +If `Rpl_encryption::enable` acquires `LOCK_log` first, then BOLT is blocked in `commit_large_transaction` before it tries to promote the temp file. By the time BOLT acquires `LOCK_log`, `Rpl_encryption::enable` has completed and `Rpl_encryption::is_enabled` returns true, so BOLT sees that encryption is enabled and falls back to the standard commit code path. + +If BOLT acquires `LOCK_log` first, there are two sub cases: + +- If `Rpl_encryption::enable` has started but not yet finished recovering the master key (`recover_master_key()` is still in progress), `Rpl_encryption::is_enabled` returns false, so BOLT proceeds with writing a plaintext file. The same gap exists in `Binlog_ofile::open()` and BOLT has the same behaviour. +- If `Rpl_encryption::enable` has completed `recover_master_key()` and is blocked on `rotate_logs()` because it needs `LOCK_log` to rotate, then `Rpl_encryption::is_enabled` returns true at this point, so BOLT falls back. + +--- \ No newline at end of file diff --git a/design/binary-log/683-binlog-large-transaction-optimization/performance-benchmark.md b/design/binary-log/683-binlog-large-transaction-optimization/performance-benchmark.md new file mode 100644 index 000000000000..9e222ab015ef --- /dev/null +++ b/design/binary-log/683-binlog-large-transaction-optimization/performance-benchmark.md @@ -0,0 +1,137 @@ +# Public Benchmark Results + +We benchmarked BOLT and validated its performance across commit latency, throughput, and recovery. + +## Experiment 1: Commit Latency + +This experiment validates BOLT's impact on large transaction commit latency. + +### EC2 Setup + +- HDD-backed host: r7i.xlarge with a 1 TiB st1 HDD EBS volume for MySQL data and binary logs. The InnoDB buffer pool is 12 GiB. +- SSD-backed host: r7i.xlarge with a 1 TiB gp3 SSD EBS volume for MySQL data and binary logs. The InnoDB buffer pool is 12 GiB. + +### MySQL Params Setup + +- `log_bin=ON` +- `binlog_format=ROW` +- `sync_binlog=1` +- `binlog_transaction_compression=OFF` +- `binlog_large_transaction_optimization_enabled` is set to ON or OFF before each test. + +### Workload Setup + +- Large transaction: one transaction inserts 8 MiB `LONGBLOB` chunks into a dedicated table and commits once. +- Payload sizes are 1 GiB, 5 GiB, 10 GiB, and 50 GiB. +- Concurrent Sysbench workload: 1,500 threads against 500 tables with 100,000 rows per table (50 million rows total). + +### Results + +BOLT also affects overall system throughput, but here we share only the commit-latency results for the large transactions. The results are in seconds and are averaged over three runs. + +| Storage | BOLT | 1 GiB | 5 GiB | 10 GiB | 50 GiB | +| --- | --- | --- | --- | --- | --- | +| HDD | ON | 3.325s | 3.081s | 3.204s | 2.741s | +| HDD | OFF | 17.473s | 92.322s | 202.495s | 1091.222s | +| SSD | ON | 2.253s | 2.351s | 2.215s | 2.419s | +| SSD | OFF | 9.940s | 44.067s | 88.983s | 435.713s | + +The table shows that BOLT makes commit latency effectively independent of transaction size. With BOLT ON, latency stays in the 2s to 3.5s range across all payload sizes. With BOLT OFF, latency grows roughly linearly with the transaction. The improvement widens as transactions grow. On SSD it goes from about 4x at 1 GiB to about 180x at 50 GiB. On HDD it goes from about 5x at 1 GiB to about 400x at 50 GiB. + +## Experiment 2: Binlog Recovery + +This experiment validates BOLT's impact on binlog crash recovery time for large transactions. The setup is the same as Experiment 1, but here we force recovery of a large transaction on restart. + +BOLT writes a padding event (`Large_transaction_header_event`) that records the offset of the end of a large transaction. On restart, the fast recovery path reads this offset and jumps directly to the end of the transaction instead of scanning the entire transaction body. Recovery time therefore becomes nearly independent of transaction size. With BOLT OFF, recovery falls back to the old path and scans the full transaction body. + +Note that we measure how long the binlog recovery stage takes. We do not measure how long it takes to roll back or roll forward prepared transactions in InnoDB. + +| Storage | BOLT | 1 GiB | 5 GiB | 10 GiB | 50 GiB | +| --- | --- | --- | --- | --- | --- | +| HDD | ON | 0.22s | 0.21s | 0.19s | 0.23s | +| HDD | OFF | 14.6s | 76.9s | 168.7s | 909.4s | +| SSD | ON | 0.09s | 0.11s | 0.12s | 0.11s | +| SSD | OFF | 5.68s | 25.2s | 50.8s | 249s | + +BOLT recovery is consistent across all transaction sizes. It does not read the transaction body and only reads the first and last event, which makes recovery constant time. This yields large recovery improvements and saves tens of minutes in availability time. + +## Experiment 3: Write Throughput + +This experiment validates BOLT's impact on write throughput when large transactions are used. + +### EC2 Setup + +Single host: r7i.16xlarge with a 1 TiB gp3 EBS volume (3,000 IOPS baseline, 125 MiB/s baseline throughput). `innodb_dedicated_server=ON`. + +### MySQL Params Setup + +- `log_bin=ON` +- `binlog_format=ROW` +- `sync_binlog=1` +- `binlog_transaction_compression=OFF` +- `binlog_large_transaction_optimization_enabled` is set to ON or OFF before each test. + +### Workload Setup + +A custom Sysbench workload inserts `LONGBLOB` payloads in 8 MiB chunks (matching Experiment 1) into one of 500 shard tables per transaction, then commits. Spreading writes across 500 tables avoids the lock contention that a single table would create among writer threads. We modified the scripts to control both the transaction size and the mix of sizes. For example, a run can be 100% 128 MB transactions, or a mix such as 20% 5 GB, 30% 128 MB, and 50% 20 KB. + +Across all experiments, we use 50 threads and run for 30 minutes. + +### Results + +#### Sub-Experiment 1: Large-Sized Transaction Workload + +We measure the performance improvement for a workload made up only of large transactions. + +| BOLT | Elapsed (s) | 5 GB Trxs Count | Throughput (MB/s) | +| --- | --- | --- | --- | +| ON | 1810 | 22 | 60.8 | +| OFF | 1810 | 7 | 19.3 | + +When BOLT is off, the system is limited by latency. As described in Experiment 1, during the transaction commit the transaction blocks all other transactions from committing. BOLT does not have this bottleneck, so it is instead limited by IO throughput. This yields roughly a 3x throughput improvement. + +#### Sub-Experiment 2: Medium-Sized Transactions Workload + +We measure the performance penalty that BOLT may add for transactions that spill over but are not promoted through BOLT. This is an edge case for BOLT because it computes the checksum for the same transaction twice: once when spilling over (introduced by BOLT) and once when copying back to the binlog file. CRC computation is highly optimized, but we still measure BOLT's impact in this scenario. Here the BOLT optimization is not enabled, since it only works when the threshold is above 128 MB. + +| BOLT | Elapsed (s) | 64 MB Trxs Count | Throughput (MB/s) | +| --- | --- | --- | --- | +| ON | 1810 | 482 | 17.0 | +| OFF | 1810 | 468 | 16.5 | + +As expected, there is no meaningful difference between ON and OFF. The CRC computation adds negligible overhead. + +#### Sub-Experiment 3: Mixed-Sized Transactions Workload + +We measure the performance improvement for a workload made up of different transaction sizes. This workload is 10% huge (5 GB), 30% medium (256 MB), 40% small (12 MB), and 20% tiny (5 KB). + +| BOLT | Elapsed (s) | 5 GB Trxs Count | 256 MB Trxs Count | 12 MB Trxs Count | 5 KB Trxs Count | Throughput (MB/s) | +| --- | --- | --- | --- | --- | --- | --- | +| ON | 1810 | 18 | 42 | 68 | 42 | 56.1 | +| OFF | 1810 | 6 | 22 | 38 | 16 | 19.9 | + +BOLT commits the 5 GB and 256 MB transactions faster and with very low commit latency, which allows more concurrency and higher parallelization. As a result, system throughput improves by about 2.8x. + +#### Sub-Experiment 4: Small-Sized Transactions Workload + +We measure the performance penalty that BOLT may add for computing the CRC checksum for events written into the binlog `IO_cache`. With BOLT enabled, events are written into memory with checksum ON, while if BOLT is disabled, events are written into cache without computing a checksum; the checksum is then computed again when copied into the binlog file. We don't anticipate this to introduce any performance penalty, as CRC computation is fast and the bottleneck on the binlog lies in the commit codepath, not the write. This experiment runs Sysbench without any modifications, using normal-sized binlog transactions. + +| BOLT | Elapsed (s) | Throughput (MB/s) | +| --- | --- | --- | +| ON | 1810 | 23.6 | +| OFF | 1810 | 23.7 | + +As expected, there is no meaningful difference between ON and OFF. The CRC computation adds no overhead. + +#### Sub-Experiment 5: Medium-Sized Transactions Workload + +BOLT also writes the CRC when the file is spilled to disk. This experiment is designed to validate the overhead for transactions that spill over to disk but are not committed through the BOLT codepath. This experiment runs a modified Sysbench workload with 12 MB transaction sizes. + +| BOLT | Elapsed (s) | 12 MB Trxs Count | Throughput (MB/s) | +| --- | --- | --- | --- | +| ON | 1810 | 472 | 16.6 | +| OFF | 1810 | 468 | 16.5 | + +Similar to Sub-Experiment 4, there is no overhead. + +--- \ No newline at end of file diff --git a/design/binary-log/683-binlog-large-transaction-optimization/requirements.md b/design/binary-log/683-binlog-large-transaction-optimization/requirements.md new file mode 100644 index 000000000000..9c135da114d5 --- /dev/null +++ b/design/binary-log/683-binlog-large-transaction-optimization/requirements.md @@ -0,0 +1,147 @@ +## Requirements + +### Functional requirements + +**FR1**. The optimization must create a dedicated directory named `#binlog_temp_files` for storing session binlog cache temp files, located in the same directory as the binlog files so that promoting a temp file into the binlog sequence is a metadata-only operation on the same filesystem. + +**FR2.** The optimization must create the `#binlog_temp_files` directory during binlog initialization at startup, after the binlog directory is known and before the server accepts connections. + +**FR2.1.** If `#binlog_temp_files` already exists at startup, the server must clean up its managed temp files according to FR28.5 through FR28.8. + +**FR3**. The optimization must exclude the `#binlog_temp_files` directory from schema-visible listings, such as `SHOW DATABASES` and `information_schema`, so that it does not appear as a schema. + +**FR4.** The optimization must assign temp files unique filenames to avoid conflicts. + +**FR5.** The optimization must provide a global, dynamically settable system variable of boolean type, `binlog_large_transaction_optimization_enabled`, that turns the optimization ON or OFF, with a default of ON. Changes must take effect for subsequent transactions across all existing and new sessions without a server restart. + +**FR5.1.** When `binlog_large_transaction_optimization_enabled` is set to OFF, every transaction must commit through the standard code path. + +**FR6**. The optimization must provide a global, dynamically settable threshold, `binlog_large_transaction_optimization_threshold`, that controls the spilled size above which a transaction is committed through the optimization's code path. Changes must take effect for subsequent transactions across all existing and new sessions without a server restart. + +**FR6.1.** The threshold is only evaluated when `binlog_large_transaction_optimization_enabled` is set to ON. When the optimization is disabled, no transaction qualifies for the optimized code path regardless of the threshold value. + +**FR7**. The optimization must use a default `binlog_large_transaction_optimization_threshold` of 128 MB, which must be appropriate for most workloads without further configuration. + +**FR8**. The optimization must enforce a minimum `binlog_large_transaction_optimization_threshold` of 10 MB. Values below 10 MB must be rejected. + +**FR9.** The optimization must commit a transaction through the optimization's code path when the transaction has spilled its binlog cache to a temp file and the spilled size exceeds `binlog_large_transaction_optimization_threshold`. + +**FR9.1.** When a transaction qualifies for the optimization, the commit must be finalized by promoting the temp file into the binlog sequence. The promoted file is renamed to the next sequential binlog file name (e.g., if the active binlog is `mysql_bin.000005`, the promoted file becomes `mysql_bin.000006`). The promoted file must then become the active binlog file and must use the existing `max_binlog_size` rotation behavior. + +**FR9.2.** Promoted files must be treated identically to standard binlog files with respect to `FLUSH BINARY LOGS`, `PURGE BINARY LOGS`, and `RESET BINARY LOGS AND GTIDS` . They appear in `SHOW BINARY LOGS`, are eligible for purge based on retention policy, and are removed on reset. + +**FR9.3.** When the optimization is enabled, the effective value of `binlog_large_transaction_optimization_threshold` is set to the maximum of `binlog_large_transaction_optimization_threshold` and `binlog_cache_size`. This constraint is enforced upon engine startup and on every SET statement that alters either variable. + +**FR9.4.** When the optimization is enabled, if `binlog_large_transaction_optimization_threshold` is set to a value less than `binlog_cache_size`, the threshold is adjusted to match `binlog_cache_size` and the server must emit a warning indicating the adjustment. + +**FR9.5**. When the optimization is enabled, if `binlog_cache_size` is set to a value greater than `binlog_large_transaction_optimization_threshold`, the threshold is adjusted to match `binlog_cache_size` and the server must emit a warning indicating the adjustment. + +**FR10.** When a transaction has not spilled to a temp file, or its spilled size does not exceed `binlog_large_transaction_optimization_threshold`, the optimization must commit it through the standard code path and produce a binary log identical to what the standard path produces today. + +**FR11.** The optimization must not affect GTID generation or sequencing. The promoted file's transaction retains the GTID sequencing, which is assigned at commit time. + +**FR12**. The optimization must work correctly with both XA transactions (external XID) and non-XA transactions (internal XID). + +**FR13.** The optimization must introduce a `Large_transaction_header_event` binlog event type whose purpose is to speed up recovery for large transactions by allowing the server to locate the terminating event directly, skipping reading the payload of the large transaction. + +**FR13.1.** The optimization must write a `Large_transaction_header_event` into the promoted binlog file, recording the offset, type, and identity of the large transaction's terminating event. The terminating event contains either an internal XID (for DML and DDL transactions) or an external XID (for XA PREPARE and XA COMMIT ONE PHASE transactions). + +**FR14**. The optimization must compute the transaction's GTID at commit time and write the `Gtid_log_event`, write the `Previous_gtids_log_event` at the start of the file, and set the commit dependency tracking (`last_committed` / `sequence_number`) as when rotating to a new binlog file. + +**FR15**. The optimization must delete the temp file with no trace left in the binlog or the binlog index when a transaction is rolled back before promotion, since a rolled-back transaction is never written to the binlog file. + +**FR16.** The optimization must handle `ROLLBACK TO SAVEPOINT` which truncates the temp file to the byte offset recorded when the savepoint was established. + +**FR16.1.** When the optimization spills a large transaction to a temp file, all existing savepoint offsets must be adjusted to account for the reserved header space. + +**FR16.2.** When a savepoint rollback is activated and the remaining transaction size is larger than `binlog_large_transaction_optimization_threshold` then the spilled file is maintained and the optimization continues. Otherwise, it falls back to the standard code path of copying the transaction into the existing binlog file. + +**FR17.** The optimization must evaluate fallback conditions and commit the transaction through the standard code path if any of the conditions are met, emit a diagnostic message, and increment `binlog_large_transaction_optimization_missed_count`. + +**FR17.1.** The fallback conditions are: (1)`binlog_format` is not ROW; (2) the reserved header space is insufficient for the required header events; (3) binlog encryption is enabled;(4) binlog transaction compression is enabled; (5) `binlog_checksum` changed while the transaction was in progress, so the transaction's cached events do not match the current checksum configuration; (6) a single statement updated both transactional and non-transactional tables. + +**FR17.2**. A fallback transaction must produce the same event sequence in the active binlog as a transaction written through the standard code path. It achieves this by overriding the spilled `log_pos` and recomputing the checksum. + +**FR18.** The optimization must be safe against concurrent binlog rotation. The promotion workflow computes the binlog sequence number and writes header events while holding `LOCK_log`, ensuring no concurrent rotation can invalidate the promoted file's position in the sequence. + +**FR19.** The optimization must not change the existing binlog crash recovery flow. + +**FR19.1.** As an exception to FR19 during recovery the optimization may skip reading a large transaction's payload by using the terminating-event offset recorded in `Large_transaction_header_event` to locate the terminating event directly. This reduces recovery time to constant time regardless of transaction size. + +**FR19.2.** If the server crashes before promotion completes, recovery must discard the temp file and roll back the transaction. + +**FR19.3.** If the server crashes after promotion completes but before InnoDB commit completes, recovery must roll forward the transaction. + +**FR19.4.** If the server crashes after both promotion and InnoDB commit complete, the transaction must be durable with no data loss. + +**FR19.5.** Before adding a GTID to the executed set during optimized recovery, recovery must verify that the recorded offset identifies a complete binlog event whose event type and XID match the values recorded in `Large_transaction_header_event`. + +**FR19.6.** If the verification required by FR19.5 fails, recovery must use the standard sequential scan for a large transaction. + +**FR19.7.** When `source_verify_checksum` is ON, recovery must use the standard sequential scan for a large transaction. + +**FR20.** The optimization must ensure that a replica ignores the `Large_transaction_header_event` when it receives it, since the event is not relevant to the replica. This is achieved by marking `Large_transaction_header_event` as an ignorable event per the MySQL replication protocol. + +**FR21.** `START REPLICA UNTIL` must function identically regardless of whether the transaction was committed through the optimization or the standard code path. + +**FR22.** When `binlog_format` is set to STATEMENT or MIXED, binlog encryption is enabled, or binlog transaction compression is enabled, the optimization must emit a warning to the error log indicating that the optimization is disabled and transactions will fall back to the standard code path. + +**FR23**. The optimization must expose a status variable, `binlog_large_transaction_optimization_count`, reporting the number of transactions that successfully executed the optimization's code path since server startup. + +**FR24**. The optimization must expose a status variable, `binlog_large_transaction_optimization_missed_count`, reporting the number of transactions that exceeded `binlog_large_transaction_optimization_threshold` but could not execute the optimization's code path since server startup. + +**FR25**. The optimization must expose `binlog_large_transaction_optimization_count` and `binlog_large_transaction_optimization_missed_count` through `SHOW GLOBAL STATUS` and `performance_schema.global_status`. + +**FR26.** The optimization must not change the output of `SHOW BINARY LOGS`, `SHOW BINLOG EVENTS`, or `SHOW BINARY LOG STATUS`. Promoted files appear in these statements identically to standard binlog files. + +**FR27.** The optimization must not change the output of the `performance_schema.log_status` table. + +**FR28.** The optimization must store its temp files on the same filesystem as the binlog files. [LTO-V3-R1] + +**FR28.1.** Operators must account for the additional binlog-volume space consumed by concurrent large transactions, which previously resided in the system `tmpdir`. + +**FR28.2.** At startup, if the `#binlog_temp_files` path exists but is not a directory (e.g., a symlink or regular file), the server must log an error and disable the optimization. + +**FR28.3.** If directory creation or permission acquisition fails at startup, the server must log an error and disable the optimization. + +**FR28.4.** If disk space is exhausted while writing to a temp file during transaction execution, the write error is handled identically to a disk-full error on the standard binlog cache temp file. + +**FR28.5.** A temp file created by the optimization must have a name matching the pattern `bolt_`, where `` is a lowercase identifier unique within `#binlog_temp_files` (BOLT stands for "Binlog Optimization for Large Transaction"). + +**FR28.6.** Startup cleanup must accept for deletion only regular files whose basename matches the optimization's temp file naming pattern and must reject all other directory entries. + +**FR28.7.** Startup cleanup must delete each accepted file. + +**FR28.8.** If startup cleanup cannot delete an accepted file, the server must log `ER_BINLOG_CANT_DELETE_FILE` from MYSQL_BIN_LOG and disable the optimization. + +**FR29**. The optimization may cause binlog files to be rotated before they reach `max_binlog_size`. Because promoting a temp file inserts it into the binlog sequence as a new binlog file and forces a rotation, the previously active binlog file is closed early, potentially well below `max_binlog_size`, if a large transaction is issued while little or no concurrent workload is writing to the active file. This results in smaller than configured binlog files around large transactions. + +**FR29.1.** When multiple concurrent transactions each qualify for the optimization, each transaction is committed to its own promoted binlog file. The promoted files may be smaller than `max_binlog_size`. Each promoted file becomes part of the binlog sequence independently. + +**FR30**. The optimization may produce a binlog file that contains transactions other than the large transaction. Although the large transaction is generally the only transaction in the promoted binlog file, subsequent small transactions can also be written into it if the file's size is still below `max_binlog_size` before the next rotation occurs. + +### Non-functional requirements + +**NFR1**. The optimization must not impact the durability invariant: InnoDB must never mark a large transaction executing on the optimization's code path as committed unless the corresponding promoted binlog file is durable on disk. + +**NFR2**. The optimization must preserve `sync_binlog` durability semantics. Before a temp file is promoted into the binlog sequence, it must be synced to durable storage, so that promotion does not weaken the durability guarantees provided by `sync_binlog`. + +**NFR3**. The optimization must still recover consistently if MySQL users manually delete files from `#binlog_temp_files` at recovery time, since any file remaining in that directory is a temp file whose promotion into the binlog sequence did not complete, so its transaction was never committed. + +**NFR4**. The optimization must complete commit in constant time under `LOCK_log` regardless of transaction size. + +**NFR5**. The optimization must complete binlog crash recovery in constant time regardless of transaction size. + +**NFR6**. The optimization must not degrade performance for transactions that do not exceed `binlog_large_transaction_optimization_threshold`. + +**NFR7**. The optimization must be safe under concurrent access, including multiple sessions spilling to their own temp files, concurrent commits from both the optimized and standard paths, and concurrent rotations. + +**NFR8**. The optimization must not require changes to the MySQL client-server protocol or the replication protocol. The `Large_transaction_header_event` must be of ignorable event type, so it does not impact compatibility with existing replicas or binlog tooling. + +**NFR9.** The optimization must be transparent to backup tools. Promoted binlog files must be readable and processable by these tools identically to standard binlog files, with no special handling required. + +**NFR10.** The optimization must ensure that on downgrade to a version that does not recognize `Large_transaction_header_event`, the server safely skips the event because it is marked as ignorable. + +**NFR11.** The optimization must not require special migration steps on upgrade, since it only produces `Large_transaction_header_event` for new transactions committed after the upgrade. Binlog files produced before the upgrade remain unchanged and fully compatible. + +**NFR12**. The optimization must create temp files with the same file permissions and ownership as binlog files, since they contain equally sensitive data and reside alongside the binlog files. diff --git a/include/my_sys.h b/include/my_sys.h index ee6920fb08bc..b4cb3befb6ab 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -431,6 +431,13 @@ struct IO_CACHE /* Used when caching files */ void *arg{nullptr}; /* for use by pre/post_read */ char *file_name{nullptr}; /* if used with 'open_cached_file' */ char *dir{nullptr}, *prefix{nullptr}; + /* + With 'open_cached_file': create the lazily created temporary file as a + named file in the filesystem namespace instead of an anonymous file + unlinked at creation. Its name is then recorded in file_name, and closing + the cache deletes the file. + */ + bool named_file{false}; File file{-1}; /* file descriptor */ PSI_file_key file_key{PSI_NOT_INSTRUMENTED}; /* instrumented file key */ diff --git a/libs/mysql/binlog/event/CMakeLists.txt b/libs/mysql/binlog/event/CMakeLists.txt index 7864a3d15251..eb2daba69fa8 100644 --- a/libs/mysql/binlog/event/CMakeLists.txt +++ b/libs/mysql/binlog/event/CMakeLists.txt @@ -42,6 +42,7 @@ SET(TARGET_HEADERS debug_vars.h event_reader_macros.h event_reader.h + large_transaction_header_event.h load_data_events.h rows_event.h statement_events.h @@ -65,6 +66,7 @@ SET(TARGET_SRCS binlog_event.cpp control_events.cpp event_reader.cpp + large_transaction_header_event.cpp load_data_events.cpp rows_event.cpp statement_events.cpp diff --git a/libs/mysql/binlog/event/binlog_event.cpp b/libs/mysql/binlog/event/binlog_event.cpp index 1f9edc9012a0..f9a39e58f4f5 100644 --- a/libs/mysql/binlog/event/binlog_event.cpp +++ b/libs/mysql/binlog/event/binlog_event.cpp @@ -45,39 +45,41 @@ bool debug_simulate_invalid_address = false; namespace mysql::binlog::event { static const std::unordered_map - event_type_to_string = {{STOP_EVENT, "Stop"}, - {QUERY_EVENT, "Query"}, - {ROTATE_EVENT, "Rotate"}, - {INTVAR_EVENT, "Intvar"}, - {APPEND_BLOCK_EVENT, "Append_block"}, - {DELETE_FILE_EVENT, "Delete_file"}, - {RAND_EVENT, "RAND"}, - {USER_VAR_EVENT, "User var"}, - {XID_EVENT, "Xid"}, - {FORMAT_DESCRIPTION_EVENT, "Format_desc"}, - {TABLE_MAP_EVENT, "Table_map"}, - {OBSOLETE_WRITE_ROWS_EVENT_V1, "Write_rows_v1"}, - {OBSOLETE_UPDATE_ROWS_EVENT_V1, "Update_rows_v1"}, - {OBSOLETE_DELETE_ROWS_EVENT_V1, "Delete_rows_v1"}, - {BEGIN_LOAD_QUERY_EVENT, "Begin_load_query"}, - {EXECUTE_LOAD_QUERY_EVENT, "Execute_load_query"}, - {INCIDENT_EVENT, "Incident"}, - {IGNORABLE_LOG_EVENT, "Ignorable"}, - {ROWS_QUERY_LOG_EVENT, "Rows_query"}, - {WRITE_ROWS_EVENT, "Write_rows"}, - {UPDATE_ROWS_EVENT, "Update_rows"}, - {DELETE_ROWS_EVENT, "Delete_rows"}, - {GTID_LOG_EVENT, "Gtid"}, - {ANONYMOUS_GTID_LOG_EVENT, "Anonymous_Gtid"}, - {PREVIOUS_GTIDS_LOG_EVENT, "Previous_gtids"}, - {HEARTBEAT_LOG_EVENT, "Heartbeat"}, - {TRANSACTION_CONTEXT_EVENT, "Transaction_context"}, - {VIEW_CHANGE_EVENT, "View_change"}, - {XA_PREPARE_LOG_EVENT, "XA_prepare"}, - {PARTIAL_UPDATE_ROWS_EVENT, "Update_rows_partial"}, - {TRANSACTION_PAYLOAD_EVENT, "Transaction_payload"}, - {GTID_TAGGED_LOG_EVENT, "Gtid_tagged_log_event"}, - {UNKNOWN_EVENT, "Unknown"}}; + event_type_to_string = { + {STOP_EVENT, "Stop"}, + {QUERY_EVENT, "Query"}, + {ROTATE_EVENT, "Rotate"}, + {INTVAR_EVENT, "Intvar"}, + {APPEND_BLOCK_EVENT, "Append_block"}, + {DELETE_FILE_EVENT, "Delete_file"}, + {RAND_EVENT, "RAND"}, + {USER_VAR_EVENT, "User var"}, + {XID_EVENT, "Xid"}, + {FORMAT_DESCRIPTION_EVENT, "Format_desc"}, + {TABLE_MAP_EVENT, "Table_map"}, + {OBSOLETE_WRITE_ROWS_EVENT_V1, "Write_rows_v1"}, + {OBSOLETE_UPDATE_ROWS_EVENT_V1, "Update_rows_v1"}, + {OBSOLETE_DELETE_ROWS_EVENT_V1, "Delete_rows_v1"}, + {BEGIN_LOAD_QUERY_EVENT, "Begin_load_query"}, + {EXECUTE_LOAD_QUERY_EVENT, "Execute_load_query"}, + {INCIDENT_EVENT, "Incident"}, + {IGNORABLE_LOG_EVENT, "Ignorable"}, + {ROWS_QUERY_LOG_EVENT, "Rows_query"}, + {WRITE_ROWS_EVENT, "Write_rows"}, + {UPDATE_ROWS_EVENT, "Update_rows"}, + {DELETE_ROWS_EVENT, "Delete_rows"}, + {GTID_LOG_EVENT, "Gtid"}, + {ANONYMOUS_GTID_LOG_EVENT, "Anonymous_Gtid"}, + {PREVIOUS_GTIDS_LOG_EVENT, "Previous_gtids"}, + {HEARTBEAT_LOG_EVENT, "Heartbeat"}, + {TRANSACTION_CONTEXT_EVENT, "Transaction_context"}, + {VIEW_CHANGE_EVENT, "View_change"}, + {XA_PREPARE_LOG_EVENT, "XA_prepare"}, + {PARTIAL_UPDATE_ROWS_EVENT, "Update_rows_partial"}, + {TRANSACTION_PAYLOAD_EVENT, "Transaction_payload"}, + {GTID_TAGGED_LOG_EVENT, "Gtid_tagged_log_event"}, + {LARGE_TRANSACTION_HEADER_EVENT, "Large_transaction_header"}, + {UNKNOWN_EVENT, "Unknown"}}; const std::string &get_event_type_as_string(Log_event_type type) { try { diff --git a/libs/mysql/binlog/event/binlog_event.h b/libs/mysql/binlog/event/binlog_event.h index 1036d5e781b6..9ba9616ea2ca 100644 --- a/libs/mysql/binlog/event/binlog_event.h +++ b/libs/mysql/binlog/event/binlog_event.h @@ -365,6 +365,11 @@ enum Log_event_type { HEARTBEAT_LOG_EVENT_V2 = 41, GTID_TAGGED_LOG_EVENT = 42, + + /** + A binlog event used by the large transaction optimization. + */ + LARGE_TRANSACTION_HEADER_EVENT = 43, /** Add new events here - right above this comment! Existing events (except ENUM_END_EVENT) should never change their numbers diff --git a/libs/mysql/binlog/event/control_events.cpp b/libs/mysql/binlog/event/control_events.cpp index 8d43b28c774b..19258a2a669c 100644 --- a/libs/mysql/binlog/event/control_events.cpp +++ b/libs/mysql/binlog/event/control_events.cpp @@ -124,7 +124,7 @@ Format_description_event::Format_description_event(uint8_t binlog_ver, IGNORABLE_HEADER_LEN, TRANSACTION_CONTEXT_HEADER_LEN, VIEW_CHANGE_HEADER_LEN, XA_PREPARE_HEADER_LEN, ROWS_HEADER_LEN_V2, TRANSACTION_PAYLOAD_EVENT, 0 /* HEARTBEAT_LOG_EVENT_V2*/, - 0 /* GTID_TAGGED_LOG_EVENT */ + 0 /* GTID_TAGGED_LOG_EVENT */, 0 /* LARGE_TRANSACTION_HEADER_EVENT */ }; /* Allows us to sanity-check that all events initialized their diff --git a/libs/mysql/binlog/event/large_transaction_header_event.cpp b/libs/mysql/binlog/event/large_transaction_header_event.cpp new file mode 100644 index 000000000000..42bb5b7df170 --- /dev/null +++ b/libs/mysql/binlog/event/large_transaction_header_event.cpp @@ -0,0 +1,63 @@ +#include "mysql/binlog/event/large_transaction_header_event.h" + +#include "mysql/binlog/event/control_events.h" // Format_description_event +#include "mysql/binlog/event/event_reader_macros.h" + +/** + @file + @brief Deserialization of Large_transaction_header_event. The matching + serialization lives in the server class under sql/log_event.* +*/ + +namespace mysql::binlog::event { + +Large_transaction_header_event::Large_transaction_header_event( + const char *buf, const Format_description_event *fde) + : Binary_log_event(&buf, fde) { + BAPI_ENTER( + "Large_transaction_header_event::" + "Large_transaction_header_event(const char*, ...)"); + READER_TRY_INITIALIZATION; + READER_ASSERT_POSITION(fde->common_header_len); + + READER_TRY_SET(m_version, read); + if (m_version == 0 || m_version > kVersion) { + READER_THROW("Invalid Large_transaction_header version"); + } + READER_TRY_SET(m_terminating_event_offset, read); + /* Read unconditionally: a truncated event missing the type byte is + reported as a read error rather than silently defaulting the type. */ + READER_TRY_SET(m_terminating_event_type, read); + + /* The remainder of the body is padding; its contents are ignored. */ + m_padding_size = READER_CALL(available_to_read); + + READER_CATCH_ERROR; + BAPI_VOID_RETURN; +} + +Large_transaction_header_event::Large_transaction_header_event( + uint64_t terminating_event_offset, uint8_t terminating_event_type, + uint64_t padding_size) + : Binary_log_event(LARGE_TRANSACTION_HEADER_EVENT), + m_terminating_event_offset(terminating_event_offset), + m_terminating_event_type(terminating_event_type), + m_padding_size(padding_size) {} + +#ifndef HAVE_MYSYS +void Large_transaction_header_event::print_event_info(std::ostream &info) { + info << "terminating event offset " << m_terminating_event_offset; +} + +void Large_transaction_header_event::print_long_info(std::ostream &info) { + info << "Timestamp: " << header()->when.tv_sec; + info << "\tVersion: " << static_cast(m_version); + info << "\tTerminating event offset: " << m_terminating_event_offset; + info << "\tTerminating event type: " + << static_cast(m_terminating_event_type); + info << "\tPadding: " << m_padding_size << " bytes"; + info << "\n"; +} +#endif + +} // namespace mysql::binlog::event diff --git a/libs/mysql/binlog/event/large_transaction_header_event.h b/libs/mysql/binlog/event/large_transaction_header_event.h new file mode 100644 index 000000000000..8152fef1dcfd --- /dev/null +++ b/libs/mysql/binlog/event/large_transaction_header_event.h @@ -0,0 +1,148 @@ +/** + @file large_transaction_header_event.h + + @brief Deserialization of the Large_transaction_header + event. All serialization logic lives in the server class under + sql/log_event.* +*/ + +#ifndef MYSQL_BINLOG_EVENT_LARGE_TRANSACTION_HEADER_EVENT_H +#define MYSQL_BINLOG_EVENT_LARGE_TRANSACTION_HEADER_EVENT_H + +#include + +#include "mysql/binlog/event/binlog_event.h" + +/// @addtogroup GroupLibsMysqlBinlogEvent +/// @{ + +namespace mysql::binlog::event { + +/** + @class Large_transaction_header_event + + Event needed for the large transaction optimization. It serves two + purposes: + + 1. It records the offset of the transaction's terminating event, so + that binary log recovery can seek directly past the transaction + body instead of scanning it. + 2. Its variable-length padding fills the file's reserved header region + exactly, so the transaction body starts at the offset that was + assumed while events were being spilled. + + Always written with the LOG_EVENT_IGNORABLE_F flag set: replicas and + binlog tools that do not recognize the type skip it. + + @section Large_transaction_header_event_binary_format + Binary Format + + The post-header is empty. The Body has the following components: + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Body for Large_transaction_header_event
NameFormatDescription
version1 byte unsigned integerEvent format version; currently 1. Retained for future + extensibility.
terminating_event_offset8 byte unsigned little-endian integerOffset, within this binary log file, of the transaction's + terminating event.
terminating_event_type1 byte unsigned integerBinary log event type of the terminating event.
paddingvariable-length byte sequenceFiller occupying the remainder of the reserved header region; + contents are undefined and ignored on read.
+*/ +class Large_transaction_header_event : public Binary_log_event { + public: + /** Current version of the event's body format. The version field is + retained for future extensibility; only this version exists today. */ + static constexpr uint8_t kVersion = 1; + /** Body bytes before padding: version (1) + offset (8) + event type (1). */ + static constexpr size_t kFixedBodyLength = 1 + 8 + 1; + + /** + Deserializing constructor. + + @param buf Contains the serialized event. + @param fde An FDE event (see Rotate_event constructor for more info). + */ + Large_transaction_header_event(const char *buf, + const Format_description_event *fde); + + /** + Creates an event with the given terminating-event metadata and + padding size (used by the server when writing a promoted binary log + file's header). + + @param terminating_event_offset Offset of the transaction's + terminating event in the file. + @param terminating_event_type Type code of the transaction's + terminating event in the file. + @param padding_size Number of filler bytes to occupy the + remainder of the reserved region. + */ + Large_transaction_header_event(uint64_t terminating_event_offset, + uint8_t terminating_event_type, + uint64_t padding_size); + + ~Large_transaction_header_event() override = default; + + /// @return Format version of this event's body, so a later server can tell + /// which fields are present. + uint8_t get_version() const { return m_version; } + + /// @return Byte offset, from the start of the file, at which the + /// transaction's terminating event begins. + uint64_t get_terminating_event_offset() const { + return m_terminating_event_offset; + } + + /// @return Type code of the transaction's terminating event. + uint8_t get_terminating_event_type() const { + return m_terminating_event_type; + } + + /// @return Number of filler bytes this event carries in order to occupy the + /// remainder of the file's reserved header region exactly. + uint64_t get_padding_size() const { return m_padding_size; } + +#ifndef HAVE_MYSYS + void print_event_info(std::ostream &info) override; + void print_long_info(std::ostream &info) override; +#endif + + protected: + /** Body format version read from (or to be written to) the wire. */ + uint8_t m_version{kVersion}; + /** Offset of the transaction's terminating event in this file. */ + uint64_t m_terminating_event_offset{0}; + /** Type code of the terminating event; absent in version-1 headers. */ + uint8_t m_terminating_event_type{0}; + /** Number of filler bytes following the fixed body fields. */ + uint64_t m_padding_size{0}; +}; + +} // namespace mysql::binlog::event + +/// @} + +#endif // MYSQL_BINLOG_EVENT_LARGE_TRANSACTION_HEADER_EVENT_H diff --git a/libs/mysql/binlog/event/trx_boundary_parser.cpp b/libs/mysql/binlog/event/trx_boundary_parser.cpp index 4d30b66eef4a..63d5ae476e8f 100644 --- a/libs/mysql/binlog/event/trx_boundary_parser.cpp +++ b/libs/mysql/binlog/event/trx_boundary_parser.cpp @@ -252,6 +252,7 @@ Transaction_boundary_parser::get_event_boundary_type( case mysql::binlog::event::SLAVE_EVENT: case mysql::binlog::event::DELETE_FILE_EVENT: case mysql::binlog::event::TRANSACTION_CONTEXT_EVENT: + case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT: boundary_type = EVENT_BOUNDARY_TYPE_IGNORE; break; diff --git a/mysql-test/common/binlog/validate_bolt_file.inc b/mysql-test/common/binlog/validate_bolt_file.inc new file mode 100644 index 000000000000..3764919017b6 --- /dev/null +++ b/mysql-test/common/binlog/validate_bolt_file.inc @@ -0,0 +1,96 @@ +# Requires $bolt_header_file to name a standalone BOLT-promoted binlog. +# Decode the promoted file and validate its LTH metadata and layout. +# These checks keep a plausible but corrupt LTH from silently directing +# optimized recovery to an incorrect terminal event. +# The awk program avoids `$` field syntax because mysqltest expands it. + +--let $bolt_promoted_dump = $MYSQLTEST_VARDIR/tmp/bolt_promoted_dump.txt +--let $bolt_validator_script = $MYSQLTEST_VARDIR/tmp/validate_bolt_header.awk +--exec $MYSQL_BINLOG --force-if-open --verify-binlog-checksum --verbose $MYSQLD_DATADIR/$bolt_header_file > $bolt_promoted_dump +--exec grep -m 1 -E 'last_committed=.*sequence_number=' $bolt_promoted_dump | grep -Eq 'last_committed=0[[:space:]]+sequence_number=1' +--write_file $bolt_validator_script EOF +# Extract the end position printed by mysqlbinlog for the current event. +# Keep the full line in a variable so this does not rely on awk `$` fields. +function end_pos(line, value) { + value = line + sub(/^.*end_log_pos /, "", value) + sub(/ .*/, "", value) + return value + 0 +} + +BEGIN { + # Read mysqlbinlog's verbose output and associate each event description + # with the preceding '# at ' line. + while ((getline line) > 0) { + if (line ~ /^# at [0-9][0-9]*/) { + split(line, words, " ") + event_start = words[3] + 0 + continue + } + + # The preceding Previous-GTIDs event must end where the LTH starts. + if (line ~ /Previous-GTIDs/ && event_start) { + previous_start = event_start + previous_end = end_pos(line) + continue + } + + # A promoted file contains exactly one ignorable LTH in its prefix. + if (line ~ /Large transaction header[[:space:]]+Ignorable/ && event_start) { + if (lth_start) exit 1 + lth_start = event_start + lth_end = end_pos(line) + continue + } + + # Parse the LTH's recovery hint: the terminal-event offset, its expected + # type code, and the remaining padding that fills the reserved region. + if (line ~ /Terminating event offset /) { + if (!lth_start || lth_offset) exit 1 + value = line + sub(/^.*Terminating event offset /, "", value) + split(value, fields, ", type ") + lth_offset = fields[1] + 0 + split(fields[2], fields, ", padding ") + lth_type = fields[1] + 0 + sub(/ bytes/, "", fields[2]) + lth_padding = fields[2] + 0 + continue + } + + # Select only the first GTID after the LTH. The promoted active file can + # later contain additional GTIDs, which are not part of its header. + if (line ~ /GTID.*last_committed=.*sequence_number=/ && event_start == lth_end) { + if (gtid_start) exit 1 + gtid_start = event_start + gtid_end = end_pos(line) + continue + } + + # The recorded terminal offset must identify a decoded event with an end + # position beyond its start. + if (line ~ /end_log_pos/ && event_start == lth_offset) + terminal_end = end_pos(line) + } + + # The event type is byte 4 of a binlog event header. Read the raw byte at + # the LTH offset so metadata cannot merely agree with a mis-decoded event. + command = "dd if=\"" binlog_file "\" bs=1 skip=" (lth_offset + 4) " count=1 2>/dev/null | od -An -tu1" + command | getline terminal_type + close(command) + gsub(/[[:space:]]/, "", terminal_type) + + # Check the promoted prefix is contiguous, the first GTID ends exactly at + # the 64 KiB reserved boundary, the LTH points beyond that boundary, and + # the stored type is a recovery-supported terminal event (Query, XID, or + # XA PREPARE) that matches the raw event-header byte. + if (!(previous_start >= 4 && previous_end == lth_start && + lth_end == gtid_start && gtid_end == 65536 && + lth_offset >= gtid_end && terminal_end > lth_offset && + lth_padding > 0 && (lth_type == 2 || lth_type == 16 || lth_type == 38) && + terminal_type == lth_type)) exit 1 +} +EOF +--exec awk -v binlog_file=$MYSQLD_DATADIR/$bolt_header_file -f $bolt_validator_script $bolt_promoted_dump +--remove_file $bolt_promoted_dump +--remove_file $bolt_validator_script diff --git a/mysql-test/r/all_persisted_variables.result b/mysql-test/r/all_persisted_variables.result index 367de83ec142..50f354178870 100644 --- a/mysql-test/r/all_persisted_variables.result +++ b/mysql-test/r/all_persisted_variables.result @@ -48,7 +48,7 @@ include/assert.inc [Expect 500+ variables in the table. Due to open Bugs, we are # Test SET PERSIST -include/assert.inc [Expect 452 persisted variables in the table.] +include/assert.inc [Expect 454 persisted variables in the table.] ************************************************************ * 3. Restart server, it must preserve the persisted variable @@ -56,9 +56,9 @@ include/assert.inc [Expect 452 persisted variables in the table.] ************************************************************ # restart -include/assert.inc [Expect 452 persisted variables in persisted_variables table.] -include/assert.inc [Expect 452 persisted variables shown as PERSISTED in variables_info table.] -include/assert.inc [Expect 452 persisted variables with matching peristed and global values.] +include/assert.inc [Expect 454 persisted variables in persisted_variables table.] +include/assert.inc [Expect 454 persisted variables shown as PERSISTED in variables_info table.] +include/assert.inc [Expect 454 persisted variables with matching peristed and global values.] ************************************************************ * 4. Test RESET PERSIST IF EXISTS. Verify persisted variable diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result index 4f102873a851..868b22ad5719 100644 --- a/mysql-test/r/mysqld--help-notwin.result +++ b/mysql-test/r/mysqld--help-notwin.result @@ -187,6 +187,22 @@ The following options may be given as the first argument: --binlog-ignore-db=name Exclude updates to the specified database when writing the binary log. + --binlog-large-transaction-optimization-enabled + Enables the large transaction optimization, which keeps + large-transaction commit latency low, avoids stalling + concurrent commits, and keeps binary log crash recovery + fast regardless of transaction size. When ON (the + default), a transaction whose spilled size exceeds + binlog_large_transaction_optimization_threshold is + committed by promoting its temporary file into the binary + log sequence. When OFF, all transactions commit through + the standard code path. + (Defaults to on; use --skip-binlog-large-transaction-optimization-enabled to disable.) + --binlog-large-transaction-optimization-threshold=# + The spilled size in bytes above which a transaction + qualifies for the large transaction optimization. Has no + effect while + binlog_large_transaction_optimization_enabled is OFF. --binlog-max-flush-queue-time=# The maximum time that the binary log group commit will keep reading transactions before it flush the @@ -1701,6 +1717,8 @@ binlog-format ROW binlog-group-commit-sync-delay 0 binlog-group-commit-sync-no-delay-count 0 binlog-gtid-simple-recovery TRUE +binlog-large-transaction-optimization-enabled TRUE +binlog-large-transaction-optimization-threshold 134217728 binlog-max-flush-queue-time 0 binlog-order-commits TRUE binlog-rotate-encryption-master-key-at-startup FALSE diff --git a/mysql-test/suite/binlog/inc/validate_bolt_header.inc b/mysql-test/suite/binlog/inc/validate_bolt_header.inc new file mode 100644 index 000000000000..79f2e2ad5970 --- /dev/null +++ b/mysql-test/suite/binlog/inc/validate_bolt_header.inc @@ -0,0 +1,3 @@ +# Compatibility wrapper for existing BOLT tests. +# The merged promoted-file validator is shared from mysql-test/common. +--source common/binlog/validate_bolt_file.inc diff --git a/mysql-test/suite/binlog/r/binlog_bolt_2pc_recovery.result b/mysql-test/suite/binlog/r/binlog_bolt_2pc_recovery.result new file mode 100644 index 000000000000..a8d61e79f354 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_2pc_recovery.result @@ -0,0 +1,38 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +# Pre-TC-prepare crash rolls back. +include/assert.inc [A pre-TC-prepare crash rolls back all rows.] +# Post-TC-prepare crash rolls back. +include/assert.inc [A post-TC-prepare crash rolls back all rows.] +# Post-header-sync crash rolls back. +include/assert.inc [A post-header-sync crash rolls back all rows.] +# Post-purge-index-sync crash rolls back. +include/assert.inc [A post-purge-index-sync crash rolls back all rows.] +# Post-promote-rename crash rolls back. +include/assert.inc [A post-promote-rename crash rolls back all rows.] +# Post-rotate-event-sync crash rolls back and destroys the orphan. +include/assert.inc [A post-rotate-event-sync crash rolls back all rows.] +include/assert.inc [Recovery leaves no third binary log from the deleted orphan.] +include/assert.inc [The orphan's slot holds a freshly opened binary log.] +# Post-main-index-update crash commits exactly once. +include/assert.inc [A post-main-index-update crash commits all rows exactly once.] +include/assert.inc [A post-main-index-update crash preserves the original rows.] +# Post-purge-index-removal crash commits exactly once. +include/assert.inc [A post-purge-index-removal crash commits all rows exactly once.] +include/assert.inc [A post-purge-index-removal crash preserves the original rows.] +# Pre-engine-commit crash commits exactly once. +include/assert.inc [A pre-engine-commit crash commits all rows exactly once.] +include/assert.inc [A pre-engine-commit crash preserves the original rows.] +# Post-main-index XA PREPARE crash remains prepared and commits exactly once. +include/assert.inc [A post-main-index XA PREPARE remains prepared until XA COMMIT.] +include/assert.inc [XA COMMIT commits the recovered promoted rows exactly once.] +# Post-binlog-commit crash commits exactly once. +include/assert.inc [A post-binlog-commit crash commits all rows exactly once.] +include/assert.inc [A post-binlog-commit crash preserves the original rows.] +# Pre-rotation crash commits exactly once. +include/assert.inc [A pre-rotation crash commits all rows exactly once.] +include/assert.inc [A pre-rotation crash preserves the original rows.] +# Post-rotation crash commits exactly once. +include/assert.inc [A post-rotation crash commits all rows exactly once.] +include/assert.inc [A post-rotation crash preserves the original rows.] +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_commit_rollback.result b/mysql-test/suite/binlog/r/binlog_bolt_commit_rollback.result new file mode 100644 index 000000000000..6f23e4aff9e0 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_commit_rollback.result @@ -0,0 +1,232 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +Warnings: +Note 1051 Unknown table 'test.t2' +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (x INT PRIMARY KEY AUTO_INCREMENT, y LONGTEXT) ENGINE=InnoDB; +# Rollback removes the spill file and leaves the active binlog unchanged. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +ROLLBACK; +include/assert.inc [Rollback leaves the BOLT promotion count unchanged.] +include/assert.inc [Rollback keeps the active binary log unchanged.] +include/assert.inc [Rollback removes all transaction rows.] +# Retry the same transaction and promote a new active binary log. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +include/assert.inc [Retry promotes one new active binary log.] +include/assert.inc [Retry changes the active binary log file.] +include/assert.inc [Retry commits the four rows exactly once.] +# The first of two consecutive promotions on one session. +# The second promotion, whose predecessor is itself a promoted file. +include/assert.inc [Two consecutive transactions promote twice.] +include/assert.inc [Neither consecutive promotion is a missed optimization.] +include/assert.inc [The first promotion leaves the original active binary log.] +include/assert.inc [The second promotion opens a file of its own.] +include/assert.inc [Both consecutive promotions commit all eight rows.] +include/assert.inc [Both consecutive promotions preserve every payload in full.] +# XA PREPARE promotes the large transaction; XA COMMIT makes it visible. +XA START 'bolt_prepare_commit'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_prepare_commit'; +XA PREPARE 'bolt_prepare_commit'; +include/assert.inc [XA PREPARE promotes one large transaction.] +include/assert.inc [XA PREPARE keeps its rows invisible until resolution.] +XA COMMIT 'bolt_prepare_commit'; +include/assert.inc [XA COMMIT does not promote the prepared transaction again.] +include/assert.inc [XA COMMIT commits all prepared rows exactly once.] +# XA ROLLBACK resolves a promoted prepared transaction without committing rows. +XA START 'bolt_prepare_rollback'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_prepare_rollback'; +XA PREPARE 'bolt_prepare_rollback'; +include/assert.inc [XA PREPARE before rollback promotes one large transaction.] +XA ROLLBACK 'bolt_prepare_rollback'; +include/assert.inc [XA ROLLBACK does not promote the prepared transaction again.] +include/assert.inc [XA ROLLBACK removes all prepared rows.] +# XA COMMIT ONE PHASE promotes and commits a large transaction once. +XA START 'bolt_one_phase'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_one_phase'; +XA COMMIT 'bolt_one_phase' ONE PHASE; +include/assert.inc [XA COMMIT ONE PHASE promotes one large transaction.] +include/assert.inc [XA COMMIT ONE PHASE commits all rows exactly once.] +Warnings: +Note 1051 Unknown table 'test.t2' +# DDL implicitly commits and promotes the preceding large row transaction. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +CREATE TABLE t2 (id INT PRIMARY KEY) ENGINE=InnoDB; +include/assert.inc [The implicit pre-DDL commit promotes the large row transaction.] +include/assert.inc [DDL commits all preceding large transaction rows.] +include/assert.inc [DDL completes after the promoted transaction.] +# KILL QUERY after spill leaves the same session able to roll back and retry. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (CONCAT(REPEAT('abcdefghijklmnop', 250000), IF(GET_LOCK('bolt_kill_query_lock', 100), '', ''))); +KILL QUERY ID; +ERROR 70100: Query execution was interrupted +ROLLBACK; +include/assert.inc [KILL QUERY followed by rollback does not promote a binary log.] +include/assert.inc [KILL QUERY followed by rollback removes all transaction rows.] +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +include/assert.inc [The same killed-query session promotes on its retry.] +include/assert.inc [The same killed-query session commits all retry rows once.] +# Disconnecting a session with a spill removes the temporary file. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +include/assert.inc [Disconnecting an active spill does not promote a binary log.] +include/assert.inc [Disconnecting an active spill rolls back all rows.] +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +include/assert.inc [A new session promotes the disconnected transaction's retry.] +include/assert.inc [A new session commits all retry rows once.] +# A duplicate-key error preserves the spill until explicit rollback. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (1, REPEAT('abcdefghijklmnop', 250000)); +ERROR 23000: Duplicate entry '1' for key 't1.PRIMARY' +ROLLBACK; +include/assert.inc [A statement error followed by rollback does not promote a binary log.] +include/assert.inc [A statement error followed by rollback keeps the active binary log unchanged.] +include/assert.inc [A statement error followed by rollback removes all rows.] +SET GLOBAL max_binlog_size = 4096; +# A promoted oversized file becomes binary-log history and opens a fresh active file. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +SET GLOBAL max_binlog_size = 1073741824; +include/assert.inc [The oversized transaction is promoted once before rotation.] +include/assert.inc [Immediate max-binlog-size rotation opens a fresh active binary log.] +include/assert.inc [Immediate max-binlog-size rotation keeps all promoted rows.] +SHOW BINARY LOGS; +Log_name File_size Encrypted +binlog.000001 203 No +binlog.000002 16066082 No +binlog.000003 199 No +BEGIN; +INSERT INTO t1 (x, y) VALUES (101, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (102, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (103, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (104, REPEAT('abcdefghijklmnop', 250000)); +BEGIN; +INSERT INTO t1 (x, y) VALUES (201, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (202, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (203, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (204, REPEAT('abcdefghijklmnop', 250000)); +BEGIN; +INSERT INTO t1 (x, y) VALUES (301, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (302, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (303, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (304, REPEAT('abcdefghijklmnop', 250000)); +BEGIN; +INSERT INTO t1 (x, y) VALUES (401, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (402, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (403, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (404, REPEAT('abcdefghijklmnop', 250000)); +BEGIN; +INSERT INTO t1 (x, y) VALUES (501, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (502, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (503, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (504, REPEAT('abcdefghijklmnop', 250000)); +# Five waiting spills do not prevent a small transaction using the active binlog. +INSERT INTO t1 (x, y) VALUES (1, 'small transaction'); +include/assert.inc [The small transaction stays in the existing active binary log.] +include/assert.inc [Five waiting large transactions leave five temporary files.] +COMMIT; +COMMIT; +COMMIT; +ROLLBACK; +ROLLBACK; +include/assert.inc [Three committed spills produce exactly three BOLT promotions.] +include/assert.inc [Three committed spills create three new binary log files.] +include/assert.inc [Only the three committed large transactions and small transaction persist.] +include/assert.inc [The committed transaction ranges are preserved exactly.] +XA START 'bolt_concurrent_one_phase_1'; +INSERT INTO t1 (x, y) VALUES (101, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (102, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (103, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (104, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_1'; +XA START 'bolt_concurrent_one_phase_2'; +INSERT INTO t1 (x, y) VALUES (201, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (202, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (203, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (204, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_2'; +XA START 'bolt_concurrent_one_phase_3'; +INSERT INTO t1 (x, y) VALUES (301, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (302, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (303, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (304, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_3'; +XA START 'bolt_concurrent_prepare_1'; +INSERT INTO t1 (x, y) VALUES (401, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (402, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (403, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (404, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_prepare_1'; +XA START 'bolt_concurrent_prepare_2'; +INSERT INTO t1 (x, y) VALUES (501, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (502, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (503, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (504, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_prepare_2'; +# Five waiting XA branches leave the active binlog available for a small transaction. +INSERT INTO t1 (x, y) VALUES (1, 'small transaction'); +include/assert.inc [The small transaction stays in the existing active binary log with XA spills pending.] +XA ROLLBACK 'bolt_concurrent_one_phase_1'; +XA ROLLBACK 'bolt_concurrent_one_phase_2'; +XA COMMIT 'bolt_concurrent_one_phase_3' ONE PHASE; +XA PREPARE 'bolt_concurrent_prepare_1'; +XA ROLLBACK 'bolt_concurrent_prepare_1'; +XA PREPARE 'bolt_concurrent_prepare_2'; +XA COMMIT 'bolt_concurrent_prepare_2'; +include/assert.inc [One XA one-phase commit and two XA PREPARE paths produce three promotions.] +include/assert.inc [The XA commit decisions create exactly three new binary log files.] +include/assert.inc [Only the committed XA branches and small transaction persist.] +include/assert.inc [The committed XA transaction ranges are preserved exactly.] +DROP TABLE t2; +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_compression_fallback.result b/mysql-test/suite/binlog/r/binlog_bolt_compression_fallback.result new file mode 100644 index 000000000000..75818165efe0 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_compression_fallback.result @@ -0,0 +1,26 @@ +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# An uncompressed cache promotes a binary log. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 5000000)); +COMMIT; +include/assert.inc [An uncompressed cache promotes a binary log.] +include/assert.inc [An uncompressed cache does not increment the missed counter.] +include/assert.inc [The promoted commit opens its promoted binary log.] +include/assert.inc [The promoted transaction commits all rows.] +# A compressed cache uses the standard commit path. +SET @old_debug = @@SESSION.debug; +SET SESSION debug = "+d,force_large_trx_compression_fallback"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 5000000)); +COMMIT; +SET SESSION debug = @old_debug; +include/assert.inc [A compressed cache does not promote a binary log.] +include/assert.inc [A compressed cache increments the missed counter once.] +include/assert.inc [A compression fallback stays in the active binary log.] +include/assert.inc [A compression fallback commits all rows.] +include/assert.inc [A compression fallback preserves the full payload.] +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_encryption.result b/mysql-test/suite/binlog/r/binlog_bolt_encryption.result new file mode 100644 index 000000000000..0378a0f28daf --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_encryption.result @@ -0,0 +1,66 @@ +CALL mtr.add_suppression('Could not optimize large transaction execution in the binary log because'); +# ---------------------------------------------------------------------- +# Setup +# Creating local configuration file for keyring component: component_keyring_file +# Creating manifest file for current MySQL server instance +# Re-starting mysql server with manifest file +# ---------------------------------------------------------------------- +# Restart the server with the keyring available and encryption OFF. +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# A plaintext spilled file is promoted while encryption is OFF. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +include/assert.inc [A plaintext file with encryption OFF promotes a binary log.] +include/assert.inc [A plaintext file with encryption OFF does not increment the missed counter.] +include/assert.inc [The promoted commit opens a new active binary log.] +include/assert.inc [The promoted transaction commits all rows.] +include/assert.inc [The promoted binary log is plaintext while encryption is OFF.] +# Enabling encryption mid-transaction prevents promoting a plaintext file. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('d', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('e', 4000000)); +SET GLOBAL binlog_encryption = ON; +INSERT INTO t1 VALUES (3, REPEAT('f', 4000000)); +COMMIT; +include/assert.inc [Encryption enabled mid-transaction does not promote a binary log.] +include/assert.inc [Encryption enabled mid-transaction increments the missed counter once.] +include/assert.inc [The mid-transaction encryption fallback stays in the active binary log.] +include/assert.inc [The mid-transaction encryption fallback commits all rows.] +include/assert.inc [The mid-transaction encryption fallback preserves the full payload.] +include/assert.inc [Enabling encryption mid-transaction rotates the binary log.] +include/assert.inc [The mid-transaction encryption fallback wrote into an encrypted binary log.] +SET GLOBAL binlog_encryption = OFF; +# Turning encryption back OFF restores promotion. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('g', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('h', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('i', 4000000)); +COMMIT; +include/assert.inc [Promotion resumes once encryption is turned back OFF.] +include/assert.inc [The resumed promotion does not increment the missed counter.] +include/assert.inc [The resumed promotion opens a new active binary log.] +include/assert.inc [The resumed promotion commits all rows.] +SET GLOBAL binlog_encryption = ON; +# An encrypted spilled file is not promoted after encryption is turned OFF. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('j', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('k', 4000000)); +SET GLOBAL binlog_encryption = OFF; +INSERT INTO t1 VALUES (3, REPEAT('l', 4000000)); +COMMIT; +include/assert.inc [An encrypted file is not promoted after encryption is turned OFF.] +include/assert.inc [The encrypted-file fallback increments the missed counter once.] +include/assert.inc [The encrypted-file fallback stays in the active binary log.] +include/assert.inc [The encrypted-file fallback commits all rows.] +SET GLOBAL binlog_encryption = OFF; +DROP TABLE t1; +# ---------------------------------------------------------------------- +# Teardown +# Removing manifest file for current MySQL server instance +# Removing local keyring file for keyring component: component_keyring_file +# Removing local configuration file for keyring component: component_keyring_file +# Restarting server without the manifest file +# ---------------------------------------------------------------------- diff --git a/mysql-test/suite/binlog/r/binlog_bolt_error_action.result b/mysql-test/suite/binlog/r/binlog_bolt_error_action.result new file mode 100644 index 000000000000..79ca794b3c8c --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_error_action.result @@ -0,0 +1,15 @@ +CALL mtr.add_suppression("An error occurred during flush stage of the commit"); +CALL mtr.add_suppression("Binary logging not possible"); +CALL mtr.add_suppression("Hence turning logging off for the whole duration"); +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# A spill-sync failure with ABORT_SERVER aborts the server. +SET GLOBAL binlog_error_action = ABORT_SERVER; +include/assert.inc [An aborted spill-sync failure commits no rows.] +include/assert.inc [An aborted spill-sync failure writes nothing to the binary log.] +# A spill-sync failure with IGNORE_ERROR keeps the commit and stops logging. +SET GLOBAL binlog_error_action = IGNORE_ERROR; +include/assert.inc [An ignored spill-sync failure still commits all rows.] +# restart +include/assert_grep.inc [An ignored spill-sync failure writes no row events to the binary log.] +include/assert_grep.inc [An ignored spill-sync failure writes no transaction to the binary log.] +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_fallback.result b/mysql-test/suite/binlog/r/binlog_bolt_fallback.result new file mode 100644 index 000000000000..304a907d37a0 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_fallback.result @@ -0,0 +1,73 @@ +CALL mtr.add_suppression('Could not optimize large transaction execution in the binary log because'); +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE t2 (id INT PRIMARY KEY, marker INT) ENGINE=MyISAM; +# A large STATEMENT transaction uses the standard commit path. +SET SESSION binlog_format = STATEMENT; +SET @bolt_statement_payload = REPEAT('s', 12000000); +BEGIN; +INSERT INTO t1 VALUES (1, @bolt_statement_payload); +COMMIT; +SET @bolt_statement_payload = NULL; +include/assert.inc [A large STATEMENT transaction does not promote a binary log.] +include/assert.inc [A large STATEMENT transaction increments the missed counter once.] +include/assert.inc [A large STATEMENT transaction stays in the active binary log.] +include/assert.inc [A large STATEMENT fallback commits its row.] +# A large MIXED transaction with a statement event uses the standard path. +SET SESSION binlog_format = MIXED; +SET @bolt_mixed_payload = REPEAT('m', 12000000); +BEGIN; +INSERT INTO t1 VALUES (1, @bolt_mixed_payload); +COMMIT; +SET @bolt_mixed_payload = NULL; +include/assert.inc [A large MIXED transaction does not promote a binary log.] +include/assert.inc [A large MIXED transaction increments the missed counter once.] +include/assert.inc [A large MIXED transaction stays in the active binary log.] +include/assert.inc [A large MIXED fallback commits its row.] +# A checksum change during a large transaction uses the standard path. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('c', 5000000)); +SET GLOBAL binlog_checksum = IF(@@GLOBAL.binlog_checksum = 'CRC32', 'NONE', 'CRC32'); +INSERT INTO t1 VALUES (2, REPEAT('d', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('e', 5000000)); +COMMIT; +include/assert.inc [A checksum change does not promote a binary log.] +include/assert.inc [A checksum change increments the missed counter once.] +include/assert.inc [A checksum-change fallback commits all rows.] +# Restoring the original checksum before commit preserves promotion. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('r', 5000000)); +SET GLOBAL binlog_checksum = IF(@@GLOBAL.binlog_checksum = 'CRC32', 'NONE', 'CRC32'); +SET GLOBAL binlog_checksum = CRC32; +INSERT INTO t1 VALUES (2, REPEAT('s', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('t', 5000000)); +COMMIT; +include/assert.inc [Restoring the checksum before commit promotes the transaction.] +include/assert.inc [Restoring the checksum before commit does not increment the missed counter.] +include/assert.inc [The restored-checksum commit opens its promoted binary log.] +include/assert.inc [The restored-checksum promoted transaction commits all rows.] +include/assert.inc [The executed GTID set exceeds BOLT's reserved header.] +# A dynamically reserved Previous_gtids header preserves promotion. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('g', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('h', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('i', 5000000)); +COMMIT; +include/assert.inc [A dynamically reserved Previous_gtids header promotes a binary log.] +include/assert.inc [A dynamically reserved Previous_gtids header does not increment the missed counter.] +include/assert.inc [The dynamically reserved-header commit opens its promoted binary log.] +include/assert.inc [A dynamically reserved-header commit commits all rows.] +INSERT INTO t1 VALUES (1, REPEAT('n', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('o', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('p', 5000000)); +INSERT INTO t2 VALUES (1, 0), (2, 0), (3, 0); +# A mixed InnoDB/MyISAM row statement falls back for its statement cache. +BEGIN; +UPDATE t1 JOIN t2 USING (id) SET t1.data = CONCAT(t1.data, 'x'), t2.marker = 1; +COMMIT; +include/assert.inc [An InnoDB/MyISAM statement promotes its qualifying InnoDB cache.] +include/assert.inc [An InnoDB/MyISAM statement increments the missed counter once.] +include/assert.inc [An InnoDB/MyISAM statement rotates to the promoted InnoDB binary log.] +include/assert.inc [An InnoDB/MyISAM fallback commits every InnoDB row.] +include/assert.inc [An InnoDB/MyISAM fallback updates every MyISAM row.] +DROP TABLE t2; +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_fallback_debug.result b/mysql-test/suite/binlog/r/binlog_bolt_fallback_debug.result new file mode 100644 index 000000000000..72c4b0377a78 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_fallback_debug.result @@ -0,0 +1,30 @@ +CALL mtr.add_suppression('Could not optimize large transaction execution in the binary log because'); +CALL mtr.add_suppression("An incident event has been written to the binary log"); +CALL mtr.add_suppression("An error occurred during flush stage of the commit"); +CALL mtr.add_suppression("Binary logging not possible"); +CALL mtr.add_suppression("Hence turning logging off for the whole duration"); +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# A transaction with a logging incident uses the standard commit path. +SET SESSION debug = "+d,binlog_inject_incident"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('w', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('x', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('y', 5000000)); +COMMIT; +SET SESSION debug = "-d,binlog_inject_incident"; +include/assert.inc [An incident does not promote a binary log.] +include/assert.inc [An incident increments the missed counter once.] +include/assert.inc [An incident fallback force-rotates to a new active binary log.] +include/assert.inc [An incident fallback commits all rows.] +include/assert.inc [An incident fallback preserves the full payload.] +SET GLOBAL binlog_error_action = IGNORE_ERROR; +# A session spills a large transaction and holds it open. +# Another session's spill-sync failure turns logging off and closes the log. +# The held transaction commits through the standard path. +COMMIT; +include/assert.inc [A closed binary log does not promote.] +include/assert.inc [A closed binary log increments the missed counter once.] +include/assert.inc [A closed-log fallback still commits all rows.] +include/assert.inc [A closed-log fallback preserves the full payload.] +# restart +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_optimized_recovery.result b/mysql-test/suite/binlog/r/binlog_bolt_optimized_recovery.result new file mode 100644 index 000000000000..8775c42087f9 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_optimized_recovery.result @@ -0,0 +1,30 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# Commit a qualifying transaction for optimized recovery. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +include/assert.inc [The transaction was promoted into a new binary log file.] +# Kill and restart: --binlog-large-transaction-optimization-enabled=ON --binlog-large-transaction-optimization-threshold=10M --max-binlog-size=1G --binlog-transaction-compression=OFF +# Recovery logs the optimized large-transaction seek. +Pattern "Optimized binlog recovery by avoiding a sequential read of a large transaction body and seeking to its terminating event at offset" found +include/assert.inc [The committed promoted transaction survives recovery.] +# Commit a second qualifying transaction for corrupt-LTH recovery. +BEGIN; +INSERT INTO t1 VALUES (4, REPEAT('d', 4000000)); +INSERT INTO t1 VALUES (5, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (6, REPEAT('f', 4000000)); +COMMIT; +include/assert.inc [The second transaction was promoted into a new binary log file.] +# Truncate the active promoted file and reject the invalid LTH. +# Kill the server +Pattern "contains an invalid large transaction header" found +# restart +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_reserved_header_fallback.result b/mysql-test/suite/binlog/r/binlog_bolt_reserved_header_fallback.result new file mode 100644 index 000000000000..b579a8a9070f --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_reserved_header_fallback.result @@ -0,0 +1,26 @@ +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# A promotable large transaction promotes when the reserved region fits. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 5000000)); +COMMIT; +include/assert.inc [A fitting reserved region promotes a binary log.] +include/assert.inc [A fitting reserved region does not increment the missed counter.] +include/assert.inc [The promoted commit opens its promoted binary log.] +include/assert.inc [The promoted transaction commits all rows.] +# A reserved region too small for the header events uses the standard path. +SET @old_debug = @@SESSION.debug; +SET SESSION debug = "+d,force_large_trx_reserved_header_fallback"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 5000000)); +COMMIT; +SET SESSION debug = @old_debug; +include/assert.inc [A reserved region too small does not promote a binary log.] +include/assert.inc [A reserved region too small increments the missed counter once.] +include/assert.inc [A reserved-header fallback stays in the active binary log.] +include/assert.inc [A reserved-header fallback commits all rows.] +include/assert_grep.inc [The fallback transaction is present in the active binary log.] +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_rotate_purge.result b/mysql-test/suite/binlog/r/binlog_bolt_rotate_purge.result new file mode 100644 index 000000000000..b9106494ae48 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_rotate_purge.result @@ -0,0 +1,26 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +FLUSH BINARY LOGS; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +# Purge older history while the qualifying spill file is still open. +PURGE BINARY LOGS TO 'binlog.000002'; +# Rotate while the qualifying spill file is still open. +FLUSH BINARY LOGS; +include/assert.inc [FLUSH opens a new active binary log while a BOLT spill exists.] +COMMIT; +include/assert.inc [The spill promotes after PURGE and FLUSH complete.] +include/assert.inc [The promoted transaction preserves every row.] +# Rotate the promoted active file, then purge it as ordinary history. +FLUSH BINARY LOGS; +include/assert.inc [FLUSH rotates the promoted active binary log.] +PURGE BINARY LOGS TO 'binlog.000005'; +SHOW BINARY LOGS; +Log_name File_size Encrypted +binlog.000005 199 No +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_savepoint.result b/mysql-test/suite/binlog/r/binlog_bolt_savepoint.result new file mode 100644 index 000000000000..988424676af0 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_savepoint.result @@ -0,0 +1,76 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# Savepoint rollback below binlog_cache_size keeps the active binlog. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 512000)); +SAVEPOINT before_spill; +INSERT INTO t1 VALUES (2, REPEAT('b', 2000000)); +ROLLBACK TO SAVEPOINT before_spill; +COMMIT; +include/assert.inc [A final cache below binlog_cache_size is copied to the active binlog.] +include/assert.inc [A final cache below binlog_cache_size does not promote.] +include/assert.inc [A final cache below binlog_cache_size was never a candidate.] +include/assert.inc [Savepoint rollback retains only the pre-spill row.] +include/assert.inc [Savepoint rollback discards the post-savepoint row.] +# Savepoint rollback below the BOLT threshold keeps the active binlog. +BEGIN; +INSERT INTO t1 VALUES (11, REPEAT('c', 2000000)); +SAVEPOINT before_discard; +INSERT INTO t1 VALUES (12, REPEAT('d', 3000000)); +ROLLBACK TO SAVEPOINT before_discard; +COMMIT; +include/assert.inc [A final cache below the BOLT threshold is copied to the active binlog.] +include/assert.inc [A final cache below the BOLT threshold does not promote.] +include/assert.inc [A final cache below the BOLT threshold was never a candidate.] +include/assert.inc [Savepoint rollback retains the above-cache pre-savepoint row.] +include/assert.inc [Savepoint rollback discards the above-cache post-savepoint row.] +# Savepoint rollback above the BOLT threshold promotes a new binlog. +BEGIN; +INSERT INTO t1 VALUES (21, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (22, REPEAT('f', 4000000)); +INSERT INTO t1 VALUES (23, REPEAT('g', 4000000)); +SAVEPOINT before_discard; +INSERT INTO t1 VALUES (24, REPEAT('h', 2000000)); +ROLLBACK TO SAVEPOINT before_discard; +COMMIT; +include/assert.inc [A final cache above the BOLT threshold promotes a new active binlog.] +include/assert.inc [A final cache above the BOLT threshold increments the optimization counter once.] +include/assert.inc [A promoted savepoint transaction does not increment the missed counter.] +include/assert.inc [Savepoint rollback retains the promoted pre-savepoint rows.] +include/assert.inc [Savepoint rollback discards the promoted post-savepoint row.] +# Re-append after rollback below the threshold keeps the active binlog. +BEGIN; +INSERT INTO t1 VALUES (31, REPEAT('a', 2000000)); +SAVEPOINT sp_reappend; +INSERT INTO t1 VALUES (32, REPEAT('b', 3000000)); +ROLLBACK TO SAVEPOINT sp_reappend; +INSERT INTO t1 VALUES (33, REPEAT('c', 3000000)); +COMMIT; +include/assert.inc [A re-appended cache below the threshold is copied to the active binlog.] +include/assert.inc [A re-appended cache below the threshold does not promote.] +include/assert.inc [A re-appended cache below the threshold was never a candidate.] +include/assert.inc [Re-append after rollback keeps the pre-savepoint and re-appended rows.] +include/assert.inc [Re-append after rollback discards only the rolled-back row.] +include/assert.inc [The rolled-back row is absent after re-append.] +include/assert.inc [The re-appended row has its full length.] +# Re-append after rollback above the threshold promotes a new binlog. +BEGIN; +INSERT INTO t1 VALUES (41, REPEAT('d', 5000000)); +SAVEPOINT sp_recross; +INSERT INTO t1 VALUES (42, REPEAT('e', 8000000)); +ROLLBACK TO SAVEPOINT sp_recross; +INSERT INTO t1 VALUES (43, REPEAT('f', 8000000)); +COMMIT; +include/assert.inc [A re-appended cache above the threshold promotes a new active binlog.] +include/assert.inc [A re-appended cache above the threshold increments the optimization counter once.] +include/assert.inc [A re-crossed promotion does not increment the missed counter.] +include/assert.inc [Re-crossing the threshold keeps the pre-savepoint and re-appended rows.] +include/assert.inc [Re-crossing the threshold discards only the rolled-back row.] +include/assert.inc [The rolled-back row is absent after promotion.] +include/assert.inc [The re-appended row has its full length in the promoted file.] +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_savepoint_recovery.result b/mysql-test/suite/binlog/r/binlog_bolt_savepoint_recovery.result new file mode 100644 index 000000000000..0390f5badf35 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_savepoint_recovery.result @@ -0,0 +1,10 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +# Post-main-index-update crash after a savepoint rollback rolls forward. +include/assert.inc [A post-main-index-update crash commits the kept rows.] +include/assert.inc [The rolled-back row is not recovered.] +include/assert.inc [The last kept row survived the chsize trim intact.] +# Post-promote-rename crash after a savepoint rollback rolls back. +include/assert.inc [A post-promote-rename crash rolls back all rows.] +include/assert.inc [Recovery left no third, orphaned binary log.] +DROP TABLE t1; diff --git a/mysql-test/suite/binlog/r/binlog_bolt_sysvars.result b/mysql-test/suite/binlog/r/binlog_bolt_sysvars.result new file mode 100644 index 000000000000..b8acf7ea84e1 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_sysvars.result @@ -0,0 +1,95 @@ +# restart: +# Case 1: BOLT is ON; binlog_cache_size > binlog_large_transaction_optimization_threshold. +include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 20971520; +Warnings: +Warning 6914 Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size. +include/assert_error_log.inc [server: 1, pattern: Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.]] +include/assert.inc [Increasing binlog_cache_size raises the optimization threshold.] +include/save_error_log_position.inc +# restart: --binlog-large-transaction-optimization-enabled=ON --binlog-large-transaction-optimization-threshold=10M --binlog-cache-size=20M +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +threshold +20971520 +include/assert_error_log.inc [server: 1, pattern: Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.]] +# Case 2: BOLT is ON; binlog_large_transaction_optimization_threshold < binlog_cache_size. +include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +Warnings: +Warning 6914 Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size. +include/assert_error_log.inc [server: 1, pattern: Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.]] +include/assert.inc [A lower optimization threshold is raised to match binlog_cache_size.] +# Case 3: BOLT is ON; binlog_large_transaction_optimization_threshold > binlog_cache_size. +include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 22020096; +include/assert_error_log.inc [server: 1, pattern: NONE] +include/assert.inc [A threshold above binlog_cache_size is unchanged.] +include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 10485760; +include/assert_error_log.inc [server: 1, pattern: NONE] +include/assert.inc [Lowering binlog_cache_size does not lower the threshold.] +include/save_error_log_position.inc +# restart: --binlog-large-transaction-optimization-enabled=ON --binlog-cache-size=20M --binlog-large-transaction-optimization-threshold=21M +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +threshold +22020096 +include/assert_error_log.inc [server: 1, pattern: NONE] +# restart: +# Case 4: BOLT is OFF; binlog_cache_size > binlog_large_transaction_optimization_threshold. +include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 20971520; +include/assert_error_log.inc [server: 1, pattern: NONE] +include/assert.inc [Increasing binlog_cache_size raises the threshold while BOLT is off.] +include/save_error_log_position.inc +# restart: --binlog-large-transaction-optimization-enabled=OFF --binlog-large-transaction-optimization-threshold=10M --binlog-cache-size=20M +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +threshold +20971520 +include/assert_error_log.inc [server: 1, pattern: NONE] +# Case 5: BOLT is OFF; binlog_large_transaction_optimization_threshold < binlog_cache_size. +include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +include/assert_error_log.inc [server: 1, pattern: NONE] +include/assert.inc [A lower threshold is raised while BOLT is off.] +# Case 6: BOLT is OFF; binlog_large_transaction_optimization_threshold > binlog_cache_size. +include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 22020096; +include/assert_error_log.inc [server: 1, pattern: NONE] +include/assert.inc [A threshold above binlog_cache_size is unchanged while BOLT is off.] +include/save_error_log_position.inc +# restart: --binlog-large-transaction-optimization-enabled=OFF --binlog-cache-size=20M --binlog-large-transaction-optimization-threshold=21M +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +threshold +22020096 +include/assert_error_log.inc [server: 1, pattern: NONE] +# restart: +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# Case 7: Turning the knob off mid-transaction does not affect it. +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +include/assert.inc [A transaction latched with the knob on still promotes after it is turned off.] +include/assert.inc [The knob-off promotion commits all rows.] +# Case 8: Turning the knob on mid-transaction does not affect it either. +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +include/assert.inc [A transaction latched with the knob off does not promote after it is turned on.] +include/assert.inc [A transaction latched with the knob off is not a missed optimization.] +include/assert.inc [The knob-on transaction still commits all rows.] +# Case 9: Raising the threshold mid-transaction does not affect it. +SET GLOBAL binlog_large_transaction_optimization_threshold = 1073741824; +include/assert.inc [A transaction latched below the threshold still promotes after it is raised.] +include/assert.inc [The raised-threshold promotion commits all rows.] +# Case 10: a cache emptied by a statement rollback re-reads the knob. +BEGIN; +COMMIT; +include/assert.inc [The rolled-back statement left the cache empty.] +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +include/assert.inc [A cache emptied by a statement rollback does not promote the next transaction after the knob is turned off.] +# Case 11: a cache emptied by a savepoint rollback re-reads the knob. +BEGIN; +SAVEPOINT s1; +ROLLBACK TO SAVEPOINT s1; +COMMIT; +include/assert.inc [The savepoint rollback left the cache empty.] +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +include/assert.inc [A cache emptied by a savepoint rollback does not promote the next transaction after the knob is turned off.] +DROP TABLE t1; +# restart: diff --git a/mysql-test/suite/binlog/r/binlog_bolt_tmp_dir_cleanup.result b/mysql-test/suite/binlog/r/binlog_bolt_tmp_dir_cleanup.result new file mode 100644 index 000000000000..0f0468caaa95 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_bolt_tmp_dir_cleanup.result @@ -0,0 +1,14 @@ +Created 3 managed BOLT spill files +# restart +Startup cleanup removed all managed BOLT spill files +# Remove a pre-rename mkstemp name containing upper case. +# restart +Startup cleanup removed the pre-rename mkstemp files +# Reject a symbolic link for #binlog_temp_files. +# restart +# Reject an unrecognized regular entry. +# restart +# Reject a symbolic-link entry, even with a managed file name. +# restart +# Reject a nested directory, even with a managed file name. +# restart diff --git a/mysql-test/suite/binlog/r/binlog_encryption_random_access.result b/mysql-test/suite/binlog/r/binlog_encryption_random_access.result index 6fbe198b995d..b10cb177653d 100644 --- a/mysql-test/suite/binlog/r/binlog_encryption_random_access.result +++ b/mysql-test/suite/binlog/r/binlog_encryption_random_access.result @@ -9,7 +9,19 @@ CALL mtr.add_suppression('Unsafe statement written to the binary log using state CREATE TABLE t1 (c1 INT PRIMARY KEY, c2 TEXT, pos INT); # Inserting 100 random transaction # Asserting we can show binlog events from each transaction -DROP TABLE t1; +# Binlog encryption forces a qualifying row transaction to use the standard path. +Warnings: +Warning 1287 '@@binlog_format' is deprecated and will be removed in a future release. +CREATE TABLE t_large (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +BEGIN; +INSERT INTO t_large VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t_large VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t_large VALUES (3, REPEAT('c', 5000000)); +COMMIT; +include/assert.inc [An encrypted binary log uses the standard path.] +include/assert.inc [An encrypted binary log increments the missed counter once.] +include/assert.inc [The encrypted fallback transaction commits all rows.] +DROP TABLE t_large, t1; # ---------------------------------------------------------------------- # Teardown # Removing manifest file for current MySQL server instance diff --git a/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery-master.opt b/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery-master.opt new file mode 100644 index 000000000000..638d53cb49e8 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery-master.opt @@ -0,0 +1,5 @@ +--binlog-large-transaction-optimization-enabled=ON +--binlog-large-transaction-optimization-threshold=10M +--max-binlog-size=1G +--innodb-flush-log-at-trx-commit=1 +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery.test b/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery.test new file mode 100644 index 000000000000..63e14578942e --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_2pc_recovery.test @@ -0,0 +1,493 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT crash recovery follows the binary-log 2PC decision. +# +# === Requirements === +# R1. A crash before the main-index update rolls the transaction back. +# R2. A crash after the main-index update recovers the transaction once. +# R3. A post-main-index XA PREPARE remains prepared until XA COMMIT. +# +# === Implementation === +# 1. Initialize BOLT and a clean test table. +# 2. Crash at each durable decision boundary around promotion. +# 3. Restart and verify rollback or rollforward according to the boundary. +# 4. Verify the same post-main-index decision for XA PREPARE. +# +--source include/not_crashrep.inc +--source include/not_valgrind.inc +--source include/have_debug.inc +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_TEMP_DIR = $MYSQLD_DATADIR/#binlog_temp_files + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_max_binlog_size = query_get_value(SELECT @@GLOBAL.max_binlog_size, @@GLOBAL.max_binlog_size, 1) +--let $save_sync_binlog = query_get_value(SELECT @@GLOBAL.sync_binlog, @@GLOBAL.sync_binlog, 1) + +# Setup +--let $BOLT_RESTART = restart: +--let $BOLT_RESTART = $BOLT_RESTART --binlog-large-transaction-optimization-enabled=ON +--let $BOLT_RESTART = $BOLT_RESTART --binlog-large-transaction-optimization-threshold=10M +--let $BOLT_RESTART = $BOLT_RESTART --max-binlog-size=1G +--let $BOLT_RESTART = $BOLT_RESTART --sync-binlog=1 +--let $BOLT_RESTART = $BOLT_RESTART --innodb-flush-log-at-trx-commit=1 +# REPEAT() payloads compress to almost nothing, so a compressed cache would +# never reach the threshold and the transaction would never be promoted. +--let $BOLT_RESTART = $BOLT_RESTART --binlog-transaction-compression=OFF + +--disable_query_log +DROP TABLE IF EXISTS t1; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +RESET BINARY LOGS AND GTIDS; +--enable_query_log + +############################################################################ +# Case 1: Crash before TC prepare. +# Expected behavior: rollback. +############################################################################ +--echo # Pre-TC-prepare crash rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +SET SESSION DEBUG="+d,crash_before_tc_prepare"; +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A pre-TC-prepare crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 2: Crash after TC prepare before TC commit. +# Expected behavior: rollback. +############################################################################ +--echo # Post-TC-prepare crash rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +SET SESSION DEBUG="+d,crash_after_tc_prepare"; +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-TC-prepare crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 3: Crash during binlog commit after durable header in promoted file. +# Expected behavior: rollback. +############################################################################ +--echo # Post-header-sync crash rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_header_sync"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-header-sync crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 4: Crash during binlog commit after writing to purge_index_file. +# Expected behavior: rollback. +############################################################################ +--echo # Post-purge-index-sync crash rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_purge_index_sync"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-purge-index-sync crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 5: Crash during binlog commit after promoting the file. +# Expected behavior: rollback. +############################################################################ +--echo # Post-promote-rename crash rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_promote_rename"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-promote-rename crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 6: Crash after syncing the predecessor's Rotate event, before the +# main-index update. +# Expected behavior: rollback. +# +# At this instant the promoted file exists under its final name and the +# predecessor's Rotate event points at it, but only the purge record owns it: +# the main index does not list it yet. Recovery must delete the file and roll +# the transaction back. +# +# Resets the binary log first so the file count is deterministic. Afterwards +# there must be just the predecessor and the file opened at startup; a third +# would mean the orphan was kept. +############################################################################ +--echo # Post-rotate-event-sync crash rolls back and destroys the orphan. +--disable_query_log +RESET BINARY LOGS AND GTIDS; +--enable_query_log +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_rotate_event_sync"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-rotate-event-sync crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--let $bolt_third_log = query_get_value(SHOW BINARY LOGS, Log_name, 3) +--let $assert_text = Recovery leaves no third binary log from the deleted orphan. +--let $assert_cond = "$bolt_third_log" = "No such row" +--source include/assert.inc +# The orphan's slot holds a freshly opened log, not the 12 MB promoted file. +--let $bolt_second_log_size = query_get_value(SHOW BINARY LOGS, File_size, 2) +--let $assert_text = The orphan's slot holds a freshly opened binary log. +--let $assert_cond = $bolt_second_log_size < 1000000 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 7: Crash during binlog commit after writing to main index. +# Expected behavior: rollforward. +############################################################################ +--echo # Post-main-index-update crash commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_main_index_update"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-main-index-update crash commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = A post-main-index-update crash preserves the original rows. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 8: Crash during binlog commit after removing entry from purge_index_file. +# Expected behavior: rollforward. +############################################################################ +--echo # Post-purge-index-removal crash commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_purge_index_remove"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-purge-index-removal crash commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = A post-purge-index-removal crash preserves the original rows. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 9: Crash after binlog commit before innodb commit. +# Expected behavior: commit. +############################################################################ +--echo # Pre-engine-commit crash commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_before_engine_commit"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A pre-engine-commit crash commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = A pre-engine-commit crash preserves the original rows. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 10: XA PREPARE crash after main-index update. +# Expected behavior: remain prepared, then commit. +############################################################################ +--echo # Post-main-index XA PREPARE crash remains prepared and commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_main_index_update"; +XA START 'bolt_post_index'; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +XA END 'bolt_post_index'; +--error 2013 +XA PREPARE 'bolt_post_index'; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +# Case 10 intentionally leaves one XA transaction prepared for recovery. +--exec echo "$BOLT_RESTART --log-error-suppression-list=MY-010225" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--disable_query_log +--disable_result_log +XA RECOVER; +XA COMMIT 'bolt_post_index'; +--enable_result_log +--enable_query_log +--let $assert_text = A post-main-index XA PREPARE remains prepared until XA COMMIT. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = XA COMMIT commits the recovered promoted rows exactly once. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 11: Crash after the whole BOLT commit returned. +# Expected behavior: commit. +# +# This is past every 2PC boundary: the promoted file is in the index and the +# engines are committed, so the crash only proves nothing later can undo it. +# The crash point is gated on used_bolt_promotion, so reaching it is itself +# evidence the optimized path ran. +############################################################################ +--echo # Post-binlog-commit crash commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_binlog_commit"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-binlog-commit crash commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = A post-binlog-commit crash preserves the original rows. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Cases 11 and 12: crash around the post-promotion max_binlog_size rotation. +# +# rotate_if_needed() runs after the transaction is already committed in both +# the binary log and the engines, so neither crash may lose it. These cases +# need max_binlog_size below the promoted file's size to reach the rotation at +# all, so they restart with a small limit; everything above uses 1G precisely +# so that no rotation interferes. +############################################################################ +--let $BOLT_RESTART_SMALL_MAX = restart: +--let $BOLT_RESTART_SMALL_MAX = $BOLT_RESTART_SMALL_MAX --binlog-large-transaction-optimization-enabled=ON +--let $BOLT_RESTART_SMALL_MAX = $BOLT_RESTART_SMALL_MAX --binlog-large-transaction-optimization-threshold=10M +--let $BOLT_RESTART_SMALL_MAX = $BOLT_RESTART_SMALL_MAX --max-binlog-size=4096 +--let $BOLT_RESTART_SMALL_MAX = $BOLT_RESTART_SMALL_MAX --sync-binlog=1 +--let $BOLT_RESTART_SMALL_MAX = $BOLT_RESTART_SMALL_MAX --innodb-flush-log-at-trx-commit=1 + +--let $restart_parameters = $BOLT_RESTART_SMALL_MAX +--source include/restart_mysqld_no_echo.inc + +############################################################################ +# Case 12: Crash before the post-promotion rotation. +# Expected behavior: commit; the rotation simply never happened. +############################################################################ +--echo # Pre-rotation crash commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_before_max_size_rotate"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART_SMALL_MAX" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A pre-rotation crash commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = A pre-rotation crash preserves the original rows. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 13: Crash after the post-promotion rotation. +# Expected behavior: commit; the rotation completed before the crash. +############################################################################ +--echo # Post-rotation crash commits exactly once. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_max_size_rotate"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART_SMALL_MAX" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--let $assert_text = A post-rotation crash commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = A post-rotation crash preserves the original rows. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 6 +--source include/assert.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +--enable_query_log + +DROP TABLE t1; +--disable_query_log +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL max_binlog_size = $save_max_binlog_size +--eval SET GLOBAL sync_binlog = $save_sync_binlog +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback-master.opt b/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback-master.opt new file mode 100644 index 000000000000..0123ec2d69f6 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback-master.opt @@ -0,0 +1,4 @@ +--binlog-cache-size=1M +--binlog-large-transaction-optimization-enabled=ON +--binlog-large-transaction-optimization-threshold=10M +--max-binlog-size=1G diff --git a/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback.test b/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback.test new file mode 100644 index 000000000000..fdb6e4f3107f --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_commit_rollback.test @@ -0,0 +1,668 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT safely discards rolled-back spill files and handles normal, XA, +# and DDL transaction endings without leaving stale promotion state. +# +# === Requirements === +# R1. A rolled-back spilled transaction leaves no temporary file or binlog. +# R2. Retrying the same four-insert transaction commits as one promoted binlog. +# R2b. Two promotions back to back on one session each open their own binary +# log with a valid header. +# R3. XA PREPARE resolves correctly through both XA COMMIT and XA ROLLBACK. +# R4. XA COMMIT ONE PHASE and the implicit commit before DDL are promoted. +# R5. A killed query after spill can roll back and retry on the same session. +# R6. Disconnecting with an active spill discards it; a new session can retry. +# R7. A statement error after spill leaves the transaction rollback-safe. +# R8. A promoted file larger than max_binlog_size immediately rotates. +# R9. Five simultaneously spilled transactions leave the active binlog usable +# for a small transaction and promote only the three that commit. +# R10. The same five-spill boundary handles XA one-phase and prepared branches +# with the correct commit and rollback decisions. +# R11. Every promoted file has a checksummed Previous_gtids, LTH, GTID prefix +# with the GTID dependency clock reset for its new binary log. +# +# === Implementation === +# The source uses a 1 MiB cache and a 10 MiB BOLT threshold. Four 4 MB rows +# are used because four REPEAT('abcdefghijklmnop', 1000) values are only 64 KB, +# below BOLT's required promotion threshold. The killed-query case uses a user +# lock and PROCESSLIST polling rather than timing assumptions. The concurrent +# cases use five independent sessions with non-overlapping primary-key ranges. +# Each promoted file is decoded with checksum verification before its header +# ordering and logical timestamps are asserted. Transaction compression is +# disabled because BOLT and size-triggered rotation require an uncompressed +# promoted file. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc +--source include/not_binlog_transaction_compression_on.inc + +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_TEMP_DIR = $MYSQLD_DATADIR/#binlog_temp_files +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_max_binlog_size = query_get_value(SELECT @@GLOBAL.max_binlog_size, @@GLOBAL.max_binlog_size, 1) + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +--enable_query_log +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (x INT PRIMARY KEY AUTO_INCREMENT, y LONGTEXT) ENGINE=InnoDB; + +############################################################################ +# Case 1: Roll back a spilled transaction, then retry and commit. +############################################################################ +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Rollback removes the spill file and leaves the active binlog unchanged. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +ROLLBACK; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = Rollback leaves the BOLT promotion count unchanged. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = Rollback keeps the active binary log unchanged. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = Rollback removes all transaction rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc + +--echo # Retry the same transaction and promote a new active binary log. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +COMMIT; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_header_file = $binlog_file_after +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = Retry promotes one new active binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = Retry changes the active binary log file. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = Retry commits the four rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc + +############################################################################ +# Case 2: Two promotions back to back on the same session. +# +# A promoted commit resets its cache with the spilled file preserved, since +# that file is now a binary log. This checks the cache is reusable afterwards: +# a stale descriptor, reservation size or terminating-event offset carried into +# the second transaction would corrupt its header while leaving the rows +# intact, so both promoted files are validated. +# +# It is also the only case where a promotion's predecessor is itself a promoted +# file, which the second promotion names itself after and chains its Rotate +# event to. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # The first of two consecutive promotions on one session. +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +--enable_result_log +--enable_query_log +--let $bolt_first_promoted = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_header_file = $bolt_first_promoted +--source ../inc/validate_bolt_header.inc + +--echo # The second promotion, whose predecessor is itself a promoted file. +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +--enable_result_log +--enable_query_log +--let $bolt_second_promoted = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_header_file = $bolt_second_promoted +--source ../inc/validate_bolt_header.inc +--exec test ! -e $BOLT_TEMP_DIR/bolt_* + +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text = Two consecutive transactions promote twice. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 2 +--source include/assert.inc +--let $assert_text = Neither consecutive promotion is a missed optimization. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = The first promotion leaves the original active binary log. +--let $assert_cond = "$bolt_first_promoted" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = The second promotion opens a file of its own. +--let $assert_cond = "$bolt_second_promoted" != "$bolt_first_promoted" +--source include/assert.inc +--let $assert_text = Both consecutive promotions commit all eight rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 8 +--source include/assert.inc +--let $assert_text = Both consecutive promotions preserve every payload in full. +--let $assert_cond = [SELECT SUM(OCTET_LENGTH(y)) AS payload FROM t1, payload, 1] = 32000000 +--source include/assert.inc + +############################################################################ +# Case 3: XA PREPARE followed by XA COMMIT. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--echo # XA PREPARE promotes the large transaction; XA COMMIT makes it visible. +XA START 'bolt_prepare_commit'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_prepare_commit'; +XA PREPARE 'bolt_prepare_commit'; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after_prepare = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = XA PREPARE promotes one large transaction. +--let $assert_cond = $bolt_count_after_prepare = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = XA PREPARE keeps its rows invisible until resolution. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +XA COMMIT 'bolt_prepare_commit'; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = XA COMMIT does not promote the prepared transaction again. +--let $assert_cond = $bolt_count_after = $bolt_count_after_prepare +--source include/assert.inc +--let $assert_text = XA COMMIT commits all prepared rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc + +############################################################################ +# Case 4: XA PREPARE followed by XA ROLLBACK. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--echo # XA ROLLBACK resolves a promoted prepared transaction without committing rows. +XA START 'bolt_prepare_rollback'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_prepare_rollback'; +XA PREPARE 'bolt_prepare_rollback'; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after_prepare = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = XA PREPARE before rollback promotes one large transaction. +--let $assert_cond = $bolt_count_after_prepare = $bolt_count_before + 1 +--source include/assert.inc +XA ROLLBACK 'bolt_prepare_rollback'; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = XA ROLLBACK does not promote the prepared transaction again. +--let $assert_cond = $bolt_count_after = $bolt_count_after_prepare +--source include/assert.inc +--let $assert_text = XA ROLLBACK removes all prepared rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc + +############################################################################ +# Case 5: XA COMMIT ONE PHASE. +############################################################################ +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--echo # XA COMMIT ONE PHASE promotes and commits a large transaction once. +XA START 'bolt_one_phase'; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_one_phase'; +XA COMMIT 'bolt_one_phase' ONE PHASE; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = XA COMMIT ONE PHASE promotes one large transaction. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = XA COMMIT ONE PHASE commits all rows exactly once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc + +############################################################################ +# Case 6: Large DML transaction followed by DDL. +############################################################################ +--disable_query_log +TRUNCATE t1; +DROP TABLE IF EXISTS t2; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--echo # DDL implicitly commits and promotes the preceding large row transaction. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +CREATE TABLE t2 (id INT PRIMARY KEY) ENGINE=InnoDB; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = The implicit pre-DDL commit promotes the large row transaction. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = DDL commits all preceding large transaction rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc +--let $assert_text = DDL completes after the promoted transaction. +--let $ddl_table_exists = query_get_value(SELECT COUNT(*) AS count FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2', count, 1) +--let $assert_cond = $ddl_table_exists = 1 +--source include/assert.inc + +############################################################################ +# Case 7: KILL QUERY after spill, rollback, and same-session retry. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +connect (kill_blocker, localhost, root,,); +connect (kill_victim, localhost, root,,); +connection kill_blocker; +--disable_query_log +--disable_result_log +SELECT GET_LOCK('bolt_kill_query_lock', 0); +--enable_result_log +--enable_query_log +connection kill_victim; +--let $victim_id = query_get_value(SELECT CONNECTION_ID() AS id, id, 1) +--echo # KILL QUERY after spill leaves the same session able to roll back and retry. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +send INSERT INTO t1 (y) VALUES (CONCAT(REPEAT('abcdefghijklmnop', 250000), IF(GET_LOCK('bolt_kill_query_lock', 100), '', ''))); +connection kill_blocker; +--let $wait_condition = SELECT COUNT(*) = 1 FROM information_schema.processlist WHERE ID = $victim_id AND STATE = 'User lock'; +--source include/wait_condition_or_abort.inc +--replace_result $victim_id ID +--eval KILL QUERY $victim_id +connection kill_victim; +--error ER_QUERY_INTERRUPTED +reap; +ROLLBACK; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = KILL QUERY followed by rollback does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = KILL QUERY followed by rollback removes all transaction rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = The same killed-query session promotes on its retry. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = The same killed-query session commits all retry rows once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc +connection kill_blocker; +--disable_query_log +--disable_result_log +SELECT RELEASE_LOCK('bolt_kill_query_lock'); +--enable_result_log +--enable_query_log +connection kill_victim; +disconnect kill_victim; +connection kill_blocker; +disconnect kill_blocker; +connection default; + +############################################################################ +# Case 8: Disconnect with an active spill, then retry from a new session. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--source include/count_sessions.inc +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +connect (spill_client, localhost, root,,); +connection spill_client; +--echo # Disconnecting a session with a spill removes the temporary file. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +disconnect spill_client; +connection default; +--source include/wait_until_count_sessions.inc +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = Disconnecting an active spill does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = Disconnecting an active spill rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +connect (spill_retry, localhost, root,,); +connection spill_retry; +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +connection default; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = A new session promotes the disconnected transaction's retry. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = A new session commits all retry rows once. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc +connection spill_retry; +disconnect spill_retry; +connection default; +--source include/wait_until_count_sessions.inc + +############################################################################ +# Case 9: Statement error after spill, followed by rollback. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A duplicate-key error preserves the spill until explicit rollback. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +--error ER_DUP_ENTRY +INSERT INTO t1 (x, y) VALUES (1, REPEAT('abcdefghijklmnop', 250000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null +ROLLBACK; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A statement error followed by rollback does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A statement error followed by rollback keeps the active binary log unchanged. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A statement error followed by rollback removes all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc + +############################################################################ +# Case 10: A promoted file larger than max_binlog_size rotates immediately. +############################################################################ +--disable_query_log +TRUNCATE t1; +RESET BINARY LOGS AND GTIDS; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +SET GLOBAL max_binlog_size = 4096; +--echo # A promoted oversized file becomes binary-log history and opens a fresh active file. +BEGIN; +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (y) VALUES (REPEAT('abcdefghijklmnop', 250000)); +COMMIT; +SET GLOBAL max_binlog_size = 1073741824; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--exec test -e $MYSQLD_DATADIR/binlog.000002 +--exec test -e $MYSQLD_DATADIR/binlog.000003 +--let $bolt_header_file = query_get_value(SHOW BINARY LOGS, Log_name, 2) +--source ../inc/validate_bolt_header.inc +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = The oversized transaction is promoted once before rotation. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = Immediate max-binlog-size rotation opens a fresh active binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = Immediate max-binlog-size rotation keeps all promoted rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 4 +--source include/assert.inc +SHOW BINARY LOGS; + +############################################################################ +# Case 11: Five spilled transactions with regular commit and rollback. +############################################################################ +--disable_query_log +TRUNCATE t1; +RESET BINARY LOGS AND GTIDS; +--enable_query_log +--source include/count_sessions.inc +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +connect (promote_one, localhost, root,,); +BEGIN; +INSERT INTO t1 (x, y) VALUES (101, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (102, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (103, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (104, REPEAT('abcdefghijklmnop', 250000)); +connect (promote_two, localhost, root,,); +BEGIN; +INSERT INTO t1 (x, y) VALUES (201, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (202, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (203, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (204, REPEAT('abcdefghijklmnop', 250000)); +connect (promote_three, localhost, root,,); +BEGIN; +INSERT INTO t1 (x, y) VALUES (301, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (302, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (303, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (304, REPEAT('abcdefghijklmnop', 250000)); +connect (promote_four, localhost, root,,); +BEGIN; +INSERT INTO t1 (x, y) VALUES (401, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (402, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (403, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (404, REPEAT('abcdefghijklmnop', 250000)); +connect (promote_five, localhost, root,,); +BEGIN; +INSERT INTO t1 (x, y) VALUES (501, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (502, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (503, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (504, REPEAT('abcdefghijklmnop', 250000)); +connection default; +--let $binlog_file_before_small = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Five waiting spills do not prevent a small transaction using the active binlog. +INSERT INTO t1 (x, y) VALUES (1, 'small transaction'); +--let $binlog_file_after_small = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--exec ls $BOLT_TEMP_DIR/bolt_* | wc -l | grep -E '^[[:space:]]*5[[:space:]]*$' > /dev/null +--let $assert_text = The small transaction stays in the existing active binary log. +--let $assert_cond = "$binlog_file_after_small" = "$binlog_file_before_small" +--source include/assert.inc +--let $assert_text = Five waiting large transactions leave five temporary files. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 1 +--source include/assert.inc +connection promote_one; +COMMIT; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +connection promote_three; +COMMIT; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +connection promote_five; +COMMIT; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +connection promote_two; +ROLLBACK; +connection promote_four; +ROLLBACK; +connection default; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = Three committed spills produce exactly three BOLT promotions. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 3 +--source include/assert.inc +--let $assert_text = Three committed spills create three new binary log files. +--let $assert_cond = "$binlog_file_after" = "binlog.000004" +--source include/assert.inc +--let $assert_text = Only the three committed large transactions and small transaction persist. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 13 +--source include/assert.inc +--let $assert_text = The committed transaction ranges are preserved exactly. +--let $assert_cond = [SELECT SUM(x) AS id_sum FROM t1, id_sum, 1] = 3631 +--source include/assert.inc + +############################################################################ +# Case 12: Five spilled XA branches with one-phase and prepared resolution. +############################################################################ +--disable_query_log +TRUNCATE t1; +RESET BINARY LOGS AND GTIDS; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +connection promote_one; +XA START 'bolt_concurrent_one_phase_1'; +INSERT INTO t1 (x, y) VALUES (101, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (102, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (103, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (104, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_1'; +connection promote_two; +XA START 'bolt_concurrent_one_phase_2'; +INSERT INTO t1 (x, y) VALUES (201, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (202, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (203, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (204, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_2'; +connection promote_three; +XA START 'bolt_concurrent_one_phase_3'; +INSERT INTO t1 (x, y) VALUES (301, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (302, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (303, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (304, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_one_phase_3'; +connection promote_four; +XA START 'bolt_concurrent_prepare_1'; +INSERT INTO t1 (x, y) VALUES (401, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (402, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (403, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (404, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_prepare_1'; +connection promote_five; +XA START 'bolt_concurrent_prepare_2'; +INSERT INTO t1 (x, y) VALUES (501, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (502, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (503, REPEAT('abcdefghijklmnop', 250000)); +INSERT INTO t1 (x, y) VALUES (504, REPEAT('abcdefghijklmnop', 250000)); +XA END 'bolt_concurrent_prepare_2'; +connection default; +--let $binlog_file_before_small = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Five waiting XA branches leave the active binlog available for a small transaction. +INSERT INTO t1 (x, y) VALUES (1, 'small transaction'); +--let $binlog_file_after_small = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--exec ls $BOLT_TEMP_DIR/bolt_* | wc -l | grep -E '^[[:space:]]*5[[:space:]]*$' > /dev/null +--let $assert_text = The small transaction stays in the existing active binary log with XA spills pending. +--let $assert_cond = "$binlog_file_after_small" = "$binlog_file_before_small" +--source include/assert.inc +connection promote_one; +XA ROLLBACK 'bolt_concurrent_one_phase_1'; +connection promote_two; +XA ROLLBACK 'bolt_concurrent_one_phase_2'; +connection promote_three; +XA COMMIT 'bolt_concurrent_one_phase_3' ONE PHASE; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +connection promote_four; +XA PREPARE 'bolt_concurrent_prepare_1'; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +XA ROLLBACK 'bolt_concurrent_prepare_1'; +connection promote_five; +XA PREPARE 'bolt_concurrent_prepare_2'; +--let $bolt_header_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source ../inc/validate_bolt_header.inc +XA COMMIT 'bolt_concurrent_prepare_2'; +connection default; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = One XA one-phase commit and two XA PREPARE paths produce three promotions. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 3 +--source include/assert.inc +--let $assert_text = The XA commit decisions create exactly three new binary log files. +--let $assert_cond = "$binlog_file_after" = "binlog.000004" +--source include/assert.inc +--let $assert_text = Only the committed XA branches and small transaction persist. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 9 +--source include/assert.inc +--let $assert_text = The committed XA transaction ranges are preserved exactly. +--let $assert_cond = [SELECT SUM(x) AS id_sum FROM t1, id_sum, 1] = 3221 +--source include/assert.inc +connection promote_one; +disconnect promote_one; +connection promote_two; +disconnect promote_two; +connection promote_three; +disconnect promote_three; +connection promote_four; +disconnect promote_four; +connection promote_five; +disconnect promote_five; +connection default; +--source include/wait_until_count_sessions.inc + +# Cleanup +DROP TABLE t2; +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL max_binlog_size = $save_max_binlog_size +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_compression_fallback.test b/mysql-test/suite/binlog/t/binlog_bolt_compression_fallback.test new file mode 100644 index 000000000000..1cf9b350d572 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_compression_fallback.test @@ -0,0 +1,143 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify the compression fallback: a transaction whose cache carries a +# compressed payload must commit through the standard binary log path, because +# promotion publishes the cache as a binary log file and a compressed cache is +# not one. +# +# === Requirements === +# R1. A promotable large transaction promotes when its cache is uncompressed. +# R2. A compressed cache prevents promotion: the optimization counter does not +# move and the missed counter increments exactly once. +# R3. The fallback does not rotate; the transaction stays in the active file. +# R4. The fallback commits every row. +# R5. The fallback reports the compression reason in the error log. +# +# === Implementation === +# The condition cannot be reached by configuration alone. Enabling +# binlog_transaction_compression makes the cache record +# BINLOG_CHECKSUM_ALG_OFF, which lets Binlog_cache_compressor proceed and +# rewrite the cache as a single compressed payload event during finalize(). +# That happens before the promotion decision, so the cache is then usually +# below the threshold and the transaction is never a candidate at all, which +# exits silently rather than through the compression check. Reaching the check +# would require a payload that compresses just enough to set a compression type +# yet stays above 10 MiB afterwards. +# +# The debug symbol force_large_trx_compression_fallback therefore stands in for +# the condition: it records the compression reason and returns no cache, at the +# same point the real check would. This exercises the reason's message, its +# counter, and the routing back to the standard commit path, but not the +# predicate that selects it. +# +# Case 1 runs the identical transaction without the symbol, so the opposite +# outcome in case 2 is attributable to the symbol and not to the workload. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc +--source include/have_debug.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_binlog_format = query_get_value(SELECT @@SESSION.binlog_format, @@SESSION.binlog_format, 1) +--let $save_compression = query_get_value(SELECT @@SESSION.binlog_transaction_compression, @@SESSION.binlog_transaction_compression, 1) + +# Setup +--disable_query_log +--disable_warnings +CALL mtr.add_suppression('Could not optimize large transaction execution in the binary log because'); +DROP TABLE IF EXISTS t1; +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +############################################################################ +# Case 1: an uncompressed cache promotes. Control for case 2: same session +# settings, same statements, same sizes. +############################################################################ +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # An uncompressed cache promotes a binary log. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 5000000)); +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = An uncompressed cache promotes a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = An uncompressed cache does not increment the missed counter. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = The promoted commit opens its promoted binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = The promoted transaction commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc + +############################################################################ +# Case 2: the same transaction with the compression reason forced. +############################################################################ +# Case 1 promoted its spilled file, and a promoted file becomes the active +# binary log, so rotate to a fresh file first: the assertion below requires the +# active file to be unchanged by this case, which is only meaningful when this +# case is the only writer. +--disable_query_log +TRUNCATE t1; +FLUSH BINARY LOGS; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A compressed cache uses the standard commit path. +SET @old_debug = @@SESSION.debug; +SET SESSION debug = "+d,force_large_trx_compression_fallback"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 5000000)); +COMMIT; +SET SESSION debug = @old_debug; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A compressed cache does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A compressed cache increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = A compression fallback stays in the active binary log. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A compression fallback commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = A compression fallback preserves the full payload. +--let $assert_cond = [SELECT SUM(OCTET_LENGTH(data)) AS payload FROM t1, payload, 1] = 15000000 +--source include/assert.inc +# Confirms the reason reported is the compression one, which also pins the +# message to its enum slot in kFallbackDetails. +--exec grep -q "binary log transaction compression is enabled" $MYSQLTEST_VARDIR/log/mysqld.1.err + +# Cleanup +DROP TABLE t1; +--disable_query_log +--disable_warnings +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET SESSION binlog_format = $save_binlog_format +--eval SET SESSION binlog_transaction_compression = $save_compression +--enable_warnings +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_encryption.test b/mysql-test/suite/binlog/t/binlog_bolt_encryption.test new file mode 100644 index 000000000000..d5f3d1576c3f --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_encryption.test @@ -0,0 +1,291 @@ +# === Purpose === +# Verify that promotion and the binary-log encryption policy can never +# disagree. Promotion turns a spilled temporary file into a binary log file, +# and nothing re-encrypts or decrypts a file that already holds bytes, so a +# transaction may only be promoted when its spilled file already matches the +# policy in force at the moment the file joins the binary log index. +# +# === Requirements === +# R1. A spilled plaintext file is promoted while binlog_encryption is OFF. +# R2. A spilled plaintext file is not promoted when binlog_encryption is +# turned ON after the file spilled, and the fallback names encryption as +# the reason. +# R3. Such a fallback still commits every row, and the events are readable +# from the encrypted binary log the fallback wrote them to. +# R4. Turning binlog_encryption back OFF restores promotion for later +# transactions. +# R5. A file that spilled while binlog_encryption was ON is not promoted after +# the policy is turned OFF again. +# +# R2 is the case this test was written for. The check used to consult only the +# spilled file's own encryption state, which is decided when the cache opens +# its temporary file. A transaction that spilled in plaintext and then saw +# encryption enabled mid-flight still reported "not encrypted", so its +# plaintext file was eligible for promotion into a binary log that the policy +# required to be encrypted. R5 is the opposite direction, which the file-state +# check already covered, and is included so both directions are pinned. +# +# === Implementation === +# Each case creates a transaction above the 10 MiB threshold so its cache +# spills, changes binlog_encryption from a second session at the point of +# interest, and then commits. Promotion is observed through +# Binlog_large_transaction_optimization_count and the missed counter, and +# through whether the active binary log file changed: a promoted transaction +# opens a new file, a fallback does not. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc +--source include/have_component_keyring_file.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_binlog_format = query_get_value(SELECT @@SESSION.binlog_format, @@SESSION.binlog_format, 1) +--let $save_compression = query_get_value(SELECT @@SESSION.binlog_transaction_compression, @@SESSION.binlog_transaction_compression, 1) + +CALL mtr.add_suppression('Could not optimize large transaction execution in the binary log because'); + +--source ../mysql-test/suite/component_keyring_file/inc/setup_component.inc + +# The keyring component must be loaded for binlog_encryption to be settable at +# runtime, but the server starts with encryption OFF so a cache can spill in +# plaintext first. +--echo # Restart the server with the keyring available and encryption OFF. +--let $restart_parameters = restart:--binlog_encryption=OFF $PLUGIN_DIR_OPT +--source include/restart_mysqld_no_echo.inc + +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` + +--disable_query_log +--disable_warnings +DROP TABLE IF EXISTS t1; +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +connect (encryption_switch, localhost, root,,); +connection default; + +############################################################################ +# Case 1: Plaintext spilled file, encryption OFF throughout. +# Expected behavior: promote. +############################################################################ +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A plaintext spilled file is promoted while encryption is OFF. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A plaintext file with encryption OFF promotes a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = A plaintext file with encryption OFF does not increment the missed counter. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = The promoted commit opens a new active binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = The promoted transaction commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +# Positive control for the encryption checks in the later cases: with the policy +# OFF the promoted file must carry no encryption key ID. +--let $rpl_log_file = $MYSQLD_DATADIR$binlog_file_after +--source include/rpl/get_log_encryption_key_id.inc +--let $assert_text = The promoted binary log is plaintext while encryption is OFF. +--let $assert_cond = "$rpl_encryption_key_id" = "None" +--source include/assert.inc + +############################################################################ +# Case 2: Plaintext spilled file, encryption turned ON mid-transaction. +# Expected behavior: fall back. +# +# The cache spilled while the policy said OFF, so the file is plaintext and +# stays plaintext; enabling encryption part way through cannot change that. +# Promoting it would publish a plaintext binary log under a policy requiring +# encryption. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_at_begin = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Enabling encryption mid-transaction prevents promoting a plaintext file. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('d', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('e', 4000000)); +# The cache has spilled in plaintext by this point. Enabling encryption now +# rotates the binary log, so the file the fallback writes into is encrypted +# while this session's spilled file is not. +connection encryption_switch; +SET GLOBAL binlog_encryption = ON; +connection default; +INSERT INTO t1 VALUES (3, REPEAT('f', 4000000)); +--let $binlog_file_before_commit = query_get_value(SHOW BINARY LOG STATUS, File, 1) +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after_commit = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = Encryption enabled mid-transaction does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = Encryption enabled mid-transaction increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = The mid-transaction encryption fallback stays in the active binary log. +--let $assert_cond = "$binlog_file_after_commit" = "$binlog_file_before_commit" +--source include/assert.inc +--let $assert_text = The mid-transaction encryption fallback commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = The mid-transaction encryption fallback preserves the full payload. +--let $assert_cond = [SELECT SUM(OCTET_LENGTH(data)) AS payload FROM t1, payload, 1] = 12000000 +--source include/assert.inc +--exec grep -q "binary log encryption was enabled while the transaction was running" $MYSQLTEST_VARDIR/log/mysqld.1.err + +# Confirm the policy really took effect rather than the fallback happening for +# some unrelated reason: enabling encryption rotates the binary log, and the +# file the fallback then wrote into must carry an encryption key ID. +--let $assert_text = Enabling encryption mid-transaction rotates the binary log. +--let $assert_cond = "$binlog_file_before_commit" != "$binlog_file_at_begin" +--source include/assert.inc +--let $rpl_log_file = $MYSQLD_DATADIR$binlog_file_after_commit +--source include/rpl/get_log_encryption_key_id.inc +--let $assert_text = The mid-transaction encryption fallback wrote into an encrypted binary log. +--let $assert_cond = "$rpl_encryption_key_id" != "None" +--source include/assert.inc + +############################################################################ +# Case 3: Encryption turned back OFF. +# Expected behavior: promote again. +# +# Turning the policy off rotates to a plaintext binary log, and a cache that +# opens afterwards spills in plaintext, so promotion is available again. This +# guards against the fix latching the fallback on permanently. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +connection encryption_switch; +SET GLOBAL binlog_encryption = OFF; +connection default; +# Use a fresh session so its transaction cache opens after the policy change. +connect (plaintext_again, localhost, root,,); +connection plaintext_again; +--disable_query_log +--disable_warnings +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Turning encryption back OFF restores promotion. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('g', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('h', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('i', 4000000)); +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = Promotion resumes once encryption is turned back OFF. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = The resumed promotion does not increment the missed counter. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = The resumed promotion opens a new active binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = The resumed promotion commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +connection default; +disconnect plaintext_again; +connection default; + +############################################################################ +# Case 4: Encrypted spilled file, encryption turned OFF mid-transaction. +# Expected behavior: fall back. +# +# The mirror of case 2. The cache spilled while the policy said ON, so the file +# is encrypted and stays encrypted. Promoting it after the policy went OFF +# would publish an encrypted binary log file under a policy requiring +# plaintext. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +connection encryption_switch; +SET GLOBAL binlog_encryption = ON; +connection default; +# Fresh session again, so this cache opens while the policy says ON and its +# spilled file is therefore encrypted. +connect (encrypted_cache, localhost, root,,); +connection encrypted_cache; +--disable_query_log +--disable_warnings +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--echo # An encrypted spilled file is not promoted after encryption is turned OFF. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('j', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('k', 4000000)); +connection encryption_switch; +SET GLOBAL binlog_encryption = OFF; +connection encrypted_cache; +INSERT INTO t1 VALUES (3, REPEAT('l', 4000000)); +--let $binlog_file_before_commit = query_get_value(SHOW BINARY LOG STATUS, File, 1) +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after_commit = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = An encrypted file is not promoted after encryption is turned OFF. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = The encrypted-file fallback increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = The encrypted-file fallback stays in the active binary log. +--let $assert_cond = "$binlog_file_after_commit" = "$binlog_file_before_commit" +--source include/assert.inc +--let $assert_text = The encrypted-file fallback commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +connection default; +disconnect encrypted_cache; +connection default; + +# Cleanup +connection encryption_switch; +SET GLOBAL binlog_encryption = OFF; +connection default; +disconnect encryption_switch; +connection default; +DROP TABLE t1; +--disable_query_log +--disable_warnings +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET SESSION binlog_format = $save_binlog_format +--eval SET SESSION binlog_transaction_compression = $save_compression +--enable_warnings +--enable_query_log +--source ../mysql-test/suite/component_keyring_file/inc/teardown_component.inc diff --git a/mysql-test/suite/binlog/t/binlog_bolt_error_action-master.opt b/mysql-test/suite/binlog/t/binlog_bolt_error_action-master.opt new file mode 100644 index 000000000000..0f044b09f549 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_error_action-master.opt @@ -0,0 +1 @@ +--skip-core-file --log-error=$MYSQLTEST_VARDIR/tmp/binlog_bolt_error_action.err --binlog-large-transaction-optimization-enabled=ON --binlog-large-transaction-optimization-threshold=10M --max-binlog-size=1G --sync-binlog=1 --innodb-flush-log-at-trx-commit=1 --binlog-transaction-compression=OFF diff --git a/mysql-test/suite/binlog/t/binlog_bolt_error_action.test b/mysql-test/suite/binlog/t/binlog_bolt_error_action.test new file mode 100644 index 000000000000..fa015491e497 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_error_action.test @@ -0,0 +1,153 @@ +# +# === Purpose === +# Verify that a failure to make the spilled file durable in the BOLT commit path +# applies binlog_error_action, as the standard group-commit path does. +# +# BOLT publishes one transaction on its own, so there is no group leader to +# apply the action on its behalf, and finish_commit() by contract reports only +# CE_COMMIT_ERROR. Without the action being applied here a flush error would be +# reported to the client as a successful commit. +# +# === Requirements === +# R1. With binlog_error_action = ABORT_SERVER the failure aborts the server, the +# client is told binary logging is not possible, and after the restart the +# transaction is in neither the engine nor the binary log. +# R2. With binlog_error_action = IGNORE_ERROR the failure is reported in the +# error log and the transaction still commits in the engine, while nothing +# of it reaches the binary log. +# +# === Implementation === +# Force the failure with the fail_bolt_spill_file_sync debug point on a +# transaction over the 10M threshold, once per binlog_error_action value, and +# check the client error, the error log, the rows in the engine and the binary +# log contents. Each case resets the binary log first so the file at index 1 is +# the active one. +# +# The error log is redirected to its own file because ABORT_SERVER goes through +# the crash handler, whose backtrace MTR would otherwise flag. +# +--source include/not_crashrep.inc +--source include/not_valgrind.inc +--source include/have_debug.inc +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $save_error_action = query_get_value(SELECT @@GLOBAL.binlog_error_action, @@GLOBAL.binlog_error_action, 1) +--let $save_compression = query_get_value(SELECT @@SESSION.binlog_transaction_compression, @@SESSION.binlog_transaction_compression, 1) +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` + +CALL mtr.add_suppression("An error occurred during flush stage of the commit"); +CALL mtr.add_suppression("Binary logging not possible"); +CALL mtr.add_suppression("Hence turning logging off for the whole duration"); + +--disable_query_log +--disable_warnings +DROP TABLE IF EXISTS t1; +# The payload repeats a single byte, so compression would shrink the cache far +# below the threshold and the transaction would never spill. +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +############################################################################ +# Case 1: binlog_error_action = ABORT_SERVER. +# Expected behavior: the server aborts, and the transaction reaches neither +# the engine nor the binary log. +# +# The spilled file is never promoted, so the transaction is in no binary log +# file and recovery has no reason to roll it forward. +############################################################################ +--echo # A spill-sync failure with ABORT_SERVER aborts the server. +SET GLOBAL binlog_error_action = ABORT_SERVER; +--disable_query_log +RESET BINARY LOGS AND GTIDS; +--enable_query_log +--let $log_size_before = query_get_value("SHOW BINARY LOGS", File_size, 1) +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +SET SESSION DEBUG="+d,fail_bolt_spill_file_sync"; +--error ER_BINLOG_LOGGING_IMPOSSIBLE +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc +--exec grep -q "An error occurred during flush stage of the commit. 'binlog_error_action' is set to 'ABORT_SERVER'." $MYSQLTEST_VARDIR/tmp/binlog_bolt_error_action.err +--let $assert_text = An aborted spill-sync failure commits no rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +--let $log_size_after = query_get_value("SHOW BINARY LOGS", File_size, 1) +--let $assert_text = An aborted spill-sync failure writes nothing to the binary log. +--let $assert_cond = $log_size_after = $log_size_before +--source include/assert.inc + +############################################################################ +# Case 2: binlog_error_action = IGNORE_ERROR. +# Expected behavior: binary logging is turned off for the rest of the server +# process and the transaction commits in the engine. +# +# That binlog/engine divergence is what IGNORE_ERROR means, and is what the +# standard path does for the same error. +############################################################################ +--echo # A spill-sync failure with IGNORE_ERROR keeps the commit and stops logging. +--disable_query_log +TRUNCATE t1; +RESET BINARY LOGS AND GTIDS; +--enable_query_log +SET GLOBAL binlog_error_action = IGNORE_ERROR; +# Captured before the failure: once logging is off, SHOW BINARY LOGS fails with +# ER_NO_BINARY_LOGGING. +--let $bolt_log_file = query_get_value("SHOW BINARY LOGS", Log_name, 1) +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('d', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('f', 4000000)); +SET SESSION DEBUG="+d,fail_bolt_spill_file_sync"; +COMMIT; +SET SESSION DEBUG="-d,fail_bolt_spill_file_sync"; +--enable_result_log +--enable_query_log +--let $assert_text = An ignored spill-sync failure still commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--exec grep -q "An error occurred during flush stage of the commit. 'binlog_error_action' is set to 'IGNORE_ERROR'." $MYSQLTEST_VARDIR/tmp/binlog_bolt_error_action.err +# Binary logging is off at this point, so restart before reading the log back. +--let $do_not_echo_parameters = 1 +--source include/restart_mysqld.inc +# +# Assert on the decoded contents, not the file size: turning logging off closes +# the log with LOG_CLOSE_STOP_EVENT, and only the spilled file's sync was made +# to fail, so the binary log is healthy and that Stop event grows it by 23 +# bytes. +# +--let $bolt_dump = $MYSQLTEST_VARDIR/tmp/bolt_error_action_dump.txt +--exec $MYSQL_BINLOG --force-if-open $MYSQLD_DATADIR/$bolt_log_file > $bolt_dump +--let $assert_file = $bolt_dump +--let $assert_text = An ignored spill-sync failure writes no row events to the binary log. +--let $assert_select = Write_rows +--let $assert_count = 0 +--source include/assert_grep.inc +--let $assert_text = An ignored spill-sync failure writes no transaction to the binary log. +--let $assert_select = Xid +--let $assert_count = 0 +--source include/assert_grep.inc +--remove_file $bolt_dump + +# Cleanup +DROP TABLE t1; +--disable_query_log +--disable_warnings +--eval SET GLOBAL binlog_error_action = $save_error_action +--eval SET SESSION binlog_transaction_compression = $save_compression +RESET BINARY LOGS AND GTIDS; +--enable_warnings +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_fallback.test b/mysql-test/suite/binlog/t/binlog_bolt_fallback.test new file mode 100644 index 000000000000..3a3e78e6d74d --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_fallback.test @@ -0,0 +1,325 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT uses the ordinary binary-log commit path when a spilled, +# above-threshold transaction cannot safely be promoted, while retaining +# promotion eligibility with a dynamically sized reserved header. +# +# === Requirements === +# R1. Large STATEMENT and MIXED transactions do not promote. +# R2. A checksum change during a transaction prevents promotion, and restoring +# it before commit preserves eligibility. +# R3. A Previous_gtids event larger than 64 KiB expands the reservation and +# permits promotion. +# R4. A row statement touching InnoDB and MyISAM falls back for its +# statement-cache transaction while still promoting its InnoDB cache. +# +# === Implementation === +# Each case spills a transaction above the 10 MiB threshold and checks the two +# BOLT status counters, the active binary-log filename, and the reason reported +# in the error log. +# +# Every condition here is reachable by configuration, so this file also runs on +# release builds. The reasons that need a debug symbol are in +# binlog_bolt_fallback_debug; encryption is in binlog_bolt_encryption. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_checksum = query_get_value(SELECT @@GLOBAL.binlog_checksum, @@GLOBAL.binlog_checksum, 1) +--let $save_binlog_format = query_get_value(SELECT @@SESSION.binlog_format, @@SESSION.binlog_format, 1) +--let $save_compression = query_get_value(SELECT @@SESSION.binlog_transaction_compression, @@SESSION.binlog_transaction_compression, 1) + +CALL mtr.add_suppression('Could not optimize large transaction execution in the binary log because'); + +# Setup +--disable_query_log +--disable_warnings +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE t2 (id INT PRIMARY KEY, marker INT) ENGINE=MyISAM; + +############################################################################ +# Case 1: Large STATEMENT transaction. +############################################################################ +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A large STATEMENT transaction uses the standard commit path. +--disable_warnings +SET SESSION binlog_format = STATEMENT; +--enable_warnings +SET @bolt_statement_payload = REPEAT('s', 12000000); +BEGIN; +INSERT INTO t1 VALUES (1, @bolt_statement_payload); +COMMIT; +SET @bolt_statement_payload = NULL; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A large STATEMENT transaction does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A large STATEMENT transaction increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = A large STATEMENT transaction stays in the active binary log. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A large STATEMENT fallback commits its row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1 WHERE id = 1, count, 1] = 1 +--source include/assert.inc +--exec grep -q "transaction contains non-ROW events" $MYSQLTEST_VARDIR/log/mysqld.1.err + +############################################################################ +# Case 2: Large MIXED transaction. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A large MIXED transaction with a statement event uses the standard path. +--disable_warnings +SET SESSION binlog_format = MIXED; +--enable_warnings +SET @bolt_mixed_payload = REPEAT('m', 12000000); +BEGIN; +INSERT INTO t1 VALUES (1, @bolt_mixed_payload); +COMMIT; +SET @bolt_mixed_payload = NULL; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A large MIXED transaction does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A large MIXED transaction increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = A large MIXED transaction stays in the active binary log. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A large MIXED fallback commits its row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1 WHERE id = 1, count, 1] = 1 +--source include/assert.inc + +############################################################################ +# Case 3: Checksum change after the transaction starts. +############################################################################ +--disable_query_log +--disable_warnings +TRUNCATE t1; +SET SESSION binlog_format = ROW; +--enable_warnings +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +connect (checksum_change, localhost, root,,); +connection default; +--echo # A checksum change during a large transaction uses the standard path. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('c', 5000000)); +connection checksum_change; +SET GLOBAL binlog_checksum = IF(@@GLOBAL.binlog_checksum = 'CRC32', 'NONE', 'CRC32'); +connection default; +INSERT INTO t1 VALUES (2, REPEAT('d', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('e', 5000000)); +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text = A checksum change does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A checksum change increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = A checksum-change fallback commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +# BINLOG_CHECKSUM_ALG_OFF while binlog_checksum is CRC32. +--exec grep -q "checksum algorithm does not match binlog_checksum" $MYSQLTEST_VARDIR/log/mysqld.1.err +connection checksum_change; +disconnect checksum_change; +connection default; +--disable_query_log +--eval SET GLOBAL binlog_checksum = $save_checksum +--enable_query_log + +############################################################################ +# Case 4: Checksum changes back before commit. +############################################################################ +--disable_query_log +--disable_warnings +TRUNCATE t1; +SET SESSION binlog_format = ROW; +--enable_warnings +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +connect (checksum_restored, localhost, root,,); +connection default; +--echo # Restoring the original checksum before commit preserves promotion. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('r', 5000000)); +connection checksum_restored; +SET GLOBAL binlog_checksum = IF(@@GLOBAL.binlog_checksum = 'CRC32', 'NONE', 'CRC32'); +--eval SET GLOBAL binlog_checksum = $save_checksum +connection default; +INSERT INTO t1 VALUES (2, REPEAT('s', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('t', 5000000)); +--let $binlog_file_before_commit = query_get_value(SHOW BINARY LOG STATUS, File, 1) +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after_commit = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = Restoring the checksum before commit promotes the transaction. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = Restoring the checksum before commit does not increment the missed counter. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = The restored-checksum commit opens its promoted binary log. +--let $assert_cond = "$binlog_file_after_commit" != "$binlog_file_before_commit" +--source include/assert.inc +--let $assert_text = The restored-checksum promoted transaction commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +connection checksum_restored; +disconnect checksum_restored; +connection default; + +############################################################################ +# Case 5: Large Previous_gtids receives a dynamic reservation. +############################################################################ +--disable_query_log +--disable_warnings +TRUNCATE t1; +RESET BINARY LOGS AND GTIDS; +SET SESSION binlog_format = ROW; +--enable_warnings +--let $gtid_index = 1 +while ($gtid_index <= 1700) +{ + --let $gtid_suffix = `SELECT LPAD(HEX($gtid_index), 12, '0')` + --eval SET @@SESSION.GTID_NEXT = 'aaaaaaaa-aaaa-aaaa-aaaa-$gtid_suffix:1' + BEGIN; + COMMIT; + SET @@SESSION.GTID_NEXT = AUTOMATIC; + --inc $gtid_index +} +--enable_query_log +--let $executed_gtid_length = query_get_value(SELECT CHAR_LENGTH(@@GLOBAL.gtid_executed) AS length, length, 1) +--let $assert_text = The executed GTID set exceeds BOLT's reserved header. +--let $assert_cond = $executed_gtid_length > 64000 +--source include/assert.inc +--disable_query_log +# Publish the serialized Previous_gtids size before opening the BOLT cache. +FLUSH BINARY LOGS; +connect (dynamic_header, localhost, root,,); +connection dynamic_header; +--disable_warnings +SET SESSION binlog_format = ROW; +--enable_warnings +SET SESSION binlog_transaction_compression = OFF; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A dynamically reserved Previous_gtids header preserves promotion. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('g', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('h', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('i', 5000000)); +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A dynamically reserved Previous_gtids header promotes a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = A dynamically reserved Previous_gtids header does not increment the missed counter. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = The dynamically reserved-header commit opens its promoted binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A dynamically reserved-header commit commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +connection default; +disconnect dynamic_header; +connection default; + +############################################################################ +# Case 6: A row statement touches InnoDB and MyISAM. +############################################################################ +# Permanent transactional/nontransactional changes violate GTID consistency, +# so restart this isolated test server with GTID enforcement disabled. +--let $restart_parameters = restart:--gtid-mode=OFF --enforce-gtid-consistency=OFF +--source include/restart_mysqld_no_echo.inc +--disable_query_log +--disable_warnings +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_transaction_compression = OFF; +RESET BINARY LOGS AND GTIDS; +TRUNCATE t1; +TRUNCATE t2; +SET SESSION binlog_format = ROW; +--enable_query_log +INSERT INTO t1 VALUES (1, REPEAT('n', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('o', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('p', 5000000)); +INSERT INTO t2 VALUES (1, 0), (2, 0), (3, 0); +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A mixed InnoDB/MyISAM row statement falls back for its statement cache. +BEGIN; +UPDATE t1 JOIN t2 USING (id) SET t1.data = CONCAT(t1.data, 'x'), t2.marker = 1; +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = An InnoDB/MyISAM statement promotes its qualifying InnoDB cache. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = An InnoDB/MyISAM statement increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = An InnoDB/MyISAM statement rotates to the promoted InnoDB binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = An InnoDB/MyISAM fallback commits every InnoDB row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = An InnoDB/MyISAM fallback updates every MyISAM row. +--let $assert_cond = [SELECT SUM(marker) AS marker_sum FROM t2, marker_sum, 1] = 3 +--source include/assert.inc +--exec grep -q "statement cache is nonempty" $MYSQLTEST_VARDIR/log/mysqld.1.err + + +# Cleanup +DROP TABLE t2; +DROP TABLE t1; +--disable_query_log +--disable_warnings +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL binlog_checksum = $save_checksum +--eval SET SESSION binlog_format = $save_binlog_format +--eval SET SESSION binlog_transaction_compression = $save_compression +--enable_warnings +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_fallback_debug.test b/mysql-test/suite/binlog/t/binlog_bolt_fallback_debug.test new file mode 100644 index 000000000000..f1f6f8062aa3 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_fallback_debug.test @@ -0,0 +1,208 @@ +# +# === Purpose === +# Verify the BOLT fallback reasons that need a debug symbol to reach. The +# reasons reachable by configuration are in binlog_bolt_fallback, which stays +# release-capable by not sourcing have_debug.inc. +# +# === Requirements === +# R1. A pending logging incident prevents promotion, so the standard path can +# write the incident and force the rotation that goes with it. +# R2. A binary log closed while the transaction was running prevents promotion, +# because there is no longer a file sequence to promote into. +# +# === Implementation === +# Spill a transaction above the 10 MiB threshold, force the condition with a +# debug symbol, and check the two BOLT status counters, the reason reported in +# the error log, and the committed rows. +# +# Neither condition is configurable: the incident producers that survive to +# commit need a cache write failure on a statement that cannot roll back, and +# closing the log mid-transaction needs a binary log write to fail while +# binlog_error_action is IGNORE_ERROR. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc +--source include/have_debug.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_binlog_format = query_get_value(SELECT @@SESSION.binlog_format, @@SESSION.binlog_format, 1) +--let $save_compression = query_get_value(SELECT @@SESSION.binlog_transaction_compression, @@SESSION.binlog_transaction_compression, 1) + +CALL mtr.add_suppression('Could not optimize large transaction execution in the binary log because'); +CALL mtr.add_suppression("An incident event has been written to the binary log"); +CALL mtr.add_suppression("An error occurred during flush stage of the commit"); +CALL mtr.add_suppression("Binary logging not possible"); +CALL mtr.add_suppression("Hence turning logging off for the whole duration"); + +# Setup +--disable_query_log +--disable_warnings +DROP TABLE IF EXISTS t1; +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_format = ROW; +# REPEAT() payloads compress to almost nothing, so a compressed cache would +# never reach the threshold and the transaction would never be promoted. +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +############################################################################ +# Case 1: A pending logging incident. +# +# The group-commit flush is what writes an incident and force-rotates for it, +# and promotion bypasses that flush, so a transaction carrying an incident must +# fall back or the incident would never reach the binary log. +# +# Note the active binary log is expected to CHANGE here, unlike other fallback +# cases: handling the incident requests a force rotation. A fresh session is +# used because binlog_inject_incident fires once per session. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +connect (incident_session, localhost, root,,); +connection incident_session; +--disable_query_log +--disable_warnings +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A transaction with a logging incident uses the standard commit path. +SET SESSION debug = "+d,binlog_inject_incident"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('w', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('x', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('y', 5000000)); +COMMIT; +SET SESSION debug = "-d,binlog_inject_incident"; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = An incident does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = An incident increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +# Not the usual "stays in the active file" assertion: handling the incident +# requests a force rotation, so the standard path leaves a new active file. +--let $assert_text = An incident fallback force-rotates to a new active binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = An incident fallback commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = An incident fallback preserves the full payload. +--let $assert_cond = [SELECT SUM(OCTET_LENGTH(data)) AS payload FROM t1, payload, 1] = 15000000 +--source include/assert.inc +--exec grep -q "the transaction has a logging incident to report" $MYSQLTEST_VARDIR/log/mysqld.1.err +connection default; +disconnect incident_session; +connection default; + +############################################################################ +# Case 2: The binary log was closed while the transaction was running. +# +# binlog_error_action=IGNORE_ERROR turns binary logging off for the rest of the +# server process, which closes the log and leaves MYSQL_BIN_LOG::name null. A +# transaction that had already spilled is still a promotion candidate -- the +# captured knob, the spilled file and the byte position are all unchanged -- so +# promote_spilled_file() has to decline instead of trying to generate the next +# file name from a name that no longer exists. +# +# The victim is a bystander: the session whose transaction closes the log is a +# different one, and it is the innocent session that would otherwise crash. +# +# This case is last, and is followed by a restart, because logging stays off +# afterwards and the cleanup below needs it back. +############################################################################ +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $save_error_action = query_get_value(SELECT @@GLOBAL.binlog_error_action, @@GLOBAL.binlog_error_action, 1) +SET GLOBAL binlog_error_action = IGNORE_ERROR; + +--echo # A session spills a large transaction and holds it open. +connect (log_closed_session, localhost, root,,); +connection log_closed_session; +--disable_query_log +--disable_warnings +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 5000000)); +--enable_result_log +--enable_query_log + +--echo # Another session's spill-sync failure turns logging off and closes the log. +connection default; +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (11, REPEAT('d', 5000000)); +INSERT INTO t1 VALUES (12, REPEAT('e', 5000000)); +INSERT INTO t1 VALUES (13, REPEAT('f', 5000000)); +SET SESSION debug = "+d,fail_bolt_spill_file_sync"; +COMMIT; +SET SESSION debug = "-d,fail_bolt_spill_file_sync"; +--enable_result_log +--enable_query_log + +# Captured after the close, so only the bystander's commit moves them. +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) + +--echo # The held transaction commits through the standard path. +connection log_closed_session; +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text = A closed binary log does not promote. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A closed binary log increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = A closed-log fallback still commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 6 +--source include/assert.inc +--let $assert_text = A closed-log fallback preserves the full payload. +--let $assert_cond = [SELECT SUM(OCTET_LENGTH(data)) AS payload FROM t1, payload, 1] = 30000000 +--source include/assert.inc +--exec grep -q "the binary log was closed while the transaction was running" $MYSQLTEST_VARDIR/log/mysqld.1.err + +connection default; +disconnect log_closed_session; +connection default; +# Logging is off from here, so restore it before the cleanup below. +--disable_query_log +--eval SET GLOBAL binlog_error_action = $save_error_action +--enable_query_log +--let $do_not_echo_parameters = 1 +--source include/restart_mysqld.inc + +# Cleanup +DROP TABLE t1; +--disable_query_log +--disable_warnings +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET SESSION binlog_format = $save_binlog_format +--eval SET SESSION binlog_transaction_compression = $save_compression +RESET BINARY LOGS AND GTIDS; +--enable_warnings +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_optimized_recovery-master.opt b/mysql-test/suite/binlog/t/binlog_bolt_optimized_recovery-master.opt new file mode 100644 index 000000000000..dc3126be991c --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_optimized_recovery-master.opt @@ -0,0 +1 @@ +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/binlog/t/binlog_bolt_optimized_recovery.test b/mysql-test/suite/binlog/t/binlog_bolt_optimized_recovery.test new file mode 100644 index 000000000000..411f71f9af4c --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_optimized_recovery.test @@ -0,0 +1,115 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT recovery logs the optimized seek and rejects a truncated +# promoted binary log with an invalid Large_transaction_header. +# +# === Requirements === +# R1. Recovery logs the optimized large-transaction seek after promotion. +# R2. A promoted transaction survives normal restart recovery. +# R3. A corrupt LTH event causes the server to crash during recovery. +# +# === Implementation === +# 1. Initialize BOLT and a clean test table. +# 2. Commit a qualifying transaction and restart to inspect recovery logging. +# 3. Stop the server, truncate the promoted file at a fixed 1 MiB offset, +# and verify standalone recovery fails with the invalid-LTH diagnostic. +# +--source include/not_windows.inc +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_max_binlog_size = query_get_value(SELECT @@GLOBAL.max_binlog_size, @@GLOBAL.max_binlog_size, 1) +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_BACKUP = $MYSQLTEST_VARDIR/tmp/bolt_lth_backup +--let $BOLT_FAILURE_LOG = $MYSQLTEST_VARDIR/tmp/bolt_invalid_lth.err + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +--enable_query_log +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +############################################################################ +# Case 1: Recover an intact promoted transaction. +# Expected behavior: commit. +############################################################################ +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Commit a qualifying transaction for optimized recovery. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; + +--let $promoted_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = The transaction was promoted into a new binary log file. +--let $assert_cond = "$promoted_file" != "$binlog_file_before" +--source include/assert.inc + +--let $restart_parameters = restart: +--let $restart_parameters = $restart_parameters --binlog-large-transaction-optimization-enabled=ON +--let $restart_parameters = $restart_parameters --binlog-large-transaction-optimization-threshold=10M +--let $restart_parameters = $restart_parameters --max-binlog-size=1G +# REPEAT() payloads compress to almost nothing, so a compressed cache would +# never reach the threshold and the transaction would never be promoted. +--let $restart_parameters = $restart_parameters --binlog-transaction-compression=OFF +--source include/kill_and_restart_mysqld.inc + +--echo # Recovery logs the optimized large-transaction seek. +--let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err +--let SEARCH_PATTERN = Optimized binlog recovery by avoiding a sequential read of a large transaction body and seeking to its terminating event at offset +--source include/search_pattern.inc + +--let $assert_text = The committed promoted transaction survives recovery. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc + +############################################################################ +# Case 2: Try to recover a file with corrupt LTH +# We simulate a corrupt LTH by manually truncating the XID event, +# so the LTH points to a position that doesn't exist. +############################################################################ +--let $binlog_file_before_corruption = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # Commit a second qualifying transaction for corrupt-LTH recovery. +BEGIN; +INSERT INTO t1 VALUES (4, REPEAT('d', 4000000)); +INSERT INTO t1 VALUES (5, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (6, REPEAT('f', 4000000)); +COMMIT; + +--let $corrupt_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = The second transaction was promoted into a new binary log file. +--let $assert_cond = "$corrupt_file" != "$binlog_file_before_corruption" +--source include/assert.inc + +--echo # Truncate the active promoted file and reject the invalid LTH. +--source include/kill_mysqld.inc +--let BOLT_BINLOG = $MYSQLD_DATADIR/$corrupt_file +--exec cp $BOLT_BINLOG $BOLT_BACKUP +--exec truncate -s 1048576 $BOLT_BINLOG +--error 1 +--exec $MYSQLD_CMD --loose-console > $BOLT_FAILURE_LOG 2>&1 +--let SEARCH_FILE = $BOLT_FAILURE_LOG +--let SEARCH_PATTERN = contains an invalid large transaction header +--source include/search_pattern.inc + +--exec cp $BOLT_BACKUP $BOLT_BINLOG +--remove_file $BOLT_BACKUP +--remove_file $BOLT_FAILURE_LOG +--let $restart_parameters = restart +--source include/start_mysqld.inc + +# Cleanup +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL max_binlog_size = $save_max_binlog_size +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_reserved_header_fallback.test b/mysql-test/suite/binlog/t/binlog_bolt_reserved_header_fallback.test new file mode 100644 index 000000000000..3ca1f02c0afc --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_reserved_header_fallback.test @@ -0,0 +1,144 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify the reserved-header fallback: when the region reserved at the front of +# a spilled binary log cache cannot hold the promoted file's header events, the +# transaction must commit through the standard binary log path, leaving no +# trace of the abandoned promotion. +# +# === Requirements === +# R1. A promotable large transaction promotes when the reserved region fits. +# R2. A reserved region too small prevents promotion: the optimization counter +# does not move and the missed counter increments exactly once. +# R3. The fallback does not rotate; the transaction stays in the active file. +# R4. The fallback commits every row. +# R5. The fallback reports the reserved-header reason in the error log. +# R6. The active binary log stays readable, with valid checksums, afterwards. +# +# === Implementation === +# The reserved region is sized when the cache spills, from the serialized +# Previous_gtids size, so no configuration makes it too small while leaving the +# transaction otherwise promotable. The debug symbol +# force_large_trx_reserved_header_fallback clears the "header events fit" +# predicate immediately before the real check. That check precedes any GTID +# assignment and any file I/O, so the forced path is the same no-side-effects +# fallback the real check takes. +# +# Case 1 runs the identical transaction without the symbol, so the opposite +# outcome in case 2 is attributable to the region size and not to the workload. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc +--source include/have_debug.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_binlog_format = query_get_value(SELECT @@SESSION.binlog_format, @@SESSION.binlog_format, 1) +--let $save_compression = query_get_value(SELECT @@SESSION.binlog_transaction_compression, @@SESSION.binlog_transaction_compression, 1) +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` + +# Setup +--disable_query_log +--disable_warnings +CALL mtr.add_suppression('Could not optimize large transaction execution in the binary log because'); +DROP TABLE IF EXISTS t1; +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +--enable_warnings +--enable_query_log +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +############################################################################ +# Case 1: the reserved region fits, so the transaction promotes. This is the +# control for case 2: same session settings, same statements, same sizes. +############################################################################ +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A promotable large transaction promotes when the reserved region fits. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 5000000)); +COMMIT; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A fitting reserved region promotes a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = A fitting reserved region does not increment the missed counter. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = The promoted commit opens its promoted binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = The promoted transaction commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc + +############################################################################ +# Case 2: the same transaction with the reserved region forced too small. +############################################################################ +# Case 1 promoted its spilled file, and a promoted file becomes the active +# binary log, so it would otherwise still be receiving events here. Rotate to +# a fresh file so the R6 check below inspects only this case's transaction. +--disable_query_log +TRUNCATE t1; +FLUSH BINARY LOGS; +--enable_query_log +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--echo # A reserved region too small for the header events uses the standard path. +SET @old_debug = @@SESSION.debug; +SET SESSION debug = "+d,force_large_trx_reserved_header_fallback"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 5000000)); +COMMIT; +SET SESSION debug = @old_debug; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = A reserved region too small does not promote a binary log. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A reserved region too small increments the missed counter once. +--let $assert_cond = $missed_after = $missed_before + 1 +--source include/assert.inc +--let $assert_text = A reserved-header fallback stays in the active binary log. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A reserved-header fallback commits all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--exec grep -q "the reserved header region is too small" $MYSQLTEST_VARDIR/log/mysqld.1.err + +# R6. The abandoned promotion must not have disturbed the active binary log. +# The fallback commits into that same file, so it must still decode with +# valid checksums. --force-if-open is required because the file is active. +--let $bolt_fallback_dump = $MYSQLTEST_VARDIR/tmp/bolt_reserved_header_fallback.txt +--exec $MYSQL_BINLOG --force-if-open --verify-binlog-checksum $MYSQLD_DATADIR/$binlog_file_after > $bolt_fallback_dump +--let $assert_file = $bolt_fallback_dump +--let $assert_text = The fallback transaction is present in the active binary log. +--let $assert_select = Write_rows +--let $assert_count_condition = >= 1 +--source include/assert_grep.inc +--let $assert_count_condition = +--remove_file $bolt_fallback_dump + +# Cleanup +DROP TABLE t1; +--disable_query_log +--disable_warnings +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET SESSION binlog_format = $save_binlog_format +--eval SET SESSION binlog_transaction_compression = $save_compression +--enable_warnings +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_rotate_purge.test b/mysql-test/suite/binlog/t/binlog_bolt_rotate_purge.test new file mode 100644 index 000000000000..5353fa810873 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_rotate_purge.test @@ -0,0 +1,94 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify normal PURGE and FLUSH rotation remain safe while a BOLT spill file +# exists and after that spill is promoted into the main binary-log index. +# +# === Requirements === +# R1. PURGE and FLUSH do not discard or corrupt an active BOLT spill file. +# R2. A spill promotes after a concurrent FLUSH creates a new active binlog. +# R3. A promoted historical file can be purged after a subsequent FLUSH. +# +# === Implementation === +# Create history, keep a transaction spilled, purge older history and rotate, +# then commit the spill. Rotate the resulting promoted active file once more +# and purge it through the normal binary-log lifecycle. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc +--source include/not_binlog_transaction_compression_on.inc + +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_TEMP_DIR = $MYSQLD_DATADIR/#binlog_temp_files +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +RESET BINARY LOGS AND GTIDS; +--enable_query_log +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +# Keep one historical file that PURGE can remove while a spill exists. +FLUSH BINARY LOGS; +--let $binlog_before_spill = query_get_value(SHOW BINARY LOG STATUS, File, 1) + +connect (spill_client, localhost, root,,); +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null + +connection default; +--echo # Purge older history while the qualifying spill file is still open. +--eval PURGE BINARY LOGS TO '$binlog_before_spill' +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null + +--echo # Rotate while the qualifying spill file is still open. +FLUSH BINARY LOGS; +--let $binlog_after_spill_rotation = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = FLUSH opens a new active binary log while a BOLT spill exists. +--let $assert_cond = "$binlog_after_spill_rotation" != "$binlog_before_spill" +--source include/assert.inc +--exec ls $BOLT_TEMP_DIR/bolt_* > /dev/null + +connection spill_client; +COMMIT; +connection default; +--exec test ! -e $BOLT_TEMP_DIR/bolt_* +--let $promoted_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_header_file = $promoted_file +--source ../inc/validate_bolt_header.inc +--let $assert_text = The spill promotes after PURGE and FLUSH complete. +--let $assert_cond = "$promoted_file" != "$binlog_after_spill_rotation" +--source include/assert.inc +--let $assert_text = The promoted transaction preserves every row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc + +--echo # Rotate the promoted active file, then purge it as ordinary history. +FLUSH BINARY LOGS; +--let $successor_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = FLUSH rotates the promoted active binary log. +--let $assert_cond = "$successor_file" != "$promoted_file" +--source include/assert.inc +--exec test -e $MYSQLD_DATADIR/$promoted_file +--eval PURGE BINARY LOGS TO '$successor_file' +--let $file_does_not_exist = $MYSQLD_DATADIR/$promoted_file +--source include/file_does_not_exist.inc +SHOW BINARY LOGS; + +connection spill_client; +disconnect spill_client; +connection default; + +# Cleanup +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_savepoint-master.opt b/mysql-test/suite/binlog/t/binlog_bolt_savepoint-master.opt new file mode 100644 index 000000000000..acbe532f50a5 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_savepoint-master.opt @@ -0,0 +1,5 @@ +--binlog-cache-size=1M +--binlog-large-transaction-optimization-enabled=ON +--binlog-large-transaction-optimization-threshold=10M +--max-binlog-size=1G +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/binlog/t/binlog_bolt_savepoint.test b/mysql-test/suite/binlog/t/binlog_bolt_savepoint.test new file mode 100644 index 000000000000..6bfee510ed06 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_savepoint.test @@ -0,0 +1,252 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify a savepoint rollback uses the final transaction-cache size to choose +# either ordinary active-binlog copying or BOLT promotion. +# +# === Requirements === +# R1. A spilled cache truncated below binlog_cache_size commits normally. +# R2. A spilled cache above binlog_cache_size but below the BOLT threshold +# commits normally. +# R3. A spilled cache above the BOLT threshold promotes a new active binlog. +# R4. A case that commits normally was never a BOLT candidate, rather than a +# candidate that fell back. Neither counter moves. +# R5. A promoted file whose spilled cache was trimmed by a savepoint rollback +# is structurally valid. +# +# === Implementation === +# Configure a 1 MiB cache and a 10 MiB BOLT threshold. Each case spills the +# transaction cache, rolls back to a savepoint, commits, and checks the final +# active binary-log filename, both BOLT status counters, and the retained rows. +# +# Both counters are asserted everywhere. The filename alone would not be enough: +# the non-promoting cases expect it unchanged, which is also what a server with +# BOLT disabled produces. The missed counter is what separates "never a +# candidate" from "was a candidate and fell back". +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_max_binlog_size = query_get_value(SELECT @@GLOBAL.max_binlog_size, @@GLOBAL.max_binlog_size, 1) + +# The shared promoted-file validator resolves $bolt_header_file against this. +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +--enable_query_log +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL max_binlog_size = 1073741824; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +############################################################################ +# Case 1: Rollback leaves less than binlog_cache_size. +# Expected behavior: copy the retained cache to the current active binlog. +############################################################################ +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--echo # Savepoint rollback below binlog_cache_size keeps the active binlog. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 512000)); +SAVEPOINT before_spill; +INSERT INTO t1 VALUES (2, REPEAT('b', 2000000)); +ROLLBACK TO SAVEPOINT before_spill; +COMMIT; +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text = A final cache below binlog_cache_size is copied to the active binlog. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A final cache below binlog_cache_size does not promote. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A final cache below binlog_cache_size was never a candidate. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = Savepoint rollback retains only the pre-spill row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 1 +--source include/assert.inc +--let $assert_text = Savepoint rollback discards the post-savepoint row. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 1 +--source include/assert.inc +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 2: Rollback leaves more than binlog_cache_size but less than threshold. +# Expected behavior: copy the retained cache to the current active binlog. +############################################################################ +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--echo # Savepoint rollback below the BOLT threshold keeps the active binlog. +BEGIN; +INSERT INTO t1 VALUES (11, REPEAT('c', 2000000)); +SAVEPOINT before_discard; +INSERT INTO t1 VALUES (12, REPEAT('d', 3000000)); +ROLLBACK TO SAVEPOINT before_discard; +COMMIT; +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text = A final cache below the BOLT threshold is copied to the active binlog. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A final cache below the BOLT threshold does not promote. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A final cache below the BOLT threshold was never a candidate. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = Savepoint rollback retains the above-cache pre-savepoint row. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 1 +--source include/assert.inc +--let $assert_text = Savepoint rollback discards the above-cache post-savepoint row. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 11 +--source include/assert.inc +--disable_query_log +TRUNCATE t1; +--enable_query_log + +############################################################################ +# Case 3: Rollback leaves more than binlog_cache_size and BOLT threshold. +# Expected behavior: promote a new active binlog. +############################################################################ +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--echo # Savepoint rollback above the BOLT threshold promotes a new binlog. +BEGIN; +INSERT INTO t1 VALUES (21, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (22, REPEAT('f', 4000000)); +INSERT INTO t1 VALUES (23, REPEAT('g', 4000000)); +SAVEPOINT before_discard; +INSERT INTO t1 VALUES (24, REPEAT('h', 2000000)); +ROLLBACK TO SAVEPOINT before_discard; +COMMIT; +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text = A final cache above the BOLT threshold promotes a new active binlog. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A final cache above the BOLT threshold increments the optimization counter once. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = A promoted savepoint transaction does not increment the missed counter. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = Savepoint rollback retains the promoted pre-savepoint rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = Savepoint rollback discards the promoted post-savepoint row. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 66 +--source include/assert.inc +# The promoted file's spilled cache was trimmed by the savepoint rollback, so +# validate that its header still describes the file it ended up being. +--let $bolt_header_file = $binlog_file_after +--source ../inc/validate_bolt_header.inc + +--disable_query_log +TRUNCATE t1; +--enable_query_log +############################################################################ +# Case 4: Rollback, then re-append while staying below the threshold. +# Expected behavior: copy the retained cache to the current active binlog. +############################################################################ +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--echo # Re-append after rollback below the threshold keeps the active binlog. +BEGIN; +INSERT INTO t1 VALUES (31, REPEAT('a', 2000000)); +SAVEPOINT sp_reappend; +INSERT INTO t1 VALUES (32, REPEAT('b', 3000000)); +ROLLBACK TO SAVEPOINT sp_reappend; +INSERT INTO t1 VALUES (33, REPEAT('c', 3000000)); +COMMIT; +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text = A re-appended cache below the threshold is copied to the active binlog. +--let $assert_cond = "$binlog_file_after" = "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A re-appended cache below the threshold does not promote. +--let $assert_cond = $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text = A re-appended cache below the threshold was never a candidate. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = Re-append after rollback keeps the pre-savepoint and re-appended rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 2 +--source include/assert.inc +--let $assert_text = Re-append after rollback discards only the rolled-back row. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 64 +--source include/assert.inc +--let $assert_text = The rolled-back row is absent after re-append. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1 WHERE id = 32, count, 1] = 0 +--source include/assert.inc +--let $assert_text = The re-appended row has its full length. +--let $assert_cond = [SELECT LENGTH(data) AS len FROM t1 WHERE id = 33, len, 1] = 3000000 +--source include/assert.inc +--disable_query_log +TRUNCATE t1; +--enable_query_log +############################################################################ +# Case 5: Rollback below the threshold, then re-append back above it. +# Expected behavior: promote a new active binlog. +############################################################################ +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--echo # Re-append after rollback above the threshold promotes a new binlog. +BEGIN; +INSERT INTO t1 VALUES (41, REPEAT('d', 5000000)); +SAVEPOINT sp_recross; +INSERT INTO t1 VALUES (42, REPEAT('e', 8000000)); +ROLLBACK TO SAVEPOINT sp_recross; +INSERT INTO t1 VALUES (43, REPEAT('f', 8000000)); +COMMIT; +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text = A re-appended cache above the threshold promotes a new active binlog. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $assert_text = A re-appended cache above the threshold increments the optimization counter once. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = A re-crossed promotion does not increment the missed counter. +--let $assert_cond = $missed_after = $missed_before +--source include/assert.inc +--let $assert_text = Re-crossing the threshold keeps the pre-savepoint and re-appended rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 2 +--source include/assert.inc +--let $assert_text = Re-crossing the threshold discards only the rolled-back row. +--let $assert_cond = [SELECT SUM(id) AS id_sum FROM t1, id_sum, 1] = 84 +--source include/assert.inc +--let $assert_text = The rolled-back row is absent after promotion. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1 WHERE id = 42, count, 1] = 0 +--source include/assert.inc +--let $assert_text = The re-appended row has its full length in the promoted file. +--let $assert_cond = [SELECT LENGTH(data) AS len FROM t1 WHERE id = 43, len, 1] = 8000000 +--source include/assert.inc +# Re-crossing the threshold after a trim is the other way a promoted file can +# have been rewritten, so validate this one too. +--let $bolt_header_file = $binlog_file_after +--source ../inc/validate_bolt_header.inc +# Cleanup +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL max_binlog_size = $save_max_binlog_size +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_savepoint_recovery-master.opt b/mysql-test/suite/binlog/t/binlog_bolt_savepoint_recovery-master.opt new file mode 100644 index 000000000000..bb8062320d6b --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_savepoint_recovery-master.opt @@ -0,0 +1,6 @@ +--binlog-cache-size=1M +--binlog-large-transaction-optimization-enabled=ON +--binlog-large-transaction-optimization-threshold=10M +--max-binlog-size=1G +--sync-binlog=1 +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/binlog/t/binlog_bolt_savepoint_recovery.test b/mysql-test/suite/binlog/t/binlog_bolt_savepoint_recovery.test new file mode 100644 index 000000000000..b5418b433a61 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_savepoint_recovery.test @@ -0,0 +1,150 @@ +# === Purpose === +# Verify crash recovery of a file promoted after a savepoint rollback. That is +# the only path where the physical spill file was trimmed by my_chsize, so +# recovery depends on the terminating_event_offset recorded by finalize() +# landing exactly at the trimmed end. +# +# === Requirements === +# R1. A crash after the main-index update rolls a post-rollback promoted +# transaction forward, and the promoted file's own header still points at a +# decodable terminating event. +# R2. A crash after the promote rename, before the main-index update, rolls a +# post-rollback promoted transaction back and leaves no extra binary log +# and no spill file behind. +# +# === Implementation === +# 1. Enable BOLT with a 10 MiB threshold and a clean test table. +# 2. In each case, spill above the threshold, take a savepoint, write more, +# roll back to the savepoint while staying above the threshold, then crash +# at the boundary under test during COMMIT. +# 3. Restart and verify the 2PC decision, the row contents, and that nothing +# was orphaned. +# +--source include/not_crashrep.inc +--source include/not_valgrind.inc +--source include/have_debug.inc +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_TEMP_DIR = $MYSQLD_DATADIR/#binlog_temp_files + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_max_binlog_size = query_get_value(SELECT @@GLOBAL.max_binlog_size, @@GLOBAL.max_binlog_size, 1) +--let $save_sync_binlog = query_get_value(SELECT @@GLOBAL.sync_binlog, @@GLOBAL.sync_binlog, 1) + +# Setup +--let $BOLT_RESTART = restart: +--let $BOLT_RESTART = $BOLT_RESTART --binlog-large-transaction-optimization-enabled=ON +--let $BOLT_RESTART = $BOLT_RESTART --binlog-large-transaction-optimization-threshold=10M +--let $BOLT_RESTART = $BOLT_RESTART --max-binlog-size=1G +--let $BOLT_RESTART = $BOLT_RESTART --sync-binlog=1 +# REPEAT() payloads compress to almost nothing, so a compressed cache would +# never reach the threshold and the transaction would never be promoted. +--let $BOLT_RESTART = $BOLT_RESTART --binlog-transaction-compression=OFF + +--disable_query_log +DROP TABLE IF EXISTS t1; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +RESET BINARY LOGS AND GTIDS; +--enable_query_log + +############################################################################ +# Case 1: Savepoint rollback, still above the threshold, then crash after the +# main-index update. +# Expected behavior: rollforward. The kept rows are present, the rolled-back +# row is absent, and the promoted file validates. +############################################################################ +--echo # Post-main-index-update crash after a savepoint rollback rolls forward. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_main_index_update"; +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +SAVEPOINT sp_after_threshold; +INSERT INTO t1 VALUES (4, REPEAT('d', 5000000)); +ROLLBACK TO SAVEPOINT sp_after_threshold; +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc + +--let $assert_text = A post-main-index-update crash commits the kept rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc +--let $assert_text = The rolled-back row is not recovered. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1 WHERE id = 4, count, 1] = 0 +--source include/assert.inc +--let $assert_text = The last kept row survived the chsize trim intact. +--let $assert_cond = [SELECT LENGTH(data) AS len FROM t1 WHERE id = 3, len, 1] = 4000000 +--source include/assert.inc + +# The promoted file was trimmed by my_chsize before promotion, so its recorded +# terminating-event offset has to still land on a decodable event. The shared +# validator decodes the file with --verify-binlog-checksum and cross-checks the +# recorded offset and type against the raw event-header byte, which is the +# off-by-one check this scenario exists for. +--let $bolt_header_file = query_get_value(SHOW BINARY LOGS, Log_name, 2) +--source ../inc/validate_bolt_header.inc +--list_files $BOLT_TEMP_DIR +--disable_query_log +TRUNCATE t1; +RESET BINARY LOGS AND GTIDS; +--enable_query_log + +############################################################################ +# Case 2: Savepoint rollback, still above the threshold, then crash after the +# promote rename but before the main-index update. +# Expected behavior: rollback, and nothing left over. +############################################################################ +--echo # Post-promote-rename crash after a savepoint rollback rolls back. +--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--disable_query_log +--disable_result_log +SET SESSION DEBUG="+d,crash_bolt_after_promote_rename"; +BEGIN; +INSERT INTO t1 VALUES (11, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (12, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (13, REPEAT('c', 4000000)); +SAVEPOINT sp_after_threshold; +INSERT INTO t1 VALUES (14, REPEAT('d', 5000000)); +ROLLBACK TO SAVEPOINT sp_after_threshold; +--error 2013 +COMMIT; +--enable_result_log +--enable_query_log +--source include/wait_until_disconnected.inc +--exec echo "$BOLT_RESTART" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect +--source include/wait_until_connected_again.inc + +--let $assert_text = A post-promote-rename crash rolls back all rows. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc + +# The renamed file never reached the main index, so startup recovery deleted it +# from the data directory using the purge-index record. Its name is then reused +# by the binary log the restart opens, so the deletion cannot be checked by +# name; assert instead that exactly two binary logs exist -- the one that was +# active before the transaction, and the one the restart opened -- so no third, +# orphaned file was indexed. +--let $bolt_third_log = query_get_value(SHOW BINARY LOGS, Log_name, 3) +--let $assert_text = Recovery left no third, orphaned binary log. +--let $assert_cond = "$bolt_third_log" = "No such row" +--source include/assert.inc +--list_files $BOLT_TEMP_DIR + +# Cleanup +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL max_binlog_size = $save_max_binlog_size +--eval SET GLOBAL sync_binlog = $save_sync_binlog +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_sysvars.test b/mysql-test/suite/binlog/t/binlog_bolt_sysvars.test new file mode 100644 index 000000000000..04bd2395def0 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_sysvars.test @@ -0,0 +1,395 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT threshold and binlog-cache normalization during runtime SETs and +# server restart. +# +# === Requirements === +# R1. The optimization threshold is never below binlog_cache_size. +# R2. BOLT-on normalization is reported to the client and error log. +# R3. BOLT-off normalization preserves the invariant without BOLT warnings. +# R4. Both the knob and the threshold are captured at the transaction's first +# event, so changing either mid-transaction does not affect it. +# R5. A rollback that empties the transaction cache discards what it captured, +# so the next transaction in the session reads the configuration again. +# +# === Implementation === +# Cases 1 to 6 exercise the threshold/binlog_cache_size normalization rules, +# with BOLT on and off, after a runtime SET and after a restart. +# +# Cases 7 to 9 commit real transactions and change the configuration from a +# second session while each one is open, to check when the values are read. +# +# Cases 10 and 11 cover the other end of that lifetime. The captured values are +# cleared only when the cache is reset, and a rollback can truncate the cache +# back to empty without ending the transaction. Each case empties the cache +# that way, turns the knob off, and commits a large transaction in the same +# session: promoting it would mean the stale capture was reused. +# +--source include/have_log_bin.inc + +# Preserve the caller's configuration and restore it after the assertions. +--let $save_enabled=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $save_cache_size=query_get_value(SELECT @@GLOBAL.binlog_cache_size, @@GLOBAL.binlog_cache_size, 1) + +# Setup +# Start all BOLT-on runtime cases from a default server configuration. +--let $restart_parameters=restart: +--source include/restart_mysqld.inc +--disable_query_log +--disable_warnings +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL binlog_cache_size = 10485760; +--enable_query_log +--enable_warnings + +--echo # Case 1: BOLT is ON; binlog_cache_size > binlog_large_transaction_optimization_threshold. +# Verify normalization after a runtime SET. +--source include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 20971520; +--let $error_pattern=Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.] +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= Increasing binlog_cache_size raises the optimization threshold. +--let $assert_cond= $threshold = 20971520 +--source include/assert.inc + +# Verify normalization during restart recovery. +--source include/save_error_log_position.inc +--let $restart_parameters=restart: --binlog-large-transaction-optimization-enabled=ON --binlog-large-transaction-optimization-threshold=10M --binlog-cache-size=20M +--source include/restart_mysqld.inc +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +--let $error_pattern=Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.] +--source include/assert_error_log.inc + +--echo # Case 2: BOLT is ON; binlog_large_transaction_optimization_threshold < binlog_cache_size. +# Verify normalization after a runtime SET. +--source include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--let $error_pattern=Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 10485760 bytes to 20971520 bytes to match binlog_cache_size[.] +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= A lower optimization threshold is raised to match binlog_cache_size. +--let $assert_cond= $threshold = 20971520 +--source include/assert.inc + +--echo # Case 3: BOLT is ON; binlog_large_transaction_optimization_threshold > binlog_cache_size. +# Verify the valid relationship is unchanged after a runtime SET. +--source include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 22020096; +--let $error_pattern=NONE +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= A threshold above binlog_cache_size is unchanged. +--let $assert_cond= $threshold = 22020096 +--source include/assert.inc + +# Verify lowering binlog_cache_size does not lower the threshold. +--source include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 10485760; +--let $error_pattern=NONE +--source include/assert_error_log.inc +--let $cache_size=query_get_value(SELECT @@GLOBAL.binlog_cache_size, @@GLOBAL.binlog_cache_size, 1) +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= Lowering binlog_cache_size does not lower the threshold. +--let $assert_cond= $cache_size = 10485760 AND $threshold = 22020096 +--source include/assert.inc + +# Verify the valid configuration restarts without adjustment. +--source include/save_error_log_position.inc +--let $restart_parameters=restart: --binlog-large-transaction-optimization-enabled=ON --binlog-cache-size=20M --binlog-large-transaction-optimization-threshold=21M +--source include/restart_mysqld.inc +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +--let $error_pattern=NONE +--source include/assert_error_log.inc + +# Start all BOLT-off runtime cases from a default server configuration. +--let $restart_parameters=restart: +--source include/restart_mysqld.inc +--disable_query_log +--disable_warnings +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL binlog_cache_size = 10485760; +--enable_query_log +--enable_warnings + +--echo # Case 4: BOLT is OFF; binlog_cache_size > binlog_large_transaction_optimization_threshold. +# Verify normalization remains silent after a runtime SET. +--source include/save_error_log_position.inc +SET GLOBAL binlog_cache_size = 20971520; +--let $error_pattern=NONE +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= Increasing binlog_cache_size raises the threshold while BOLT is off. +--let $assert_cond= $threshold = 20971520 +--source include/assert.inc + +# Verify startup normalization remains silent while BOLT is off. +--source include/save_error_log_position.inc +--let $restart_parameters=restart: --binlog-large-transaction-optimization-enabled=OFF --binlog-large-transaction-optimization-threshold=10M --binlog-cache-size=20M +--source include/restart_mysqld.inc +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +--let $error_pattern=NONE +--source include/assert_error_log.inc + +--echo # Case 5: BOLT is OFF; binlog_large_transaction_optimization_threshold < binlog_cache_size. +# Verify normalization remains silent after a runtime SET. +--source include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--let $error_pattern=NONE +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= A lower threshold is raised while BOLT is off. +--let $assert_cond= $threshold = 20971520 +--source include/assert.inc + +--echo # Case 6: BOLT is OFF; binlog_large_transaction_optimization_threshold > binlog_cache_size. +# Verify the valid relationship remains unchanged after a runtime SET. +--source include/save_error_log_position.inc +SET GLOBAL binlog_large_transaction_optimization_threshold = 22020096; +--let $error_pattern=NONE +--source include/assert_error_log.inc +--let $threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $assert_text= A threshold above binlog_cache_size is unchanged while BOLT is off. +--let $assert_cond= $threshold = 22020096 +--source include/assert.inc + +# Verify the valid configuration restarts silently while BOLT is off. +--source include/save_error_log_position.inc +--let $restart_parameters=restart: --binlog-large-transaction-optimization-enabled=OFF --binlog-cache-size=20M --binlog-large-transaction-optimization-threshold=21M +--source include/restart_mysqld.inc +SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold AS threshold; +--let $error_pattern=NONE +--source include/assert_error_log.inc + +############################################################################ +# Cases 7 to 9: when the knob and the threshold are read. +# +# Both are captured at the transaction's first event, not at commit. Each case +# below starts a transaction, changes one value from a second session, and +# checks the transaction still behaves the way it started. +# +# binlog_cache_size is at its minimum so the transactions certainly spill, +# which promotion requires. binlog_format is set per session rather than by +# sourcing have_binlog_format_row.inc, so cases 1 to 6 keep running in all +# three format combinations. +############################################################################ +--let $restart_parameters=restart: +--source include/restart_mysqld.inc +--disable_query_log +--disable_warnings +DROP TABLE IF EXISTS t1; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_cache_size = 4096; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_format = ROW; +SET SESSION binlog_transaction_compression = OFF; +RESET BINARY LOGS AND GTIDS; +--enable_warnings +--enable_query_log +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +connect (bolt_switch, localhost, root,,); +connection default; + +--echo # Case 7: Turning the knob off mid-transaction does not affect it. +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('f', 4000000)); +--enable_result_log +--enable_query_log +connection bolt_switch; +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +connection default; +--disable_query_log +--disable_result_log +INSERT INTO t1 VALUES (3, REPEAT('g', 4000000)); +COMMIT; +--enable_result_log +--enable_query_log +--let $bolt_count_after=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text= A transaction latched with the knob on still promotes after it is turned off. +--let $assert_cond= $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text= The knob-off promotion commits all rows. +--let $assert_cond= [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc + +--echo # Case 8: Turning the knob on mid-transaction does not affect it either. +--disable_query_log +TRUNCATE t1; +--enable_query_log +# The knob is still OFF from case 7, so this transaction latches "disabled". +--let $bolt_count_before=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('h', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('i', 4000000)); +--enable_result_log +--enable_query_log +connection bolt_switch; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +connection default; +--disable_query_log +--disable_result_log +INSERT INTO t1 VALUES (3, REPEAT('j', 4000000)); +COMMIT; +--enable_result_log +--enable_query_log +--let $bolt_count_after=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text= A transaction latched with the knob off does not promote after it is turned on. +--let $assert_cond= $bolt_count_after = $bolt_count_before +--source include/assert.inc +--let $assert_text= A transaction latched with the knob off is not a missed optimization. +--let $assert_cond= $missed_after = $missed_before +--source include/assert.inc +--let $assert_text= The knob-on transaction still commits all rows. +--let $assert_cond= [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc + +--echo # Case 9: Raising the threshold mid-transaction does not affect it. +--disable_query_log +TRUNCATE t1; +--enable_query_log +--let $bolt_count_before=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('k', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('l', 4000000)); +--enable_result_log +--enable_query_log +# Far above this transaction's size, so a threshold read at commit time would +# disqualify it. +connection bolt_switch; +SET GLOBAL binlog_large_transaction_optimization_threshold = 1073741824; +connection default; +--disable_query_log +--disable_result_log +INSERT INTO t1 VALUES (3, REPEAT('m', 4000000)); +COMMIT; +--enable_result_log +--enable_query_log +--let $bolt_count_after=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text= A transaction latched below the threshold still promotes after it is raised. +--let $assert_cond= $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text= The raised-threshold promotion commits all rows. +--let $assert_cond= [SELECT COUNT(*) AS count FROM t1, count, 1] = 3 +--source include/assert.inc + +# Case 9 left the threshold above anything the cases below commit. Restore a +# threshold they can exceed, and the knob on so they capture "enabled". +--disable_query_log +--disable_warnings +connection bolt_switch; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +connection default; +TRUNCATE t1; +--enable_warnings +--enable_query_log + +--echo # Case 10: a cache emptied by a statement rollback re-reads the knob. +# The 4 MB row exceeds binlog_row_event_max_size, so the pending rows event is +# written into the cache, and that write is what captures the configuration. +# The duplicate key then rolls the statement back, and it is the transaction's +# first statement, so the cache truncates to empty rather than to a prefix. +BEGIN; +--disable_query_log +--disable_result_log +--error ER_DUP_ENTRY +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)), (1, REPEAT('b', 4000000)); +--enable_result_log +--enable_query_log +COMMIT; +--let $assert_text= The rolled-back statement left the cache empty. +--let $assert_cond= [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +connection bolt_switch; +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +connection default; +--let $bolt_count_before=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (10, REPEAT('c', 4000000)); +INSERT INTO t1 VALUES (11, REPEAT('d', 4000000)); +INSERT INTO t1 VALUES (12, REPEAT('e', 4000000)); +COMMIT; +--enable_result_log +--enable_query_log +--let $bolt_count_after=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text= A cache emptied by a statement rollback does not promote the next transaction after the knob is turned off. +--let $assert_cond= $bolt_count_after = $bolt_count_before +--source include/assert.inc + +--disable_query_log +--disable_warnings +connection bolt_switch; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +connection default; +TRUNCATE t1; +--enable_warnings +--enable_query_log + +--echo # Case 11: a cache emptied by a savepoint rollback re-reads the knob. +# Reaches the empty cache through the savepoint path instead. The savepoint is +# taken before any event is cached, so rolling back to it empties the cache. +BEGIN; +SAVEPOINT s1; +--disable_query_log +--disable_result_log +INSERT INTO t1 VALUES (1, REPEAT('f', 4000000)); +--enable_result_log +--enable_query_log +ROLLBACK TO SAVEPOINT s1; +COMMIT; +--let $assert_text= The savepoint rollback left the cache empty. +--let $assert_cond= [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +connection bolt_switch; +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +connection default; +--let $bolt_count_before=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--disable_query_log +--disable_result_log +BEGIN; +INSERT INTO t1 VALUES (20, REPEAT('g', 4000000)); +INSERT INTO t1 VALUES (21, REPEAT('h', 4000000)); +INSERT INTO t1 VALUES (22, REPEAT('i', 4000000)); +COMMIT; +--enable_result_log +--enable_query_log +--let $bolt_count_after=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text= A cache emptied by a savepoint rollback does not promote the next transaction after the knob is turned off. +--let $assert_cond= $bolt_count_after = $bolt_count_before +--source include/assert.inc + +connection default; +disconnect bolt_switch; +connection default; +DROP TABLE t1; + +--let $restart_parameters=restart: +--source include/restart_mysqld.inc + +# Restore the caller's variable-dependent configuration without recording +# environment-specific values in the result file. +--disable_query_log +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--eval SET GLOBAL binlog_cache_size = $save_cache_size +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--enable_query_log diff --git a/mysql-test/suite/binlog/t/binlog_bolt_tmp_dir_cleanup.test b/mysql-test/suite/binlog/t/binlog_bolt_tmp_dir_cleanup.test new file mode 100644 index 000000000000..6b481efd0f78 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_bolt_tmp_dir_cleanup.test @@ -0,0 +1,85 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify release binaries clean managed BOLT spill files and reject unsafe +# temporary-directory entries. +# +# === Requirements === +# R1. Startup removes valid managed BOLT spill files. +# R2. Startup removes a spill file still carrying its mkstemp name, including +# one with upper-case letters. +# R3. Startup rejects symlinked directories and unsafe directory entries. +# R4. Rejected startup fixtures can be removed before the MTR server restarts. +# +# === Implementation === +# 1. Initialize paths for the managed spill directory and rejection logs. +# 2. Restart with valid managed files and verify they are removed. +# 3. Restart with pre-rename mkstemp names and verify they are removed. +# 4. Attempt standalone startup with each unsafe fixture and verify rejection. +# +--source include/not_windows.inc +--source include/have_log_bin.inc +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $BOLT_CLEANUP_DIR = $MYSQLD_DATADIR/#binlog_temp_files +--let $BOLT_CLEANUP_REJECT_LOG = $MYSQL_TMP_DIR/binlog_bolt_cleanup_reject.err +--let $BOLT_CLEANUP_DIR_TARGET = $MYSQLD_DATADIR/binlog_bolt_cleanup_dir_target +--let $BOLT_CLEANUP_FILE_TARGET = $MYSQLD_DATADIR/binlog_bolt_cleanup_file_target +# Setup +--exec touch $BOLT_CLEANUP_DIR/bolt_0123456789abcdef0123456789abcdef +--exec touch $BOLT_CLEANUP_DIR/bolt_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +--exec touch $BOLT_CLEANUP_DIR/bolt_0123456789abcdefabcdef0123456789 +--echo Created 3 managed BOLT spill files +--source include/restart_mysqld.inc +--list_files $BOLT_CLEANUP_DIR +--echo Startup cleanup removed all managed BOLT spill files +--echo # Remove a pre-rename mkstemp name containing upper case. +--exec touch $BOLT_CLEANUP_DIR/bolt_aB3xY9 +--exec touch $BOLT_CLEANUP_DIR/bolt_ZZZZZZ +--source include/restart_mysqld.inc +--list_files $BOLT_CLEANUP_DIR +--echo Startup cleanup removed the pre-rename mkstemp files +# Prepare each startup fixture while the MTR server is running. Stop it only +# immediately before the independent release-binary startup attempt, then +# remove the rejected fixture before bringing the MTR server back. +--echo # Reject a symbolic link for #binlog_temp_files. +--rmdir $BOLT_CLEANUP_DIR +--exec mkdir $BOLT_CLEANUP_DIR_TARGET +--exec ln -s $BOLT_CLEANUP_DIR_TARGET $BOLT_CLEANUP_DIR +--source include/shutdown_mysqld.inc +--error 1 +--exec $MYSQLD_CMD --loose-console > $BOLT_CLEANUP_REJECT_LOG 2>&1 +--exec grep -Fq "it must be a directory and cannot be a symbolic link." $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_DIR +--rmdir $BOLT_CLEANUP_DIR_TARGET +--exec mkdir $BOLT_CLEANUP_DIR +--source include/start_mysqld.inc +--echo # Reject an unrecognized regular entry. +--exec touch $BOLT_CLEANUP_DIR/not_a_managed_bolt_file +--source include/shutdown_mysqld.inc +--error 1 +--exec $MYSQLD_CMD --loose-console > $BOLT_CLEANUP_REJECT_LOG 2>&1 +--exec grep -Fq "found unsafe entry 'not_a_managed_bolt_file'." $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_DIR/not_a_managed_bolt_file +--source include/start_mysqld.inc +--echo # Reject a symbolic-link entry, even with a managed file name. +--exec touch $BOLT_CLEANUP_FILE_TARGET +--exec ln -s $BOLT_CLEANUP_FILE_TARGET $BOLT_CLEANUP_DIR/bolt_11111111111111111111111111111111 +--source include/shutdown_mysqld.inc +--error 1 +--exec $MYSQLD_CMD --loose-console > $BOLT_CLEANUP_REJECT_LOG 2>&1 +--exec grep -Fq "found unsafe entry 'bolt_11111111111111111111111111111111'." $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_DIR/bolt_11111111111111111111111111111111 +--remove_file $BOLT_CLEANUP_FILE_TARGET +--source include/start_mysqld.inc +--echo # Reject a nested directory, even with a managed file name. +--exec mkdir $BOLT_CLEANUP_DIR/bolt_22222222222222222222222222222222 +--source include/shutdown_mysqld.inc +--error 1 +--exec $MYSQLD_CMD --loose-console > $BOLT_CLEANUP_REJECT_LOG 2>&1 +--exec grep -Fq "found unsafe entry 'bolt_22222222222222222222222222222222'." $BOLT_CLEANUP_REJECT_LOG +--remove_file $BOLT_CLEANUP_REJECT_LOG +--rmdir $BOLT_CLEANUP_DIR/bolt_22222222222222222222222222222222 +--source include/start_mysqld.inc diff --git a/mysql-test/suite/binlog/t/binlog_encryption_random_access.test b/mysql-test/suite/binlog/t/binlog_encryption_random_access.test index 9d137946b57d..83fbe864eaea 100644 --- a/mysql-test/suite/binlog/t/binlog_encryption_random_access.test +++ b/mysql-test/suite/binlog/t/binlog_encryption_random_access.test @@ -16,6 +16,10 @@ # Suppression of error messages CALL mtr.add_suppression('Unsafe statement written to the binary log using statement format'); +--let $messages = Could not optimize large transaction execution in the binary log because binary log encryption was enabled while the transaction was running; standard binary logging was used instead. +--let $suppress_silent = 1 +--source include/suppress_messages.inc +--let $suppress_silent = --source include/have_component_keyring_file.inc --source ../mysql-test/suite/component_keyring_file/inc/setup_component.inc @@ -63,6 +67,38 @@ while ($trx) } --enable_query_log +--echo # Binlog encryption forces a qualifying row transaction to use the standard path. +--let $save_enabled=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold=query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--disable_query_log +SET SESSION binlog_format = ROW; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--enable_query_log +CREATE TABLE t_large (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +--let $count_before_encryption=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_before_encryption=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +BEGIN; +INSERT INTO t_large VALUES (1, REPEAT('a', 5000000)); +INSERT INTO t_large VALUES (2, REPEAT('b', 5000000)); +INSERT INTO t_large VALUES (3, REPEAT('c', 5000000)); +COMMIT; +--let $count_after_encryption=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $missed_after_encryption=query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_missed_count', Value, 1) +--let $assert_text= An encrypted binary log uses the standard path. +--let $assert_cond= $count_after_encryption = $count_before_encryption +--source include/assert.inc +--let $assert_text= An encrypted binary log increments the missed counter once. +--let $assert_cond= $missed_after_encryption = $missed_before_encryption + 1 +--source include/assert.inc +--let $assert_text= The encrypted fallback transaction commits all rows. +--let $assert_cond= [SELECT COUNT(*) AS count FROM t_large, count, 1] = 3 +--source include/assert.inc + # Cleanup -DROP TABLE t1; +DROP TABLE t_large, t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--enable_query_log --source ../mysql-test/suite/component_keyring_file/inc/teardown_component.inc diff --git a/mysql-test/suite/binlog_gtid/r/binlog_bolt_gtid_tag.result b/mysql-test/suite/binlog_gtid/r/binlog_bolt_gtid_tag.result new file mode 100644 index 000000000000..30c7375883b2 --- /dev/null +++ b/mysql-test/suite/binlog_gtid/r/binlog_bolt_gtid_tag.result @@ -0,0 +1,29 @@ +Warnings: +Note 1051 Unknown table 'test.t1' +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +# Commit tagged and untagged transactions before the promoted transaction. +SET GTID_NEXT = 'AUTOMATIC:prior_large_trx_tag'; +INSERT INTO t1 VALUES (1, 'prior-tagged-transaction'); +SET GTID_NEXT = 'AUTOMATIC'; +INSERT INTO t1 VALUES (2, 'prior-untagged-transaction'); +SET GTID_NEXT = 'AUTOMATIC:promoted_large_trx_tag'; +BEGIN; +INSERT INTO t1 VALUES (3, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (4, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (5, REPEAT('c', 5000000)); +COMMIT; +SET GTID_NEXT = 'AUTOMATIC'; +include/assert.inc [The tagged GTID transaction was promoted.] +include/assert.inc [The tagged GTID promotion rotates to a new active binary log.] +include/assert.inc [The prior tagged GTID is persisted before promotion.] +include/assert.inc [The promoted tagged GTID is recorded in gtid_executed.] +# The promoted file retains both tag forms. +# Restart and verify both tags survive persisted GTID state reload. +# restart +include/assert.inc [The prior tagged GTID survives restart.] +include/assert.inc [The promoted tagged GTID survives restart.] +include/assert.inc [Both GTID tags remain in mysql.gtid_executed after restart.] +DROP TABLE t1; diff --git a/mysql-test/suite/binlog_gtid/r/binlog_gtid_show_binlog_events.result b/mysql-test/suite/binlog_gtid/r/binlog_gtid_show_binlog_events.result index 4397aa451e87..a95bb5dc9035 100644 --- a/mysql-test/suite/binlog_gtid/r/binlog_gtid_show_binlog_events.result +++ b/mysql-test/suite/binlog_gtid/r/binlog_gtid_show_binlog_events.result @@ -1,7 +1,7 @@ CREATE TABLE t1 (c1 INT); include/assert.inc [Verify that the starting offset (4) of an event after the invalid position 1 is displayed.] include/assert.inc [Verify that the starting offset (4) of an event at the valid position 4 is displayed.] -include/assert.inc [Verify that the starting offset (127) of an event after the invalid position 14 is displayed.] -include/assert.inc [Verify that the starting offset (158) of an event after the invalid position 127 is displayed.] -include/assert.inc [Verify that the starting offset (158) of an event at the valid position 157 is displayed.] +include/assert.inc [Verify that the starting offset (128) of an event after the invalid position 14 is displayed.] +include/assert.inc [Verify that the starting offset (159) of an event after the invalid position 129 is displayed.] +include/assert.inc [Verify that the starting offset (159) of an event at the valid position 159 is displayed.] DROP TABLE t1; diff --git a/mysql-test/suite/binlog_gtid/t/binlog_bolt_gtid_tag-master.opt b/mysql-test/suite/binlog_gtid/t/binlog_bolt_gtid_tag-master.opt new file mode 100644 index 000000000000..dc3126be991c --- /dev/null +++ b/mysql-test/suite/binlog_gtid/t/binlog_bolt_gtid_tag-master.opt @@ -0,0 +1 @@ +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/binlog_gtid/t/binlog_bolt_gtid_tag.test b/mysql-test/suite/binlog_gtid/t/binlog_bolt_gtid_tag.test new file mode 100644 index 000000000000..93473e402f8a --- /dev/null +++ b/mysql-test/suite/binlog_gtid/t/binlog_bolt_gtid_tag.test @@ -0,0 +1,101 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify BOLT promotion persists tagged GTIDs in the binary log and GTID +# execution state. +# +# === Requirements === +# R1. A qualifying tagged-GTID transaction is promoted into a new active file. +# R2. Recovery preserves its GTID in the binary log and mysql.gtid_executed. +# R3. The promoted tagged-GTID file satisfies the shared BOLT structural +# validation, including its LTH terminal-event metadata. +# +# === Implementation === +# 1. Initialize BOLT, GTID state, and a clean test table. +# 2. Commit a qualifying tagged-GTID transaction and verify file rotation. +# 3. Restart and verify persisted GTID state. +# +--source include/have_log_bin.inc +--source include/have_binlog_format_row.inc + +--let $save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +--enable_query_log +RESET BINARY LOGS AND GTIDS; +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +--echo # Commit tagged and untagged transactions before the promoted transaction. +SET GTID_NEXT = 'AUTOMATIC:prior_large_trx_tag'; +INSERT INTO t1 VALUES (1, 'prior-tagged-transaction'); +SET GTID_NEXT = 'AUTOMATIC'; +INSERT INTO t1 VALUES (2, 'prior-untagged-transaction'); + +--let $server_uuid = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) +--let $count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +SET GTID_NEXT = 'AUTOMATIC:promoted_large_trx_tag'; +BEGIN; +INSERT INTO t1 VALUES (3, REPEAT('a', 5000000)); +INSERT INTO t1 VALUES (4, REPEAT('b', 5000000)); +INSERT INTO t1 VALUES (5, REPEAT('c', 5000000)); +COMMIT; +SET GTID_NEXT = 'AUTOMATIC'; + +--let $count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = The tagged GTID transaction was promoted. +--let $assert_cond = $count_after = $count_before + 1 +--source include/assert.inc +--let $assert_text = The tagged GTID promotion rotates to a new active binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc +--let $prior_tag_rows = query_get_value(SELECT COUNT(*) AS count FROM mysql.gtid_executed WHERE source_uuid = "$server_uuid" AND gtid_tag = 'prior_large_trx_tag' AND interval_start = 1 AND interval_end = 1, count, 1) +--let $assert_text = The prior tagged GTID is persisted before promotion. +--let $assert_cond = $prior_tag_rows = 1 +--source include/assert.inc +--let $promoted_tag_executed = `SELECT GTID_SUBSET('$server_uuid:promoted_large_trx_tag:1', @@GLOBAL.GTID_EXECUTED)` +--let $assert_text = The promoted tagged GTID is recorded in gtid_executed. +--let $assert_cond = $promoted_tag_executed = 1 +--source include/assert.inc + +--echo # The promoted file retains both tag forms. +--let $promoted_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $MYSQLD_DATADIR = `SELECT @@datadir` +--let $bolt_header_file = $promoted_file +--source common/binlog/validate_bolt_file.inc +--let $gtid_output = $MYSQLTEST_VARDIR/tmp/large_trx_tagged_gtids.txt +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$promoted_file > $gtid_output +--exec grep -Eq '[0-9a-f-]+:1-2' $gtid_output +--exec grep -Eq '[0-9a-f-]+:prior_large_trx_tag:1' $gtid_output +--exec grep -Eq '[0-9a-f-]+:promoted_large_trx_tag:1' $gtid_output +--remove_file $gtid_output + +--echo # Restart and verify both tags survive persisted GTID state reload. +--let $do_not_echo_parameters = 1 +--source include/restart_mysqld.inc +--let $do_not_echo_parameters = + +--let $prior_tag_executed = `SELECT GTID_SUBSET('$server_uuid:prior_large_trx_tag:1', @@GLOBAL.GTID_EXECUTED)` +--let $assert_text = The prior tagged GTID survives restart. +--let $assert_cond = $prior_tag_executed = 1 +--source include/assert.inc +--let $promoted_tag_executed = `SELECT GTID_SUBSET('$server_uuid:promoted_large_trx_tag:1', @@GLOBAL.GTID_EXECUTED)` +--let $assert_text = The promoted tagged GTID survives restart. +--let $assert_cond = $promoted_tag_executed = 1 +--source include/assert.inc +--let $persisted_tag_rows = query_get_value(SELECT COUNT(*) AS count FROM mysql.gtid_executed WHERE source_uuid = "$server_uuid" AND (gtid_tag = 'prior_large_trx_tag' OR gtid_tag = 'promoted_large_trx_tag'), count, 1) +--let $assert_text = Both GTID tags remain in mysql.gtid_executed after restart. +--let $assert_cond = $persisted_tag_rows = 2 +--source include/assert.inc + +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $save_threshold +--enable_query_log diff --git a/mysql-test/suite/binlog_gtid/t/binlog_gtid_show_binlog_events.test b/mysql-test/suite/binlog_gtid/t/binlog_gtid_show_binlog_events.test index 2837dd838e9f..9bbba8248224 100644 --- a/mysql-test/suite/binlog_gtid/t/binlog_gtid_show_binlog_events.test +++ b/mysql-test/suite/binlog_gtid/t/binlog_gtid_show_binlog_events.test @@ -33,16 +33,16 @@ CREATE TABLE t1 (c1 INT); --let $assert_cond="[SHOW BINLOG EVENTS FROM 4 LIMIT 1, Pos, 1]" = 4 --source include/assert.inc ---let $assert_text=Verify that the starting offset (127) of an event after the invalid position 14 is displayed. ---let $assert_cond="[SHOW BINLOG EVENTS FROM 14 LIMIT 1, Pos, 1]" = 127 +--let $assert_text=Verify that the starting offset (128) of an event after the invalid position 14 is displayed. +--let $assert_cond="[SHOW BINLOG EVENTS FROM 14 LIMIT 1, Pos, 1]" = 128 --source include/assert.inc ---let $assert_text=Verify that the starting offset (158) of an event after the invalid position 127 is displayed. ---let $assert_cond="[SHOW BINLOG EVENTS FROM 128 LIMIT 1, Pos, 1]" = 158 +--let $assert_text=Verify that the starting offset (159) of an event after the invalid position 129 is displayed. +--let $assert_cond="[SHOW BINLOG EVENTS FROM 129 LIMIT 1, Pos, 1]" = 159 --source include/assert.inc ---let $assert_text=Verify that the starting offset (158) of an event at the valid position 157 is displayed. ---let $assert_cond="[SHOW BINLOG EVENTS FROM 158 LIMIT 1, Pos, 1]" = 158 +--let $assert_text=Verify that the starting offset (159) of an event at the valid position 159 is displayed. +--let $assert_cond="[SHOW BINLOG EVENTS FROM 159 LIMIT 1, Pos, 1]" = 159 --source include/assert.inc DROP TABLE t1; diff --git a/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result b/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result index 6e56ae72f930..4a43d7ab5bb3 100644 --- a/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result +++ b/mysql-test/suite/binlog_nogtid/r/binlog_persist_only_variables.result @@ -47,7 +47,7 @@ INSERT INTO aliases(name) VALUES ('slave_parallel_workers'), ('slave_pending_jobs_size_max'), ('pseudo_slave_mode'), ('skip_slave_start'); -include/assert.inc [Expect 111 variables in the table.] +include/assert.inc [Expect 113 variables in the table.] # Test SET PERSIST_ONLY SET PERSIST_ONLY binlog_cache_size = @@GLOBAL.binlog_cache_size; @@ -64,6 +64,8 @@ Warning 1287 '@@binlog_format' is deprecated and will be removed in a future rel SET PERSIST_ONLY binlog_group_commit_sync_delay = @@GLOBAL.binlog_group_commit_sync_delay; SET PERSIST_ONLY binlog_group_commit_sync_no_delay_count = @@GLOBAL.binlog_group_commit_sync_no_delay_count; SET PERSIST_ONLY binlog_gtid_simple_recovery = @@GLOBAL.binlog_gtid_simple_recovery; +SET PERSIST_ONLY binlog_large_transaction_optimization_enabled = @@GLOBAL.binlog_large_transaction_optimization_enabled; +SET PERSIST_ONLY binlog_large_transaction_optimization_threshold = @@GLOBAL.binlog_large_transaction_optimization_threshold; SET PERSIST_ONLY binlog_max_flush_queue_time = @@GLOBAL.binlog_max_flush_queue_time; Warnings: Warning 1287 '@@binlog_max_flush_queue_time' is deprecated and will be removed in a future release. @@ -258,16 +260,16 @@ Warning 1287 '@@sync_relay_log_info' is deprecated and will be removed in a futu Warning 1287 '@@sync_relay_log_info' is deprecated and will be removed in a future release. SET PERSIST_ONLY sync_source_info = @@GLOBAL.sync_source_info; -include/assert.inc [Expect 100 persisted variables in persisted_variables table.] +include/assert.inc [Expect 102 persisted variables in persisted_variables table.] ############################################################ # 2. Restart server, it must preserve the persisted variable # settings. Verify persisted configuration. # restart -include/assert.inc [Expect 100 persisted variables in persisted_variables table.] -include/assert.inc [Expect 100 persisted variables shown as PERSISTED in variables_info table.] -include/assert.inc [Expect 100 persisted variables with matching persisted and global values.] +include/assert.inc [Expect 102 persisted variables in persisted_variables table.] +include/assert.inc [Expect 102 persisted variables shown as PERSISTED in variables_info table.] +include/assert.inc [Expect 102 persisted variables with matching persisted and global values.] ############################################################ # 3. Test RESET PERSIST. Verify persisted variable settings @@ -283,6 +285,8 @@ RESET PERSIST binlog_format; RESET PERSIST binlog_group_commit_sync_delay; RESET PERSIST binlog_group_commit_sync_no_delay_count; RESET PERSIST binlog_gtid_simple_recovery; +RESET PERSIST binlog_large_transaction_optimization_enabled; +RESET PERSIST binlog_large_transaction_optimization_threshold; RESET PERSIST binlog_max_flush_queue_time; RESET PERSIST binlog_order_commits; RESET PERSIST binlog_rotate_encryption_master_key_at_startup; diff --git a/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result b/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result index 83f721409cad..2cbe35bc5654 100644 --- a/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result +++ b/mysql-test/suite/binlog_nogtid/r/binlog_persist_variables.result @@ -26,7 +26,7 @@ VARIABLE_NAME LIKE '%source%') AND 'innodb_master_thread_disabled_debug', 'innodb_replication_delay')) ORDER BY VARIABLE_NAME; -include/assert.inc [Expect 111 variables in the table.] +include/assert.inc [Expect 113 variables in the table.] # Test SET PERSIST SET PERSIST binlog_cache_size = @@GLOBAL.binlog_cache_size; @@ -44,6 +44,8 @@ SET PERSIST binlog_group_commit_sync_delay = @@GLOBAL.binlog_group_commit_sync_d SET PERSIST binlog_group_commit_sync_no_delay_count = @@GLOBAL.binlog_group_commit_sync_no_delay_count; SET PERSIST binlog_gtid_simple_recovery = @@GLOBAL.binlog_gtid_simple_recovery; ERROR HY000: Variable 'binlog_gtid_simple_recovery' is a read only variable +SET PERSIST binlog_large_transaction_optimization_enabled = @@GLOBAL.binlog_large_transaction_optimization_enabled; +SET PERSIST binlog_large_transaction_optimization_threshold = @@GLOBAL.binlog_large_transaction_optimization_threshold; SET PERSIST binlog_max_flush_queue_time = @@GLOBAL.binlog_max_flush_queue_time; Warnings: Warning 1287 '@@binlog_max_flush_queue_time' is deprecated and will be removed in a future release. @@ -235,16 +237,16 @@ Warning 1287 '@@sync_relay_log_info' is deprecated and will be removed in a futu Warning 1287 '@@sync_relay_log_info' is deprecated and will be removed in a future release. SET PERSIST sync_source_info = @@GLOBAL.sync_source_info; -include/assert.inc [Expect 89 persisted variables in persisted_variables table.] +include/assert.inc [Expect 91 persisted variables in persisted_variables table.] ############################################################ # 2. Restart server, it must preserve the persisted variable # settings. Verify persisted configuration. # restart -include/assert.inc [Expect 89 persisted variables in persisted_variables table.'] -include/assert.inc [Expect 89 persisted variables shown as PERSISTED in variables_info table.'] -include/assert.inc [Expect 89 persisted variables with matching persisted and global values.] +include/assert.inc [Expect 91 persisted variables in persisted_variables table.'] +include/assert.inc [Expect 91 persisted variables shown as PERSISTED in variables_info table.'] +include/assert.inc [Expect 91 persisted variables with matching persisted and global values.] ############################################################ # 3. Test RESET PERSIST IF EXISTS. Verify persisted variable @@ -262,6 +264,8 @@ RESET PERSIST IF EXISTS binlog_group_commit_sync_no_delay_count; RESET PERSIST IF EXISTS binlog_gtid_simple_recovery; Warnings: Warning 3615 Variable binlog_gtid_simple_recovery does not exist in persisted config file +RESET PERSIST IF EXISTS binlog_large_transaction_optimization_enabled; +RESET PERSIST IF EXISTS binlog_large_transaction_optimization_threshold; RESET PERSIST IF EXISTS binlog_max_flush_queue_time; RESET PERSIST IF EXISTS binlog_order_commits; RESET PERSIST IF EXISTS binlog_rotate_encryption_master_key_at_startup; diff --git a/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test b/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test index 3207e6bd812e..b2e5ec07912a 100644 --- a/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test +++ b/mysql-test/suite/binlog_nogtid/t/binlog_persist_only_variables.test @@ -84,7 +84,7 @@ INSERT INTO aliases(name) VALUES # If this count differs, it means a variable has been added or removed. # In that case, this testcase needs to be updated accordingly. --echo ---let $expected = 111 +--let $expected = 113 --let $assert_text = Expect $expected variables in the table. --let $assert_cond = [SELECT COUNT(*) as count FROM rplvars, count, 1] = $expected --source include/assert.inc @@ -116,7 +116,7 @@ while ( $varid <= $countvars ) } --echo ---let $expected = 100 +--let $expected = 102 --let $assert_text = Expect $expected persisted variables in persisted_variables table. --let $assert_cond = [SELECT COUNT(*) as count FROM performance_schema.persisted_variables, count, 1] = $expected --source include/assert.inc diff --git a/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test b/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test index d510a5f20cde..2cf726d51a06 100644 --- a/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test +++ b/mysql-test/suite/binlog_nogtid/t/binlog_persist_variables.test @@ -62,7 +62,7 @@ INSERT INTO rplvars (varname, varvalue) # If this count differs, it means a variable has been added or removed. # In that case, this testcase needs to be updated accordingly. --echo ---let $expected = 111 +--let $expected = 113 --let $assert_text = Expect $expected variables in the table. --let $assert_cond = [SELECT COUNT(*) as count FROM rplvars, count, 1] = $expected --source include/assert.inc @@ -85,7 +85,7 @@ while ( $varid <= $countvars ) } --echo ---let $expected = 89 +--let $expected = 91 --let $assert_text = Expect $expected persisted variables in persisted_variables table. --let $assert_cond = [SELECT COUNT(*) as count FROM performance_schema.persisted_variables, count, 1] = $expected --source include/assert.inc diff --git a/mysql-test/suite/rpl/r/binlog_bolt_replica.result b/mysql-test/suite/rpl/r/binlog_bolt_replica.result new file mode 100644 index 000000000000..e7ea3eb7ef82 --- /dev/null +++ b/mysql-test/suite/rpl/r/binlog_bolt_replica.result @@ -0,0 +1,37 @@ +include/rpl/init_source_replica.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. +[connection master] +Warnings: +Note 1051 Unknown table 'test.t1' +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_transaction_compression = OFF; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +[connection slave] +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +include/rpl/start_receiver.inc +[connection master] +# Write a qualifying promoted transaction on the source. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +include/assert.inc [The source transaction was promoted before replication.] +include/rpl/sync_to_replica_received.inc +[connection slave] +include/rpl/start_applier.inc +[connection master] +include/rpl/sync_to_replica.inc +[connection slave] +include/assert.inc [Replication applies every row from the source BOLT transaction.] +include/assert.inc [Replication preserves the full large-transaction payload.] +include/assert.inc [The replica SQL applier promotes the qualifying transaction once.] +include/assert.inc [Replica-local promotion opens a new active binary log.] +[connection master] +DROP TABLE t1; +[connection slave] +include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/r/binlog_bolt_replica_commit_order.result b/mysql-test/suite/rpl/r/binlog_bolt_replica_commit_order.result new file mode 100644 index 000000000000..0ff8c6657d37 --- /dev/null +++ b/mysql-test/suite/rpl/r/binlog_bolt_replica_commit_order.result @@ -0,0 +1,35 @@ +include/rpl/init_source_replica.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. +[connection master] +[connection master] +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +CREATE TABLE t1 (a INT, payload LONGBLOB) ENGINE = InnoDB; +include/rpl/sync_to_replica.inc +[connection slave] +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +# +# Verify the replica logs promoted transactions in source order +# +[connection master] +SET @save_debug = @@GLOBAL.debug; +SET GLOBAL debug = "+d,set_commit_parent_100"; +[connection slave] +LOCK TABLE t1 WRITE; +[connection master] +include/rpl/save_server_position.inc +[connection slave] +UNLOCK TABLES; +include/rpl/sync_with_saved.inc +include/assert.inc [R2: the replica applier promoted at least one transaction, so the order check is meaningful.] +include/assert.inc [R3: every row arrived on the replica.] +include/assert.inc [R3: the promoted transactions kept their full payload.] +GTID sequence numbers in replica binlog order: 1 2 3 4 5 6 7 +[connection master] +SET GLOBAL debug = @save_debug; +DROP TABLE t1; +include/rpl/sync_to_replica.inc +[connection slave] +include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/r/binlog_bolt_replica_filter.result b/mysql-test/suite/rpl/r/binlog_bolt_replica_filter.result new file mode 100644 index 000000000000..7c5db1bf942b --- /dev/null +++ b/mysql-test/suite/rpl/r/binlog_bolt_replica_filter.result @@ -0,0 +1,50 @@ +include/rpl/init_source_replica.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. +[connection master] +[connection master] +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE DATABASE bolt_allowed; +CREATE DATABASE bolt_filtered; +CREATE TABLE bolt_allowed.t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE bolt_filtered.t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +[connection slave] +include/rpl/start_replica.inc +[connection master] +include/rpl/sync_to_replica.inc +[connection slave] +include/rpl/stop_replica.inc +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CHANGE REPLICATION FILTER REPLICATE_IGNORE_DB = (bolt_filtered);; +include/rpl/start_receiver.inc +[connection master] +# Commit a large allowed subset together with filtered rows. +BEGIN; +INSERT INTO bolt_allowed.t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO bolt_allowed.t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO bolt_allowed.t1 VALUES (3, REPEAT('c', 4000000)); +INSERT INTO bolt_filtered.t1 VALUES (1, 'filtered'); +COMMIT; +include/rpl/sync_to_replica_received.inc +[connection slave] +include/rpl/start_applier.inc +[connection master] +include/rpl/sync_to_replica.inc +[connection slave] +include/assert.inc [The allowed database retains every large transaction row.] +include/assert.inc [The allowed database preserves the full large payload.] +include/assert.inc [The filtered database receives no rows from the transaction.] +include/assert.inc [The replica promotes the retained qualifying subset once.] +include/assert.inc [Replica-local BOLT opens a new active binary log after filtering.] +include/rpl/stop_replica.inc +CHANGE REPLICATION FILTER REPLICATE_IGNORE_DB = (); +include/rpl/start_replica.inc +[connection master] +DROP DATABASE bolt_filtered; +DROP DATABASE bolt_allowed; +include/rpl/sync_to_replica.inc +[connection slave] +include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/r/binlog_bolt_replica_until.result b/mysql-test/suite/rpl/r/binlog_bolt_replica_until.result new file mode 100644 index 000000000000..8ac2b660bb38 --- /dev/null +++ b/mysql-test/suite/rpl/r/binlog_bolt_replica_until.result @@ -0,0 +1,42 @@ +include/rpl/init_source_replica.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. +[connection master] +[connection master] +Warnings: +Note 1051 Unknown table 'test.t1' +Warnings: +Note 1051 Unknown table 'test.t2' +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE t2 (id INT PRIMARY KEY) ENGINE=InnoDB; +[connection slave] +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +include/rpl/start_receiver.inc +[connection master] +# Commit the BOLT transaction, record its end position, then add a marker. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +INSERT INTO t2 VALUES (1); +include/rpl/sync_to_replica_received.inc +[connection slave] +START REPLICA SQL_THREAD UNTIL SOURCE_LOG_FILE='SOURCE_LOG_FILE', SOURCE_LOG_POS=SOURCE_LOG_POS;; +include/rpl/wait_for_applier_to_stop.inc +include/assert.inc [START REPLICA UNTIL applies every BOLT transaction row.] +include/assert.inc [START REPLICA UNTIL stops before the later marker transaction.] +include/assert.inc [The replica SQL applier promotes the transaction once before stopping.] +include/assert.inc [Replica-local promotion opens a new active binary log before stopping.] +include/rpl/assert_replica_status.inc [Exec_Source_Log_Pos] +include/rpl/start_applier.inc +[connection master] +DROP TABLE t2; +DROP TABLE t1; +include/rpl/sync_to_replica.inc +[connection slave] +include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/r/binlog_bolt_semisync.result b/mysql-test/suite/rpl/r/binlog_bolt_semisync.result new file mode 100644 index 000000000000..03ec61cd76a2 --- /dev/null +++ b/mysql-test/suite/rpl/r/binlog_bolt_semisync.result @@ -0,0 +1,89 @@ +include/rpl/init_source_replica.inc +Warnings: +Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. +Note #### Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. +[connection master] +[connection master] +CALL mtr.add_suppression("Semi-sync source failed on net_flush.*"); +include/rpl/install_semisync_source.inc +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_transaction_compression = OFF; +SET GLOBAL rpl_semi_sync_source_timeout = 600000; +[connection slave] +CALL mtr.add_suppression("Semi-sync replica net_flush.*"); +include/rpl/install_semisync_replica.inc +[connection master] +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +include/rpl/sync_to_replica.inc +[connection master] +# A promoted transaction is acknowledged at the AFTER_SYNC wait point. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +include/assert.inc [rpl_semi_sync_source_yes_tx should be 1 + 1] +include/assert.inc [rpl_semi_sync_source_no_tx should be 0 + 0] +include/assert.inc [The AFTER_SYNC transaction took the BOLT commit path.] +include/assert.inc [The AFTER_SYNC promotion opens a new active binary log.] +include/rpl/sync_to_replica.inc +[connection slave] +include/assert.inc [The replica applies every row of the AFTER_SYNC promotion.] +include/assert.inc [The replica receives the full AFTER_SYNC payload.] +[connection master] +include/rpl/sync_to_replica.inc +[connection master] +SET GLOBAL rpl_semi_sync_source_wait_point = AFTER_COMMIT; +# A promoted transaction is acknowledged at the AFTER_COMMIT wait point. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('d', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('f', 4000000)); +COMMIT; +include/assert.inc [rpl_semi_sync_source_yes_tx should be 3 + 1] +include/assert.inc [rpl_semi_sync_source_no_tx should be 0 + 0] +include/assert.inc [The AFTER_COMMIT transaction took the BOLT commit path.] +include/assert.inc [The AFTER_COMMIT promotion opens a new active binary log.] +include/rpl/sync_to_replica.inc +[connection slave] +include/assert.inc [The replica applies every row of the AFTER_COMMIT promotion.] +[connection master] +SET GLOBAL rpl_semi_sync_source_wait_point = AFTER_SYNC; +[connection master] +include/assert.inc [Semisync master is on] +include/assert.inc [The replica is still attached as a semisync client.] +[connection slave] +include/assert.inc [Semisync is still enabled on the replica.] +[connection master] +include/rpl/sync_to_replica.inc +[connection master] +[connection master] +# The after_sync hook runs before the promoted transaction is visible. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('g', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('h', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('i', 4000000)); +SET DEBUG_SYNC = 'after_call_after_sync_observer SIGNAL bolt_in_after_sync WAIT_FOR bolt_observer_done'; +COMMIT; +SET DEBUG_SYNC = 'now WAIT_FOR bolt_in_after_sync'; +include/assert.inc [The promoted transaction is not yet committed while the after_sync hook runs.] +SET DEBUG_SYNC = 'now SIGNAL bolt_observer_done'; +[connection master] +SET DEBUG_SYNC = 'RESET'; +include/assert.inc [The after_sync ordering case took the BOLT commit path.] +include/assert.inc [The promoted transaction is committed once the hook returns.] +[connection master] +include/rpl/sync_to_replica.inc +[connection master] +DROP TABLE t1; +include/rpl/sync_to_replica.inc +[connection slave] +include/rpl/stop_replica.inc +include/rpl/uninstall_semisync_replica.inc +Warnings: +Note 3084 Replication thread(s) for channel '' are already stopped. +include/rpl/start_replica.inc +[connection master] +include/rpl/uninstall_semisync_source.inc +include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/r/rpl_trx_boundary_parser_warning.result b/mysql-test/suite/rpl/r/rpl_trx_boundary_parser_warning.result index 5921dd2d0e3f..045b1c9da9e3 100644 --- a/mysql-test/suite/rpl/r/rpl_trx_boundary_parser_warning.result +++ b/mysql-test/suite/rpl/r/rpl_trx_boundary_parser_warning.result @@ -169,7 +169,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Gtid # # SET @@SESSION.GTID_NEXT= 'Gtid_set' include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 496, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 497, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#8): @@ -194,7 +194,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # User var # # @`var`=10 include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 496, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 497, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#8): @@ -219,7 +219,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Query # # BEGIN include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 700, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 701, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#10): @@ -244,7 +244,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # User var # # @`var`=10 include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 779, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 780, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#11): @@ -269,7 +269,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Query # # BEGIN include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 779, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 780, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#11): @@ -294,7 +294,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Query # # use `test`; CREATE TABLE t1 (c1 INT) ENGINE= InnoDB include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 965, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 966, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#13): @@ -319,7 +319,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Query # # use `test`; CREATE TABLE t1 (c1 INT) ENGINE= InnoDB include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1456, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1457, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#19): @@ -344,7 +344,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Gtid # # SET @@SESSION.GTID_NEXT= 'Gtid_set' include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 965, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 966, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#13): @@ -369,7 +369,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Gtid # # SET @@SESSION.GTID_NEXT= 'Gtid_set' include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1456, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1457, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#19): @@ -394,7 +394,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # User var # # @`var`=10 include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 965, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 966, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#13): @@ -419,7 +419,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # User var # # @`var`=10 include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1456, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 1457, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#19): @@ -444,7 +444,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # User var # # @`var`=10 include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 2045, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 2046, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#26): @@ -469,7 +469,7 @@ Log_name Pos Event_type Server_id End_log_pos Info slave-relay-bin.000003 # Query # # use `test`; CREATE TABLE t1 (c1 INT) ENGINE= InnoDB include/rpl/stop_server.inc [server_number=2] include/rpl/start_server.inc [server_number=2 parameters: --debug=d,dbug_disable_relay_log_truncation] -CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 2045, RELAY_LOG_FILE = 'slave-relay-bin.000001'; +CHANGE REPLICATION SOURCE TO SOURCE_LOG_POS = 2046, RELAY_LOG_FILE = 'slave-relay-bin.000001'; include/rpl/start_receiver.inc include/rpl/sync_to_replica_received.inc # Restarted queuing the following event (#26): diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica-slave.opt b/mysql-test/suite/rpl/t/binlog_bolt_replica-slave.opt new file mode 100644 index 000000000000..4ed6f297d980 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica-slave.opt @@ -0,0 +1,2 @@ +--log-replica-updates +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica.test b/mysql-test/suite/rpl/t/binlog_bolt_replica.test new file mode 100644 index 000000000000..bc38b6ccd3e5 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica.test @@ -0,0 +1,102 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify a replica re-logging a qualifying source BOLT transaction also uses +# BOLT for its local binary log. +# +# === Requirements === +# R1. A source BOLT transaction replicates successfully to the replica. +# R2. With log_replica_updates enabled, the replica SQL applier promotes its +# local binlog cache for the qualifying ROW transaction. +# +# === Implementation === +# 1. Initialize source/replica replication, BOLT, and a clean source table. +# 2. Commit a qualifying BOLT transaction on the source and wait until the +# replica receiver has it available. +# 3. Enable BOLT before starting the replica applier and verify its local +# promotion and promoted-file header. +# +--source include/have_binlog_format_row.inc +--source include/not_binlog_transaction_compression_on.inc +--let $rpl_skip_start_slave = 1 +--source include/rpl/init_source_replica.inc + +--let $source_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $source_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $source_save_compression = query_get_value(SELECT @@SESSION.binlog_transaction_compression, @@SESSION.binlog_transaction_compression, 1) + +# Setup +--disable_query_log +DROP TABLE IF EXISTS t1; +--enable_query_log +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_transaction_compression = OFF; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +--source include/rpl/connection_replica.inc +--let $replica_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $replica_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--let $replica_bolt_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source include/rpl/start_receiver.inc +--source include/rpl/connection_source.inc + +--echo # Write a qualifying promoted transaction on the source. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +--let $source_optimized_count = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = The source transaction was promoted before replication. +--let $assert_cond = $source_optimized_count > 0 +--source include/assert.inc + +--source include/rpl/sync_to_replica_received.inc + +--source include/rpl/connection_replica.inc +--source include/rpl/start_applier.inc +--source include/rpl/connection_source.inc +--source include/rpl/sync_to_replica.inc + +--source include/rpl/connection_replica.inc +--let $replica_rows = query_get_value(SELECT COUNT(*) FROM t1, COUNT(*), 1) +--let $replica_payload_bytes = query_get_value(SELECT SUM(OCTET_LENGTH(data)) FROM t1, SUM(OCTET_LENGTH(data)), 1) +--let $replica_bolt_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = Replication applies every row from the source BOLT transaction. +--let $assert_cond = $replica_rows = 3 +--source include/assert.inc +--let $assert_text = Replication preserves the full large-transaction payload. +--let $assert_cond = $replica_payload_bytes = 12000000 +--source include/assert.inc +--let $assert_text = The replica SQL applier promotes the qualifying transaction once. +--let $assert_cond = $replica_bolt_after = $replica_bolt_before + 1 +--source include/assert.inc +--let $assert_text = Replica-local promotion opens a new active binary log. +--let $assert_cond = "$replica_binlog_after" != "$replica_binlog_before" +--source include/assert.inc +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $bolt_header_file = $replica_binlog_after +--source ../../binlog/inc/validate_bolt_header.inc + +--source include/rpl/connection_source.inc +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $source_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $source_save_threshold +--eval SET SESSION binlog_transaction_compression = $source_save_compression +--enable_query_log +--source include/rpl/connection_replica.inc +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $replica_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $replica_save_threshold +--disable_warnings +DROP TABLE IF EXISTS t1; +--enable_warnings +--enable_query_log +--let $rpl_skip_sync = 1 +--source include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_commit_order-master.opt b/mysql-test/suite/rpl/t/binlog_bolt_replica_commit_order-master.opt new file mode 100644 index 000000000000..17b2a1d3e042 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_commit_order-master.opt @@ -0,0 +1,5 @@ +--gtid-mode=ON +--enforce-gtid-consistency=ON +--binlog-transaction-compression=OFF +--binlog-cache-size=1M +--max-binlog-size=1G diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_commit_order-slave.opt b/mysql-test/suite/rpl/t/binlog_bolt_replica_commit_order-slave.opt new file mode 100644 index 000000000000..4f1135204bcc --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_commit_order-slave.opt @@ -0,0 +1,9 @@ +--gtid-mode=ON +--enforce-gtid-consistency=ON +--log-replica-updates +--replica-preserve-commit-order=ON +--replica-parallel-workers=6 +--replica-transaction-retries=0 +--binlog-transaction-compression=OFF +--binlog-cache-size=1M +--max-binlog-size=1G diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_commit_order.test b/mysql-test/suite/rpl/t/binlog_bolt_replica_commit_order.test new file mode 100644 index 000000000000..e5b38cdb3092 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_commit_order.test @@ -0,0 +1,197 @@ +# === Purpose === +# Verify that a replica applier committing through the BOLT path still honours +# replica_preserve_commit_order, so the replica's own binary log lists +# transactions in the source's order. +# +# The optimized commit path bypasses the group-commit stages, and with them the +# two calls that enforce commit order: Commit_order_manager::wait() before the +# flush stage, and Commit_order_manager::finish_one() once the session is +# enqueued. Nothing downstream compensates -- finish_commit() has no +# commit-order handling, and ha_commit_low()'s hook is gated by +# is_ha_commit_low_invoking_commit_order(), which is never true for a +# transaction that logged something. So without those calls a worker applying a +# large transaction can write its local binary log out of turn. +# +# === Requirements === +# R1. With replica_preserve_commit_order=ON, log_replica_updates=ON and several +# applier workers, the replica's binary log records transactions in source +# order even when the larger ones commit through the BOLT path. +# R2. At least one transaction really did take the BOLT path on the replica, so +# that R1 is not vacuously satisfied. +# R3. The promoted transactions keep their full contents. +# +# === Implementation === +# GTID mode is ON, so every transaction the replica applies keeps the GTID it +# was assigned on the source. The source is the only thing generating GTIDs +# here, so its sequence numbers are exactly the source's commit order. Reading +# the replica's binary logs front to back and collecting those sequence numbers +# therefore yields the order in which the replica logged them, and the order +# check is simply that the numbers ascend. +# +# All of the replica's binary logs are read, not just the current one: promotion +# *is* a rotation, since the spilled file becomes the next binary log file, so a +# run with promotions spreads the transactions over several files and the newest +# file holds only the last promoted transaction. +# +# The source keeps BOLT disabled, so its transactions are ordinary and are not +# marked as parallelization barriers -- a promoted transaction sets that flag, +# which would serialize the appliers and hide the very interleaving this test +# needs. The replica enables BOLT with a low threshold, so the large +# transactions are promoted locally, on the applier, which is the path under +# test. + +--source include/not_group_replication_plugin.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_replica_preserve_commit_order.inc +--source include/have_binlog_format_row.inc +--source include/not_binlog_transaction_compression_on.inc +--source include/rpl/init_source_replica.inc + +# A LONGBLOB payload sized so the replica's applier crosses the threshold, +# while the source, with BOLT disabled, logs the transaction the ordinary way. +# The threshold's minimum is 10 MiB (VALID_RANGE in sys_vars.cc), so the payload +# has to exceed that rather than the 1 MiB binlog_cache_size. +--let $bolt_payload_bytes = 12000000 + +--source include/rpl/connection_source.inc +--let $source_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $source_uuid = query_get_value(SELECT @@GLOBAL.server_uuid AS uuid, uuid, 1) + +# The source must NOT promote: a promoted transaction is written as a +# parallelization barrier, which would stop the replica applying neighbouring +# transactions concurrently. +SET GLOBAL binlog_large_transaction_optimization_enabled = OFF; +CREATE TABLE t1 (a INT, payload LONGBLOB) ENGINE = InnoDB; +--source include/rpl/sync_to_replica.inc + +--source include/rpl/connection_replica.inc +--let $replica_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $replica_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +# 10 MiB is the lowest value the server accepts, so the payload above has to be +# larger than that to be promoted on the applier. +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--let $replica_bolt_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) + +--echo # +--echo # Verify the replica logs promoted transactions in source order +--echo # +--source include/rpl/connection_source.inc +# Give the following transactions the same commit parent so the replica may +# apply them in parallel. +SET @save_debug = @@GLOBAL.debug; +SET GLOBAL debug = "+d,set_commit_parent_100"; + +# Six transactions, alternating small and large. The large ones exceed the +# replica's threshold and take the BOLT path there. +# Block the appliers so that all six transactions are queued, and therefore all +# registered with the commit-order manager, before any of them commits. +--source include/rpl/connection_replica.inc +LOCK TABLE t1 WRITE; +--source include/rpl/connection_source.inc + +--let $value = 1 +while ($value <= 6) +{ + if (`SELECT $value % 2 = 0`) + { + --disable_query_log + --eval INSERT INTO t1(a, payload) VALUES ($value, REPEAT('x', $bolt_payload_bytes)) + --enable_query_log + } + if (`SELECT $value % 2 = 1`) + { + --disable_query_log + --eval INSERT INTO t1(a, payload) VALUES ($value, NULL) + --enable_query_log + } + --inc $value +} +--source include/rpl/save_server_position.inc + +--source include/rpl/connection_replica.inc +# Wait until every applier is blocked on the metadata lock, which means all six +# transactions are registered in the commit-order queue. +let $wait_condition = SELECT count(*) = 6 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE State = 'Waiting for table metadata lock'; +--source include/wait_condition.inc +UNLOCK TABLES; +--source include/rpl/sync_with_saved.inc + +--let $replica_bolt_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = R2: the replica applier promoted at least one transaction, so the order check is meaningful. +--let $assert_cond = $replica_bolt_after > $replica_bolt_before +--source include/assert.inc + +--let $assert_text = R3: every row arrived on the replica. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 6 +--source include/assert.inc +--let $assert_text = R3: the promoted transactions kept their full payload. +--let $assert_cond = [SELECT SUM(OCTET_LENGTH(payload)) AS total FROM t1, total, 1] = 36000000 +--source include/assert.inc + +# +# R1. Decode every one of the replica's binary logs, in order, and check that +# the source's GTID sequence numbers ascend. The shell glob supplies the files +# in name order, which is also their creation order because the numeric suffix +# is zero padded, and mysqlbinlog concatenates them in the order given. +# +# The awk program avoids `$` field syntax because mysqltest expands it, and +# reads with getline for the same reason. +# +--let $SLAVE_MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $bolt_order_dump = $MYSQL_TMP_DIR/bolt_replica_commit_order_dump.txt +--let $bolt_order_script = $MYSQL_TMP_DIR/bolt_replica_commit_order.awk +--exec $MYSQL_BINLOG --force-if-open $SLAVE_MYSQLD_DATADIR/slave-bin.0* > $bolt_order_dump + +--write_file $bolt_order_script END_OF_SCRIPT +# Collect the sequence numbers of the source's GTIDs in the order the replica +# logged them, and fail if any of them is not larger than the one before it. +BEGIN { + previous = 0 + count = 0 + observed = "" + while ((getline line) > 0) { + if (line !~ /SET @@SESSION.GTID_NEXT= /) continue + value = line + sub(/^.*GTID_NEXT= '/, "", value) + sub(/'.*/, "", value) + split(value, parts, ":") + if (parts[1] != uuid) continue + sequence = parts[2] + 0 + count = count + 1 + if (sequence <= previous) { + printf "GTID %d logged after %d: the replica logged out of source order\n", sequence, previous + exit 1 + } + previous = sequence + observed = observed " " sequence + } + # The six test transactions plus the CREATE TABLE must all be present. A + # smaller count means the files were not all read, which would make the + # ascending check vacuous. + if (count < 7) { + printf "only %d GTIDs found, expected at least 7\n", count + exit 1 + } + printf "GTID sequence numbers in replica binlog order:%s\n", observed +} +END_OF_SCRIPT +--exec awk -v uuid=$source_uuid -f $bolt_order_script $bolt_order_dump +--remove_file $bolt_order_dump +--remove_file $bolt_order_script + +# Cleanup +--source include/rpl/connection_source.inc +SET GLOBAL debug = @save_debug; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $source_save_enabled +--enable_query_log +DROP TABLE t1; +--source include/rpl/sync_to_replica.inc +--source include/rpl/connection_replica.inc +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $replica_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $replica_save_threshold +--enable_query_log +--source include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_filter-slave.opt b/mysql-test/suite/rpl/t/binlog_bolt_replica_filter-slave.opt new file mode 100644 index 000000000000..4ed6f297d980 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_filter-slave.opt @@ -0,0 +1,2 @@ +--log-replica-updates +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_filter.test b/mysql-test/suite/rpl/t/binlog_bolt_replica_filter.test new file mode 100644 index 000000000000..be96ae6387b4 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_filter.test @@ -0,0 +1,108 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify a replica filters part of a qualifying source transaction while +# promoting the retained large ROW subset into its local binary log. +# +# === Requirements === +# R1. Rows in the allowed database apply and trigger one local BOLT promotion. +# R2. Rows in the filtered database do not apply. +# +# === Implementation === +# Synchronize both schemas, stop the replica, configure REPLICATE_IGNORE_DB, +# then apply one transaction containing large allowed rows and filtered rows. + +--source include/have_binlog_format_row.inc +--source include/not_binlog_transaction_compression_on.inc +--let $rpl_skip_start_slave = 1 +--source include/rpl/init_source_replica.inc + +--source include/rpl/connection_source.inc +--let $source_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $source_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--disable_query_log +--disable_warnings +DROP DATABASE IF EXISTS bolt_allowed; +DROP DATABASE IF EXISTS bolt_filtered; +--enable_warnings +--enable_query_log +CREATE DATABASE bolt_allowed; +CREATE DATABASE bolt_filtered; +CREATE TABLE bolt_allowed.t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE bolt_filtered.t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; + +--source include/rpl/connection_replica.inc +--source include/rpl/start_replica.inc +--source include/rpl/connection_source.inc +--source include/rpl/sync_to_replica.inc +--source include/rpl/connection_replica.inc +--source include/rpl/stop_replica.inc +--let $replica_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $replica_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--eval CHANGE REPLICATION FILTER REPLICATE_IGNORE_DB = (bolt_filtered); +--let $replica_bolt_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source include/rpl/start_receiver.inc + +--source include/rpl/connection_source.inc +--echo # Commit a large allowed subset together with filtered rows. +BEGIN; +INSERT INTO bolt_allowed.t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO bolt_allowed.t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO bolt_allowed.t1 VALUES (3, REPEAT('c', 4000000)); +INSERT INTO bolt_filtered.t1 VALUES (1, 'filtered'); +COMMIT; +--source include/rpl/sync_to_replica_received.inc + +--source include/rpl/connection_replica.inc +--source include/rpl/start_applier.inc +--source include/rpl/connection_source.inc +--source include/rpl/sync_to_replica.inc + +--source include/rpl/connection_replica.inc +--let $allowed_rows = query_get_value(SELECT COUNT(*) FROM bolt_allowed.t1, COUNT(*), 1) +--let $allowed_payload = query_get_value(SELECT SUM(OCTET_LENGTH(data)) FROM bolt_allowed.t1, SUM(OCTET_LENGTH(data)), 1) +--let $filtered_rows = query_get_value(SELECT COUNT(*) FROM bolt_filtered.t1, COUNT(*), 1) +--let $replica_bolt_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = The allowed database retains every large transaction row. +--let $assert_cond = $allowed_rows = 3 +--source include/assert.inc +--let $assert_text = The allowed database preserves the full large payload. +--let $assert_cond = $allowed_payload = 12000000 +--source include/assert.inc +--let $assert_text = The filtered database receives no rows from the transaction. +--let $assert_cond = $filtered_rows = 0 +--source include/assert.inc +--let $assert_text = The replica promotes the retained qualifying subset once. +--let $assert_cond = $replica_bolt_after = $replica_bolt_before + 1 +--source include/assert.inc +--let $assert_text = Replica-local BOLT opens a new active binary log after filtering. +--let $assert_cond = "$replica_binlog_after" != "$replica_binlog_before" +--source include/assert.inc +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $bolt_header_file = $replica_binlog_after +--source ../../binlog/inc/validate_bolt_header.inc + +--source include/rpl/stop_replica.inc +CHANGE REPLICATION FILTER REPLICATE_IGNORE_DB = (); +--source include/rpl/start_replica.inc +--source include/rpl/connection_source.inc +DROP DATABASE bolt_filtered; +DROP DATABASE bolt_allowed; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $source_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $source_save_threshold +--enable_query_log +--source include/rpl/sync_to_replica.inc +--source include/rpl/connection_replica.inc +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $replica_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $replica_save_threshold +--enable_query_log +--let $rpl_skip_sync = 1 +--source include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_until-slave.opt b/mysql-test/suite/rpl/t/binlog_bolt_replica_until-slave.opt new file mode 100644 index 000000000000..4ed6f297d980 --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_until-slave.opt @@ -0,0 +1,2 @@ +--log-replica-updates +--binlog-transaction-compression=OFF diff --git a/mysql-test/suite/rpl/t/binlog_bolt_replica_until.test b/mysql-test/suite/rpl/t/binlog_bolt_replica_until.test new file mode 100644 index 000000000000..777001a8915b --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_replica_until.test @@ -0,0 +1,96 @@ +# BOLT stands for Binary-log Optimization for Large Transactions. +# +# === Purpose === +# Verify START REPLICA UNTIL stops after a BOLT-promoted source transaction +# while the replica SQL applier also promotes its local binary log. +# +# === Requirements === +# R1. Position UNTIL applies the qualifying transaction but not a later marker. +# R2. Applying the qualifying transaction creates one replica-local BOLT file. +# +# === Implementation === +# Stop the applier initially, receive a promoted source transaction and later +# marker, then apply only through the position immediately after BOLT commit. + +--source include/have_binlog_format_row.inc +--source include/have_mta.inc +--source include/not_binlog_transaction_compression_on.inc +--let $rpl_skip_start_slave = 1 +--source include/rpl/init_source_replica.inc + +--source include/rpl/connection_source.inc +--let $source_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $source_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--disable_query_log +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +--enable_query_log +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +CREATE TABLE t2 (id INT PRIMARY KEY) ENGINE=InnoDB; + +--source include/rpl/connection_replica.inc +--let $replica_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $replica_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +--let $replica_bolt_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source include/rpl/start_receiver.inc + +--source include/rpl/connection_source.inc +--echo # Commit the BOLT transaction, record its end position, then add a marker. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; +--let $until_source_file = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $until_source_pos = query_get_value(SHOW BINARY LOG STATUS, Position, 1) +INSERT INTO t2 VALUES (1); +--source include/rpl/sync_to_replica_received.inc + +--source include/rpl/connection_replica.inc +--replace_result $until_source_file SOURCE_LOG_FILE $until_source_pos SOURCE_LOG_POS +--eval START REPLICA SQL_THREAD UNTIL SOURCE_LOG_FILE='$until_source_file', SOURCE_LOG_POS=$until_source_pos; +--source include/rpl/wait_for_applier_to_stop.inc +--let $replica_rows = query_get_value(SELECT COUNT(*) FROM t1, COUNT(*), 1) +--let $replica_markers = query_get_value(SELECT COUNT(*) FROM t2, COUNT(*), 1) +--let $replica_bolt_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $replica_binlog_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = START REPLICA UNTIL applies every BOLT transaction row. +--let $assert_cond = $replica_rows = 3 +--source include/assert.inc +--let $assert_text = START REPLICA UNTIL stops before the later marker transaction. +--let $assert_cond = $replica_markers = 0 +--source include/assert.inc +--let $assert_text = The replica SQL applier promotes the transaction once before stopping. +--let $assert_cond = $replica_bolt_after = $replica_bolt_before + 1 +--source include/assert.inc +--let $assert_text = Replica-local promotion opens a new active binary log before stopping. +--let $assert_cond = "$replica_binlog_after" != "$replica_binlog_before" +--source include/assert.inc +--let $MYSQLD_DATADIR = `SELECT @@DATADIR` +--let $bolt_header_file = $replica_binlog_after +--source ../../binlog/inc/validate_bolt_header.inc +--let $slave_param = Exec_Source_Log_Pos +--let $slave_param_value = $until_source_pos +--source include/rpl/assert_replica_status.inc + +--source include/rpl/start_applier.inc +--source include/rpl/connection_source.inc +DROP TABLE t2; +DROP TABLE t1; +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $source_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $source_save_threshold +--enable_query_log +--source include/rpl/sync_to_replica.inc +--source include/rpl/connection_replica.inc +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $replica_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $replica_save_threshold +--enable_query_log +--let $rpl_skip_sync = 1 +--source include/rpl/deinit.inc diff --git a/mysql-test/suite/rpl/t/binlog_bolt_semisync-master.opt b/mysql-test/suite/rpl/t/binlog_bolt_semisync-master.opt new file mode 100644 index 000000000000..58029d28acec --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_semisync-master.opt @@ -0,0 +1 @@ +$SEMISYNC_PLUGIN_OPT diff --git a/mysql-test/suite/rpl/t/binlog_bolt_semisync-slave.opt b/mysql-test/suite/rpl/t/binlog_bolt_semisync-slave.opt new file mode 100644 index 000000000000..58029d28acec --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_semisync-slave.opt @@ -0,0 +1 @@ +$SEMISYNC_PLUGIN_OPT diff --git a/mysql-test/suite/rpl/t/binlog_bolt_semisync.test b/mysql-test/suite/rpl/t/binlog_bolt_semisync.test new file mode 100644 index 000000000000..a3218f85277b --- /dev/null +++ b/mysql-test/suite/rpl/t/binlog_bolt_semisync.test @@ -0,0 +1,264 @@ +# === Purpose === +# Verify that semisynchronous replication keeps its guarantee for transactions +# committed through BOLT. BOLT commits on its own path rather than through +# MYSQL_BIN_LOG::ordered_commit(), so it has to invoke the same replication +# hooks, in the same order, with the same locks held. If it did not, a promoted +# transaction would be acknowledged to the client without any replica having +# confirmed it, and the loss would be silent: nothing else reports it. +# +# === Requirements === +# R1. A promoted transaction is acknowledged by the replica before the source +# returns, at the default AFTER_SYNC wait point. +# R2. The same holds at the AFTER_COMMIT wait point. +# R3. Semisync is still active after promotions, so the transactions above were +# not counted by a source that had silently switched semisync off. +# R4. A promoted transaction is applied in full on the replica. +# R5. The after_flush hook runs before the promoted file's end position is +# published, so a dump thread cannot read the transaction's events before +# the source has registered that it owes an acknowledgement for them. +# +# Every case asserts Binlog_large_transaction_optimization_count moved, because +# a transaction that quietly fell back to the standard commit path would satisfy +# all the semisync assertions while testing none of this code. +# +# === Implementation === +# 1. Bring up one source and one replica and enable semisync on both. +# 2. Commit a 12 MiB transaction, above the 10 MiB threshold, so its cache +# spills and is promoted; check the yes_tx/no_tx counters moved as they would +# for any acknowledged transaction, and that promotion did happen. +# 3. Repeat at the AFTER_COMMIT wait point. +# 4. Use the after_call_after_sync_observer debug sync point to hold the source +# inside the after_sync hook and confirm from another session that the rows +# are not yet visible, which places the hook before the engine commit. +# +--source include/have_binlog_format_row.inc +--source include/not_binlog_transaction_compression_on.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/rpl/init_source_replica.inc +--source include/have_semisync_plugin.inc + +--let $bolt_rows = 3 +--let $bolt_payload_bytes = 12000000 + +############################################################################ +# Setup +############################################################################ +--source include/rpl/connection_source.inc +CALL mtr.add_suppression("Semi-sync source failed on net_flush.*"); +--source include/rpl/install_semisync_source.inc +--source include/suppress_tls_off.inc + +--let $source_save_enabled = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_enabled, @@GLOBAL.binlog_large_transaction_optimization_enabled, 1) +--let $source_save_threshold = query_get_value(SELECT @@GLOBAL.binlog_large_transaction_optimization_threshold, @@GLOBAL.binlog_large_transaction_optimization_threshold, 1) +--let $source_save_compression = query_get_value(SELECT @@SESSION.binlog_transaction_compression, @@SESSION.binlog_transaction_compression, 1) +--let $source_save_timeout = query_get_value(SELECT @@GLOBAL.rpl_semi_sync_source_timeout, @@GLOBAL.rpl_semi_sync_source_timeout, 1) +--let $source_save_wait_point = query_get_value(SELECT @@GLOBAL.rpl_semi_sync_source_wait_point, @@GLOBAL.rpl_semi_sync_source_wait_point, 1) + +SET GLOBAL binlog_large_transaction_optimization_enabled = ON; +SET GLOBAL binlog_large_transaction_optimization_threshold = 10485760; +SET SESSION binlog_transaction_compression = OFF; +# A 12 MiB transaction can take a while to reach the replica on a loaded test +# machine. Without this the wait can time out, which would turn an +# acknowledged transaction into a no_tx and fail the test for an unrelated +# reason. +SET GLOBAL rpl_semi_sync_source_timeout = 600000; + +--source include/rpl/connection_replica.inc +CALL mtr.add_suppression("Semi-sync replica net_flush.*"); +--source include/rpl/install_semisync_replica.inc +--source include/suppress_tls_off.inc + +--source include/rpl/connection_source.inc +# The replica registers as a semisync client asynchronously, so wait for it +# before measuring anything. +--source include/rpl/wait_for_semisync_source_status_on.inc + +CREATE TABLE t1 (id INT PRIMARY KEY, data LONGBLOB) ENGINE=InnoDB; +--source include/rpl/sync_to_replica.inc + +############################################################################ +# Case 1: Promotion at the AFTER_SYNC wait point (the default). +############################################################################ +--source include/rpl/connection_source.inc +--echo # A promoted transaction is acknowledged at the AFTER_SYNC wait point. +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source include/rpl/save_semisync_yesno_tx.inc + +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('a', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('b', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('c', 4000000)); +COMMIT; + +--let $semi_sync_yes_tx_increment = 1 +--let $semi_sync_no_tx_increment = 0 +--source include/rpl/assert_semisync_yesno_tx_increment.inc + +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = The AFTER_SYNC transaction took the BOLT commit path. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = The AFTER_SYNC promotion opens a new active binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc + +--source include/rpl/sync_to_replica.inc +--source include/rpl/connection_replica.inc +--let $assert_text = The replica applies every row of the AFTER_SYNC promotion. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = $bolt_rows +--source include/assert.inc +--let $assert_text = The replica receives the full AFTER_SYNC payload. +--let $assert_cond = [SELECT SUM(OCTET_LENGTH(data)) AS payload FROM t1, payload, 1] = $bolt_payload_bytes +--source include/assert.inc + +############################################################################ +# Case 2: Promotion at the AFTER_COMMIT wait point. +# +# BOLT does not call the after_commit hook itself; it reaches it through +# finish_commit(). This case is the regression check that the promoted path +# still gets there. +############################################################################ +--source include/rpl/connection_source.inc +--disable_query_log +TRUNCATE t1; +--enable_query_log +--source include/rpl/sync_to_replica.inc + +--source include/rpl/connection_source.inc +SET GLOBAL rpl_semi_sync_source_wait_point = AFTER_COMMIT; +--echo # A promoted transaction is acknowledged at the AFTER_COMMIT wait point. +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_before = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--source include/rpl/save_semisync_yesno_tx.inc + +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('d', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('e', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('f', 4000000)); +COMMIT; + +--let $semi_sync_yes_tx_increment = 1 +--let $semi_sync_no_tx_increment = 0 +--source include/rpl/assert_semisync_yesno_tx_increment.inc + +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $binlog_file_after = query_get_value(SHOW BINARY LOG STATUS, File, 1) +--let $assert_text = The AFTER_COMMIT transaction took the BOLT commit path. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = The AFTER_COMMIT promotion opens a new active binary log. +--let $assert_cond = "$binlog_file_after" != "$binlog_file_before" +--source include/assert.inc + +--source include/rpl/sync_to_replica.inc +--source include/rpl/connection_replica.inc +--let $assert_text = The replica applies every row of the AFTER_COMMIT promotion. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = $bolt_rows +--source include/assert.inc + +--source include/rpl/connection_source.inc +--eval SET GLOBAL rpl_semi_sync_source_wait_point = $source_save_wait_point + +############################################################################ +# Case 3: Semisync is still active on both servers. +# +# If the source had dropped out of semisync at some point, the counters above +# could have been satisfied without any replica involvement. +############################################################################ +--source include/rpl/connection_source.inc +--source include/assert_semisync_source_status_on.inc +--let $semisync_clients = query_get_value(SHOW STATUS LIKE 'Rpl_semi_sync_source_clients', Value, 1) +--let $assert_text = The replica is still attached as a semisync client. +--let $assert_cond = $semisync_clients = 1 +--source include/assert.inc + +--source include/rpl/connection_replica.inc +--let $semisync_replica_status = query_get_value(SHOW STATUS LIKE 'Rpl_semi_sync_replica_status', Value, 1) +--let $assert_text = Semisync is still enabled on the replica. +--let $assert_cond = "$semisync_replica_status" = "ON" +--source include/assert.inc + +############################################################################ +# Case 4: The after_sync hook runs before the engine commit. +# +# Hold the committing session inside the after_sync hook, at the +# after_call_after_sync_observer sync point in Binlog_storage_delegate, and +# check from a second session that the transaction's rows are not yet visible. +# That is what makes the wait meaningful: at AFTER_SYNC the source must not +# have committed in the engine while it is still waiting for the replica. +############################################################################ +--source include/rpl/connection_source.inc +--disable_query_log +TRUNCATE t1; +--enable_query_log +--source include/rpl/sync_to_replica.inc + +--source include/rpl/connection_source.inc +--let $bolt_count_before = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +connect (bolt_observer, localhost, root,,); +--source include/rpl/connection_source.inc + +--echo # The after_sync hook runs before the promoted transaction is visible. +# Only the COMMIT blocks, so only it needs to be sent asynchronously. +BEGIN; +INSERT INTO t1 VALUES (1, REPEAT('g', 4000000)); +INSERT INTO t1 VALUES (2, REPEAT('h', 4000000)); +INSERT INTO t1 VALUES (3, REPEAT('i', 4000000)); +SET DEBUG_SYNC = 'after_call_after_sync_observer SIGNAL bolt_in_after_sync WAIT_FOR bolt_observer_done'; +--send COMMIT + +connection bolt_observer; +SET DEBUG_SYNC = 'now WAIT_FOR bolt_in_after_sync'; +# The committing session has been through the after_sync hook, so the replica +# has acknowledged, but finish_commit() has not run yet, so nothing is visible. +--let $assert_text = The promoted transaction is not yet committed while the after_sync hook runs. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = 0 +--source include/assert.inc +SET DEBUG_SYNC = 'now SIGNAL bolt_observer_done'; + +--source include/rpl/connection_source.inc +--reap +SET DEBUG_SYNC = 'RESET'; +--let $bolt_count_after = query_get_value(SHOW GLOBAL STATUS LIKE 'Binlog_large_transaction_optimization_count', Value, 1) +--let $assert_text = The after_sync ordering case took the BOLT commit path. +--let $assert_cond = $bolt_count_after = $bolt_count_before + 1 +--source include/assert.inc +--let $assert_text = The promoted transaction is committed once the hook returns. +--let $assert_cond = [SELECT COUNT(*) AS count FROM t1, count, 1] = $bolt_rows +--source include/assert.inc + +connection bolt_observer; +disconnect bolt_observer; +--source include/rpl/connection_source.inc +--source include/rpl/sync_to_replica.inc + +############################################################################ +# Cleanup +############################################################################ +--source include/rpl/connection_source.inc +DROP TABLE t1; +--source include/rpl/sync_to_replica.inc + +--source include/rpl/connection_replica.inc +--source include/rpl/stop_replica.inc +--source include/rpl/uninstall_semisync_replica.inc +--source include/rpl/start_replica.inc + +--source include/rpl/connection_source.inc +# Restore the plugin's own variable before uninstalling the plugin. It ceases to +# exist along with the plugin, so setting it afterwards fails with +# ER_UNKNOWN_SYSTEM_VARIABLE. +--disable_query_log +--eval SET GLOBAL rpl_semi_sync_source_timeout = $source_save_timeout +--enable_query_log +--source include/rpl/uninstall_semisync_source.inc +--disable_query_log +--eval SET GLOBAL binlog_large_transaction_optimization_enabled = $source_save_enabled +--eval SET GLOBAL binlog_large_transaction_optimization_threshold = $source_save_threshold +--eval SET SESSION binlog_transaction_compression = $source_save_compression +--enable_query_log + +--source include/rpl/deinit.inc diff --git a/mysql-test/suite/sys_vars/r/all_vars.result b/mysql-test/suite/sys_vars/r/all_vars.result index bc7056288965..6f885efb3745 100644 --- a/mysql-test/suite/sys_vars/r/all_vars.result +++ b/mysql-test/suite/sys_vars/r/all_vars.result @@ -22,6 +22,10 @@ binlog_encryption binlog_encryption binlog_expire_logs_auto_purge binlog_expire_logs_auto_purge +binlog_large_transaction_optimization_enabled +binlog_large_transaction_optimization_enabled +binlog_large_transaction_optimization_threshold +binlog_large_transaction_optimization_threshold binlog_rotate_encryption_master_key_at_startup binlog_rotate_encryption_master_key_at_startup cte_max_recursion_depth diff --git a/mysql-test/suite/sys_vars/r/binlog_cache_size_basic.result b/mysql-test/suite/sys_vars/r/binlog_cache_size_basic.result index 0d94a61dd0ab..daa4a0d4b627 100644 --- a/mysql-test/suite/sys_vars/r/binlog_cache_size_basic.result +++ b/mysql-test/suite/sys_vars/r/binlog_cache_size_basic.result @@ -23,6 +23,7 @@ SELECT @@global.binlog_cache_size; SET @@global.binlog_cache_size = 4294967295; Warnings: Warning 1292 Truncated incorrect binlog_cache_size value: '4294967295' +Warning 6914 Variable 'binlog_large_transaction_optimization_threshold' was adjusted from 134217728 bytes to 4294963200 bytes to match binlog_cache_size. SELECT @@global.binlog_cache_size; @@global.binlog_cache_size 4294963200 diff --git a/mysql-test/suite/sys_vars/t/binlog_cache_size_basic.test b/mysql-test/suite/sys_vars/t/binlog_cache_size_basic.test index e58f61be2734..2d195b5188c2 100644 --- a/mysql-test/suite/sys_vars/t/binlog_cache_size_basic.test +++ b/mysql-test/suite/sys_vars/t/binlog_cache_size_basic.test @@ -31,6 +31,17 @@ # START OF binlog_cache_size TESTS # ################################################################# +######################################################################### +# Raising binlog_cache_size above # +# binlog_large_transaction_optimization_threshold adjusts the threshold # +# to match and reports it in the error log. This test sets # +# binlog_cache_size across its whole range, so the adjustment is # +# expected here. # +######################################################################### +--disable_query_log +CALL mtr.add_suppression("was adjusted from .* to match binlog_cache_size"); +--enable_query_log + ######################################################################### # Saving initial value of binlog_cache_size in a temporary variable # ######################################################################### @@ -38,6 +49,15 @@ SET @start_value = @@global.binlog_cache_size; SELECT @start_value; +# Raising binlog_cache_size above +# binlog_large_transaction_optimization_threshold raises the threshold to match, +# and the threshold is not lowered again when binlog_cache_size is restored. +# Save it here so this test leaves the threshold as it found it. +--disable_query_log +SET @start_bolt_threshold = + @@global.binlog_large_transaction_optimization_threshold; +--enable_query_log + --echo '#--------------------FN_DYNVARS_006_01------------------------#' ######################################################################### # Display the DEFAULT value of binlog_cache_size # @@ -196,6 +216,13 @@ SELECT @@session.binlog_cache_size; SET @@global.binlog_cache_size = @start_value; SELECT @@global.binlog_cache_size; +# Restore the threshold after binlog_cache_size, so the saved value is not +# raised again to match a still-large binlog_cache_size. +--disable_query_log +SET @@global.binlog_large_transaction_optimization_threshold = + @start_bolt_threshold; +--enable_query_log + ########################################################### # END OF binlog_cache_size TESTS # diff --git a/mysql-test/t/all_persisted_variables.test b/mysql-test/t/all_persisted_variables.test index b33b0af72930..a5e4a4255360 100644 --- a/mysql-test/t/all_persisted_variables.test +++ b/mysql-test/t/all_persisted_variables.test @@ -58,7 +58,7 @@ let $total_global_vars=`SELECT COUNT(*) AND variable_name NOT LIKE '%pqc%' AND variable_name NOT LIKE '%tls_kex%'`; -let $total_persistent_vars=452; +let $total_persistent_vars=454; --echo *************************************************************** --echo * 0. Verify that variables present in performance_schema.global diff --git a/mysys/mf_cache.cc b/mysys/mf_cache.cc index e83d20addbdb..eadd7b829148 100644 --- a/mysys/mf_cache.cc +++ b/mysys/mf_cache.cc @@ -78,8 +78,17 @@ bool real_open_cached_file(IO_CACHE *cache) { DBUG_TRACE; if ((cache->file = mysql_file_create_temp( cache->file_key, name_buff, cache->dir, cache->prefix, - (O_RDWR | O_TRUNC), UNLINK_FILE, MYF(MY_WME))) >= 0) { + (O_RDWR | O_TRUNC), cache->named_file ? KEEP_FILE : UNLINK_FILE, + MYF(MY_WME))) >= 0) { error = 0; + if (cache->named_file && + (cache->file_name = my_strdup(key_memory_IO_CACHE, name_buff, + MYF(MY_WME))) == nullptr) { + (void)mysql_file_close(cache->file, MYF(0)); + (void)my_delete(name_buff, MYF(0)); + cache->file = -1; + error = 1; + } } return error; } @@ -93,6 +102,12 @@ void close_cached_file(IO_CACHE *cache) { if (file >= 0) { (void)mysql_file_close(file, MYF(0)); } + if (cache->file_name != nullptr) { + /* A named temporary file is deleted when the cache is closed. */ + (void)my_delete(cache->file_name, MYF(0)); + my_free(cache->file_name); + cache->file_name = nullptr; + } my_free(cache->dir); my_free(cache->prefix); } diff --git a/share/messages_to_clients.txt b/share/messages_to_clients.txt index 1f014496e5ea..d8c081a6c3f7 100644 --- a/share/messages_to_clients.txt +++ b/share/messages_to_clients.txt @@ -11067,6 +11067,9 @@ ER_JDV_COLUMN_TAG_NOT_SUPPORTED_FOR_SUBQUERY ER_JDV_UPDATE_COLUMN_TAG_NOT_SUPPORTED_FOR_PK eng "The UPDATE column tag is not supported for a primary key projection at JSON path '%s'." +ER_BINLOG_BOLT_THRESHOLD_ADJUSTED_SQL_WARNING + eng "Variable 'binlog_large_transaction_optimization_threshold' was adjusted from %llu bytes to %llu bytes to match binlog_cache_size." + # # End of "9.7 cal-ver compatibility lineage (starts from 26.7)" error messages (server-to-client). # diff --git a/share/messages_to_error_log.txt b/share/messages_to_error_log.txt index 6a90e7128511..a3290e1ffeb8 100644 --- a/share/messages_to_error_log.txt +++ b/share/messages_to_error_log.txt @@ -13545,6 +13545,37 @@ ER_IB_MSG_MAX_PURGE_LAG_EXCEEDED ER_IB_MSG_DML_PURGE_DELAY_STARTED eng "InnoDB DML operations are being delayed by %lu microseconds due to purge lag. History list length is %llu, innodb_max_purge_lag=%lu, innodb_max_purge_lag_delay=%lu." +ER_BINLOG_BOLT_TEMP_FILES_DIR_INVALID + eng "Failed to initialize #binlog_temp_files at '%s': it must be a directory and cannot be a symbolic link." + +ER_BINLOG_BOLT_TEMP_FILES_DIR_UNSAFE_ENTRY + eng "Failed to initialize #binlog_temp_files at '%s': found unsafe entry '%s'. Only regular managed spill files named 'bolt_' are allowed." + +ER_BINLOG_BOLT_TEMP_FILES_DIR_FAILED + eng "Failed to initialize #binlog_temp_files at '%s' (OS error %d)." + +ER_BINLOG_BOLT_TEMP_FILES_DIR_CLEANED + eng "Removed %u leftover managed binary log temporary file(s) from '%s'." + +ER_BINLOG_BOLT_LARGE_TRX_FALLBACK + eng "Could not optimize large transaction execution in the binary log because %s; standard binary logging was used instead." + +ER_BINLOG_BOLT_RECOVERY_LARGE_TRX_SKIP + eng "Optimized binlog recovery by avoiding a sequential read of a large transaction body and seeking to its terminating event at offset %llu in binary log file '%s'." + +ER_BINLOG_BOLT_RECOVERY_LARGE_TRX_CHECKSUM_VERIFICATION + eng "Could not optimize binlog recovery of a large transaction because source_verify_checksum is enabled." + +ER_BINLOG_BOLT_INVALID_LARGE_TRX_HEADER + eng "Could not recover binary log file '%s' because it contains an invalid large transaction header." + +ER_BINLOG_BOLT_THRESHOLD_ADJUSTED + eng "Variable 'binlog_large_transaction_optimization_threshold' was adjusted from %llu bytes to %llu bytes to match binlog_cache_size." + +ER_BINLOG_BOLT_RELAY_LOG_PROMOTION + eng "Could not promote a spilled large transaction file to '%s': promotion is not supported for relay logs." + + # # End of "9.7 cal-ver compatibility lineage (starts from 26.7)" messages intended to be written to the server error log. # diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 7dd60d4d2209..fe4c92b95fad 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -1142,8 +1142,11 @@ SET(BINLOG_SOURCE basic_ostream.cc binlog/binlog_ofile.cc binlog/binlog_tc_log.cc + binlog/cache_data.cc binlog/thd_backup_and_restore.cc binlog/global.cc + binlog/large_trx_commit.cc + binlog/transaction_commit_helper.cc binlog/log_sanitizer.cc binlog/recovery.cc binlog/group_commit/bgc_ticket_manager.cc diff --git a/sql/binlog.cc b/sql/binlog.cc index c0c0ec855ab7..ce248e9279ff 100644 --- a/sql/binlog.cc +++ b/sql/binlog.cc @@ -51,6 +51,7 @@ #include #endif #include +#include #include #include #include @@ -93,12 +94,15 @@ #include "scope_guard.h" #include "sql/binlog/binlog_ofile.h" // Binlog_ofile #include "sql/binlog/binlog_tc_log.h" +#include "sql/binlog/cache_data.h" // binlog_cache_data #include "sql/binlog/decompressing_event_object_istream.h" #include "sql/binlog/global.h" #include "sql/binlog/group_commit/bgc_ticket_manager.h" // Bgc_ticket_manager -#include "sql/binlog/recovery.h" // binlog::Binlog_recovery +#include "sql/binlog/large_trx_commit.h" // get_cache_for_large_trx_commit +#include "sql/binlog/recovery.h" // binlog::Binlog_recovery #include "sql/binlog/services/iterator/file_storage.h" #include "sql/binlog/thd_backup_and_restore.h" +#include "sql/binlog/transaction_commit_helper.h" #include "sql/binlog_ostream.h" #include "sql/binlog_reader.h" #include "sql/create_field.h" @@ -159,6 +163,37 @@ using std::max; using std::min; using std::string; +static bool is_valid_large_trx_terminating_event(const char *filename, + my_off_t target, + my_off_t file_size, + my_off_t expected_end, + uint8_t expected_type) { + if (target < BIN_LOG_HEADER_SIZE || target > file_size || + expected_end > file_size || expected_end <= target || + file_size - target < LOG_EVENT_HEADER_LEN) + return false; + + Binlog_file_reader verifier(true /*verify_checksum*/); + if (verifier.open(filename, target)) return false; + + std::unique_ptr event(verifier.read_event_object()); + if (event == nullptr || verifier.has_fatal_error()) return false; + + const auto type = event->get_type_code(); + if (expected_type != 0) { + const bool supported_type = + expected_type == mysql::binlog::event::QUERY_EVENT || + expected_type == mysql::binlog::event::XID_EVENT || + expected_type == mysql::binlog::event::XA_PREPARE_LOG_EVENT; + return supported_type && static_cast(type) == expected_type && + verifier.position() == expected_end; + } + + return (type == mysql::binlog::event::XID_EVENT || + type == mysql::binlog::event::XA_PREPARE_LOG_EVENT) && + verifier.position() == expected_end; +} + #define FLAGSTR(V, F) ((V) & (F) ? #F " " : "") #define YESNO(X) ((X) ? "yes" : "no") @@ -167,8 +202,6 @@ using std::string; @{ */ -#define MY_OFF_T_UNDEF (~(my_off_t)0UL) - /* Constants required for the limit unsafe warnings suppression */ @@ -340,414 +373,6 @@ static bool check_auto_purge_conditions() { @warning The class is not designed to be inherited from. */ -/** - Caches for non-transactional and transactional data before writing - it to the binary log. - - @todo All the access functions for the flags suggest that the - encapsuling is not done correctly, so try to move any logic that - requires access to the flags into the cache. -*/ -class binlog_cache_data { - public: - binlog_cache_data(class binlog_cache_mngr &cache_mngr, bool trx_cache_arg, - ulong *ptr_binlog_cache_use_arg, - ulong *ptr_binlog_cache_disk_use_arg) - : m_cache_mngr(cache_mngr), - m_pending(nullptr), - ptr_binlog_cache_use(ptr_binlog_cache_use_arg), - ptr_binlog_cache_disk_use(ptr_binlog_cache_disk_use_arg) { - flags.transactional = trx_cache_arg; - } - - bool open(my_off_t cache_size, my_off_t max_cache_size) { - return m_cache.open(cache_size, max_cache_size); - } - - Binlog_cache_storage *get_cache() { return &m_cache; } - int finalize(THD *thd, Log_event *end_event); - int finalize(THD *thd, Log_event *end_event, XID_STATE *xs); - int flush(THD *thd, my_off_t *bytes, bool *wrote_xid, - bool parallelization_barrier); - int write_event(Log_event *event); - void set_event_counter(size_t event_counter) { - m_event_counter = event_counter; - } - size_t get_event_counter() const { return m_event_counter; } - size_t get_compressed_size() const { return m_compressed_size; } - size_t get_decompressed_size() const { return m_decompressed_size; } - mysql::binlog::event::compression::type get_compression_type() const { - return m_compression_type; - } - - void set_compressed_size(size_t s) { m_compressed_size = s; } - void set_decompressed_size(size_t s) { m_decompressed_size = s; } - void set_compression_type(mysql::binlog::event::compression::type t) { - m_compression_type = t; - } - - virtual ~binlog_cache_data() { - assert(is_binlog_empty()); - m_cache.close(); - } - - bool is_binlog_empty() const { - DBUG_PRINT("debug", ("%s_cache - pending: 0x%llx, bytes: %llu", - (flags.transactional ? "trx" : "stmt"), - (ulonglong)pending(), (ulonglong)m_cache.length())); - return pending() == nullptr && m_cache.is_empty(); - } - - bool is_finalized() const { return flags.finalized; } - - Rows_log_event *pending() const { return m_pending; } - - void set_pending(Rows_log_event *const pending) { m_pending = pending; } - - /// @see handle_deferred_cache_write_incident - void set_incident( - std::string_view incident_message = - "Non-transactional changes were not written to the binlog."); - - /// @see handle_deferred_cache_write_incident - bool has_incident(void) const; - - bool has_xid() const { - // There should only be an XID event if we are transactional - assert((flags.transactional && flags.with_xid) || !flags.with_xid); - return flags.with_xid; - } - - bool is_trx_cache() const { return flags.transactional; } - - my_off_t get_byte_position() const { return m_cache.length(); } - - void cache_state_checkpoint(my_off_t pos_to_checkpoint) { - // We only need to store the cache state for pos > 0 - if (pos_to_checkpoint) { - cache_state state; - state.with_rbr = flags.with_rbr; - state.with_sbr = flags.with_sbr; - state.with_start = flags.with_start; - state.with_end = flags.with_end; - state.with_content = flags.with_content; - state.event_counter = m_event_counter; - cache_state_map[pos_to_checkpoint] = state; - } - } - - void cache_state_rollback(my_off_t pos_to_rollback) { - if (pos_to_rollback) { - std::map::iterator it; - it = cache_state_map.find(pos_to_rollback); - if (it != cache_state_map.end()) { - flags.with_rbr = it->second.with_rbr; - flags.with_sbr = it->second.with_sbr; - flags.with_start = it->second.with_start; - flags.with_end = it->second.with_end; - flags.with_content = it->second.with_content; - m_event_counter = it->second.event_counter; - } else - assert(it == cache_state_map.end()); - } - // Rolling back to pos == 0 means cleaning up the cache. - else { - flags.with_rbr = false; - flags.with_sbr = false; - flags.with_start = false; - flags.with_end = false; - flags.with_content = false; - m_event_counter = 0; - } - } - - /** - Reset the cache to unused state when the transaction is finished. It - drops all data in the cache and clears the flags of the transaction state. - */ - virtual void reset() { - compute_statistics(); - remove_pending_event(); - - if (m_cache.reset()) { - LogErr(WARNING_LEVEL, ER_BINLOG_CANT_RESIZE_CACHE); - } - - flags.with_xid = false; - flags.immediate = false; - flags.finalized = false; - flags.with_sbr = false; - flags.with_rbr = false; - flags.with_start = false; - flags.with_end = false; - flags.with_content = false; - - /* - The truncate function calls reinit_io_cache that calls my_b_flush_io_cache - which may increase disk_writes. This breaks the disk_writes use by the - binary log which aims to compute the ratio between in-memory cache usage - and disk cache usage. To avoid this undesirable behavior, we reset the - variable after truncating the cache. - */ - cache_state_map.clear(); - m_event_counter = 0; - m_compressed_size = 0; - m_decompressed_size = 0; - m_compression_type = mysql::binlog::event::compression::NONE; - assert(is_binlog_empty()); - } - - /** - Returns information about the cache content with respect to - the binlog_format of the events. - - This will be used to set a flag on GTID_LOG_EVENT stating that the - transaction may have SBR statements or not, but the binlog dump - will show this flag as "rbr_only" when it is not set. That's why - an empty transaction should return true below, or else an empty - transaction would be assumed as "rbr_only" even not having RBR - events. - - When dumping a binary log content using mysqlbinlog client program, - for any transaction assumed as "rbr_only" it will be printed a - statement changing the transaction isolation level to READ COMMITTED. - It doesn't make sense to have an empty transaction "requiring" this - isolation level change. - - @return true The cache have SBR events or is empty. - @return false The cache contains a transaction with no SBR events. - */ - bool may_have_sbr_stmts() { return flags.with_sbr || !flags.with_rbr; } - - /** - Check if the binlog cache contains an empty transaction, which has - two binlog events "BEGIN" and "COMMIT". - - @return true The binlog cache contains an empty transaction. - @return false Otherwise. - */ - bool has_empty_transaction() { - /* - The empty transaction has two events in trx/stmt binlog cache - and no changes: one is a transaction start and other is a transaction - end (there should be no SBR changing content and no RBR events). - */ - if (flags.with_start && // Has transaction start statement - flags.with_end && // Has transaction end statement - !flags.with_content) // Has no other content than START/END - { - assert(m_event_counter == 2); // Two events in the cache only - assert(!flags.with_sbr); // No statements changing content - assert(!flags.with_rbr); // No rows changing content - assert(!flags.immediate); // Not a DDL - assert(!flags.with_xid); // Not a XID trx and not an atomic DDL Query - return true; - } - return false; - } - - /** - Check if the binlog cache is empty or contains an empty transaction, - which has two binlog events "BEGIN" and "COMMIT". - - @return true The binlog cache is empty or contains an empty transaction. - @return false Otherwise. - */ - bool is_empty_or_has_empty_transaction() { - return is_binlog_empty() || has_empty_transaction(); - } - - protected: - /* - This structure should have all cache variables/flags that should be restored - when a ROLLBACK TO SAVEPOINT statement be executed. - */ - struct cache_state { - bool with_sbr; - bool with_rbr; - bool with_start; - bool with_end; - bool with_content; - size_t event_counter; - }; - /* - For every SAVEPOINT used, we will store a cache_state for the current - binlog cache position. So, if a ROLLBACK TO SAVEPOINT is used, we can - restore the cache_state values after truncating the binlog cache. - */ - std::map cache_state_map; - /* - In order to compute the transaction size (because of possible extra checksum - bytes), we need to keep track of how many events are in the binlog cache. - */ - size_t m_event_counter = 0; - - size_t m_compressed_size = 0; - size_t m_decompressed_size = 0; - mysql::binlog::event::compression::type m_compression_type = - mysql::binlog::event::compression::type::NONE; - /* - It truncates the cache to a certain position. This includes deleting the - pending event. It corresponds to rollback statement or rollback to - a savepoint. It doesn't change transaction state. - */ - void truncate(my_off_t pos) { - DBUG_PRINT("info", ("truncating to position %lu", (ulong)pos)); - remove_pending_event(); - - // TODO: check the return value. - (void)m_cache.truncate(pos); - } - - /** - Flush pending event to the cache buffer. - */ - int flush_pending_event(THD *thd) { - if (m_pending) { - m_pending->set_flags(Rows_log_event::STMT_END_F); - if (int error = write_event(m_pending)) return error; - thd->clear_binlog_table_maps(); - } - return 0; - } - - /** - Remove the pending event. - */ - int remove_pending_event() { - delete m_pending; - m_pending = nullptr; - return 0; - } - struct Flags { - /* - Defines if this is either a trx-cache or stmt-cache, respectively, a - transactional or non-transactional cache. - */ - bool transactional : 1; - - /* - This indicates that the cache should be written without BEGIN/END. - */ - bool immediate : 1; - - /* - This flag indicates that the buffer was finalized and has to be - flushed to disk. - */ - bool finalized : 1; - - /* - This indicates that either the cache contain an XID event, or it's - an atomic DDL Query-log-event. In the latter case the flag is set up - on the statement level, namely when the Query-log-event is cached - at time the DDL transaction is not committing. - The flag therefore gets reset when the cache is cleaned due to - the statement rollback, e.g in case of a DDL post-caching execution - error. - Any statement scope flag among other things must consider its - reset policy when the statement is rolled back. - */ - bool with_xid : 1; - - /* - This indicates that the cache contain statements changing content. - */ - bool with_sbr : 1; - - /* - This indicates that the cache contain RBR event changing content. - */ - bool with_rbr : 1; - - /* - This indicates that the cache contain s transaction start statement. - */ - bool with_start : 1; - - /* - This indicates that the cache contain a transaction end event. - */ - bool with_end : 1; - - /* - This indicates that the cache contain content other than START/END. - */ - bool with_content : 1; - } flags; - - /// Compress the current transaction "in-place", if possible - /// - /// This attempts to compress the transaction if it satisfies the - /// necessary pre-conditions. Otherwise it does nothing. - /// - /// @retval true Error: the cache has been corrupted and the - /// transaction must be aborted. - /// - /// @retval false Success: the transaction was either compressed - /// successfully, or compression was not attempted, or compression - /// failed and left the uncompressed transaction intact. - [[nodiscard]] bool compress(THD *thd); - - private: - /* - Reference to the cache_mngr which owns this cache. - */ - class binlog_cache_mngr &m_cache_mngr; - - /* - Storage for byte data. This binlog_cache_data will serialize - events into bytes and put them into m_cache. - */ - Binlog_cache_storage m_cache; - - /* - Pending binrows event. This event is the event where the rows are currently - written. - */ - Rows_log_event *m_pending; - - /** - This function computes binlog cache and disk usage. - */ - void compute_statistics() { - if (!is_binlog_empty()) { - (*ptr_binlog_cache_use)++; - if (m_cache.disk_writes() != 0) (*ptr_binlog_cache_disk_use)++; - } - } - - /* - Stores a pointer to the status variable that keeps track of the in-memory - cache usage. This corresponds to either - . binlog_cache_use or binlog_stmt_cache_use. - */ - ulong *ptr_binlog_cache_use; - - /* - Stores a pointer to the status variable that keeps track of the disk - cache usage. This corresponds to either - . binlog_cache_disk_use or binlog_stmt_cache_disk_use. - */ - ulong *ptr_binlog_cache_disk_use; - - binlog_cache_data &operator=(const binlog_cache_data &info); - binlog_cache_data(const binlog_cache_data &info); -}; - -class binlog_stmt_cache_data : public binlog_cache_data { - public: - binlog_stmt_cache_data(binlog_cache_mngr &cache_mngr, bool trx_cache_arg, - ulong *ptr_binlog_cache_use_arg, - ulong *ptr_binlog_cache_disk_use_arg) - : binlog_cache_data(cache_mngr, trx_cache_arg, ptr_binlog_cache_use_arg, - ptr_binlog_cache_disk_use_arg) {} - - using binlog_cache_data::finalize; - - int finalize(THD *thd); -}; - int binlog_stmt_cache_data::finalize(THD *thd) { if (flags.immediate) { if (int error = finalize(thd, nullptr)) return error; @@ -759,216 +384,7 @@ int binlog_stmt_cache_data::finalize(THD *thd) { return 0; } -class binlog_trx_cache_data : public binlog_cache_data { - public: - binlog_trx_cache_data(binlog_cache_mngr &cache_mngr, bool trx_cache_arg, - ulong *ptr_binlog_cache_use_arg, - ulong *ptr_binlog_cache_disk_use_arg) - : binlog_cache_data(cache_mngr, trx_cache_arg, ptr_binlog_cache_use_arg, - ptr_binlog_cache_disk_use_arg), - m_cannot_rollback(false), - before_stmt_pos(MY_OFF_T_UNDEF) {} - - void reset() override { - DBUG_TRACE; - DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - m_cannot_rollback = false; - before_stmt_pos = MY_OFF_T_UNDEF; - binlog_cache_data::reset(); - DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - return; - } - - bool cannot_rollback() const { return m_cannot_rollback; } - - void set_cannot_rollback() { m_cannot_rollback = true; } - - my_off_t get_prev_position() const { return before_stmt_pos; } - - void set_prev_position(my_off_t pos) { - DBUG_TRACE; - DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - before_stmt_pos = pos; - cache_state_checkpoint(before_stmt_pos); - DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - return; - } - - void restore_prev_position() { - DBUG_TRACE; - DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - binlog_cache_data::truncate(before_stmt_pos); - cache_state_rollback(before_stmt_pos); - before_stmt_pos = MY_OFF_T_UNDEF; - /* - Binlog statement rollback clears with_xid now as the atomic DDL statement - marker which can be set as early as at event creation and caching. - */ - flags.with_xid = false; - DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - return; - } - - void restore_savepoint(my_off_t pos) { - DBUG_TRACE; - DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - binlog_cache_data::truncate(pos); - if (pos <= before_stmt_pos) before_stmt_pos = MY_OFF_T_UNDEF; - cache_state_rollback(pos); - DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); - return; - } - - using binlog_cache_data::truncate; - - void truncate(THD *thd, bool all); - - private: - /* - It will be set true if any statement which cannot be rolled back safely - is put in trx_cache. - */ - bool m_cannot_rollback; - - /* - Binlog position before the start of the current statement. - */ - my_off_t before_stmt_pos; - - binlog_trx_cache_data &operator=(const binlog_trx_cache_data &info); - binlog_trx_cache_data(const binlog_trx_cache_data &info); -}; - -class binlog_cache_mngr { - /// Indicates that some events did not get into the cache(s) and most - /// likely it is incomplete. @see handle_deferred_cache_write_incident - std::string m_incident; - - public: -#ifndef NDEBUG - /// The number of times that the incident status has been set due to the - /// debug symbol binlog_inject_incident. - int m_injected_incident_count{0}; -#endif - - binlog_cache_mngr(ulong *ptr_binlog_stmt_cache_use_arg, - ulong *ptr_binlog_stmt_cache_disk_use_arg, - ulong *ptr_binlog_cache_use_arg, - ulong *ptr_binlog_cache_disk_use_arg) - : stmt_cache(*this, false, ptr_binlog_stmt_cache_use_arg, - ptr_binlog_stmt_cache_disk_use_arg), - trx_cache(*this, true, ptr_binlog_cache_use_arg, - ptr_binlog_cache_disk_use_arg) {} - - bool init() { - return stmt_cache.open(binlog_stmt_cache_size, - max_binlog_stmt_cache_size) || - trx_cache.open(binlog_cache_size, max_binlog_cache_size); - } - - binlog_cache_data *get_binlog_cache_data(bool is_transactional) { - if (is_transactional) - return &trx_cache; - else - return &stmt_cache; - } - - Binlog_cache_storage *get_stmt_cache() { return stmt_cache.get_cache(); } - Binlog_cache_storage *get_trx_cache() { return trx_cache.get_cache(); } - /** - Convenience method to check if both caches are empty. - */ - bool is_binlog_empty() const { - return stmt_cache.is_binlog_empty() && trx_cache.is_binlog_empty(); - } - - int handle_deferred_cache_write_incident(THD *thd); - - /// Check if either of the caches have an incident - /// @see handle_deferred_cache_write_incident - bool has_incident() const { return !m_incident.empty(); } - - void set_incident(std::string_view incident_message) { - assert(!incident_message.empty()); - m_incident = incident_message; - } - - /* - clear stmt_cache and trx_cache if they are not empty - */ - void reset() { - if (!stmt_cache.is_binlog_empty()) stmt_cache.reset(); - if (!trx_cache.is_binlog_empty()) trx_cache.reset(); - } - -#ifndef NDEBUG - bool dbug_any_finalized() const { - return stmt_cache.is_finalized() || trx_cache.is_finalized(); - } -#endif - - /* - Convenience method to flush both caches to the binary log. - - @param bytes_written Pointer to variable that will be set to the - number of bytes written for the flush. - @param wrote_xid Pointer to variable that will be set to @c - true if any XID event was written to the - binary log. Otherwise, the variable will not - be touched. - @return Error code on error, zero if no error. - */ - int flush(THD *thd, my_off_t *bytes_written, bool *wrote_xid) { - my_off_t stmt_bytes = 0; - my_off_t trx_bytes = 0; - assert(stmt_cache.has_xid() == 0); - - bool parallelization_barrier = false; - if (has_incident()) { - if (int error = handle_deferred_cache_write_incident(thd)) return error; - // Request force rotate - thd->rpl_thd_ctx.binlog_group_commit_ctx().set_force_rotate(); - // Set as parallelization_barrier so that dependency tracker marks all - // subsequent transactions to depend on it. - parallelization_barrier = true; - } - - int error = - stmt_cache.flush(thd, &stmt_bytes, wrote_xid, parallelization_barrier); - if (error) return error; - DEBUG_SYNC(thd, "after_flush_stm_cache_before_flush_trx_cache"); - error = - trx_cache.flush(thd, &trx_bytes, wrote_xid, parallelization_barrier); - if (error) return error; - *bytes_written = stmt_bytes + trx_bytes; - return 0; - } - - /** - Check if at least one of transactions and statement binlog caches - contains an empty transaction, other one is empty or contains an - empty transaction. - - @return true At least one of transactions and statement binlog - caches an empty transaction, other one is empty - or contains an empty transaction. - @return false Otherwise. - */ - bool has_empty_transaction() { - return (trx_cache.is_empty_or_has_empty_transaction() && - stmt_cache.is_empty_or_has_empty_transaction() && - !is_binlog_empty()); - } - - binlog_stmt_cache_data stmt_cache; - binlog_trx_cache_data trx_cache; - - private: - binlog_cache_mngr &operator=(const binlog_cache_mngr &info); - binlog_cache_mngr(const binlog_cache_mngr &info); -}; - -static binlog_cache_mngr *thd_get_cache_mngr(const THD *thd) { +binlog_cache_mngr *thd_get_cache_mngr(const THD *thd) { /* If opt_bin_log is not set, binlog_hton->slot == -1 and hence thd_get_ha_data(thd, hton) segfaults. @@ -1071,12 +487,33 @@ static int binlog_dummy_recover(handlerton *, XA_recover_txn *, uint, class Binlog_event_writer : public Basic_ostream { MYSQL_BIN_LOG::Binlog_ofile *m_binlog_file; bool have_checksum; + /* + Checksum algorithm the incoming cache events were serialized with, captured + once at the transaction's first event. Used to detect each event's stale + checksum (stale because log_pos changes on copy) so it can be skipped and + recomputed for the binlog. UNDEF means the events carry no checksum (e.g. + events written directly rather than from a cache). + */ + enum_binlog_checksum_alg m_checksum_trx_start = + mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF; + + /** + Returns true when the events streaming in carry a checksum. + */ + bool is_checksum_computed() const { + return m_checksum_trx_start != + mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF && + m_checksum_trx_start != + mysql::binlog::event::BINLOG_CHECKSUM_ALG_OFF; + } ha_checksum initial_checksum; ha_checksum checksum; uint32 end_log_pos; uchar header[LOG_EVENT_HEADER_LEN]; my_off_t header_len = 0; uint32 event_len = 0; + /* Stale checksum bytes of the finished event left to skip. */ + uint32 skip_len = 0; public: /** @@ -1098,27 +535,79 @@ class Binlog_event_writer : public Basic_ostream { } void update_header() { - event_len = uint4korr(header + EVENT_LEN_OFFSET); + /* Length of the incoming event, its stale checksum included when + it carries one. */ + uint32 event_len_incoming = uint4korr(header + EVENT_LEN_OFFSET); - // Increase end_log_pos - end_log_pos += event_len; + /* Length the event will have in the binlog file. */ + uint32 event_len_on_disk = event_len_incoming; + if (have_checksum && !is_checksum_computed()) + event_len_on_disk += BINLOG_CHECKSUM_LEN; + else if (!have_checksum && is_checksum_computed()) + event_len_on_disk -= BINLOG_CHECKSUM_LEN; - // Update event length if it has checksum - if (have_checksum) { - int4store(header + EVENT_LEN_OFFSET, event_len + BINLOG_CHECKSUM_LEN); - end_log_pos += BINLOG_CHECKSUM_LEN; - } + end_log_pos += event_len_on_disk; - // Store end_log_pos + int4store(header + EVENT_LEN_OFFSET, event_len_on_disk); int4store(header + LOG_POS_OFFSET, end_log_pos); // update the checksum if (have_checksum) checksum = my_checksum(checksum, header, header_len); + + /* The event's bytes to copy; the stale checksum is not copied, it + is skipped after them (see skip_len). */ + event_len = event_len_incoming; + /* + write() later reduces this by the header it has emitted, so require room + for the header as well as the checksum: the smallest checksummed event is + a bare header followed by its checksum. + */ + if (is_checksum_computed() && + event_len >= LOG_EVENT_HEADER_LEN + BINLOG_CHECKSUM_LEN) + event_len -= BINLOG_CHECKSUM_LEN; + } + + /** + Finish the event just copied: write the checksum computed over its + rewritten header and body, then arrange for the stale checksum it + carried in the cache to be skipped. + + @retval false Success + @retval true Error + */ + bool finish_event() { + if (have_checksum && write_checksum()) return true; + if (is_checksum_computed()) skip_len = BINLOG_CHECKSUM_LEN; + return false; + } + + /** + Write the computed checksum after the event, and restart the + checksum for the next event. + + @retval false Success + @retval true Error + */ + bool write_checksum() { + uchar checksum_buf[BINLOG_CHECKSUM_LEN]; + int4store(checksum_buf, checksum); + if (m_binlog_file->write(checksum_buf, BINLOG_CHECKSUM_LEN)) return true; + checksum = initial_checksum; + return false; } bool write(const unsigned char *buffer, my_off_t length) override { DBUG_TRACE; while (length > 0) { + /* Skip the previous event's stale checksum */ + if (skip_len > 0) { + uint32 skip = std::min(skip_len, length); + buffer += skip; + length -= skip; + skip_len -= skip; + continue; + } + /* Write event header into binlog */ if (event_len == 0) { /* data in the buf may be smaller than header size.*/ @@ -1136,6 +625,9 @@ class Binlog_event_writer : public Basic_ostream { event_len -= header_len; header_len = 0; + + /* An event with an empty body is complete already. */ + if (event_len == 0 && finish_event()) return true; } } else { my_off_t write_bytes = std::min(length, event_len); @@ -1150,15 +642,9 @@ class Binlog_event_writer : public Basic_ostream { length -= write_bytes; buffer += write_bytes; - // The whole event is copied, now add the checksum - if (have_checksum && event_len == 0) { - uchar checksum_buf[BINLOG_CHECKSUM_LEN]; - - int4store(checksum_buf, checksum); - if (m_binlog_file->write(checksum_buf, BINLOG_CHECKSUM_LEN)) - return true; - checksum = initial_checksum; - } + // The whole event is copied: write its checksum, skip its + // stale one. + if (event_len == 0 && finish_event()) return true; } } return false; @@ -1167,6 +653,16 @@ class Binlog_event_writer : public Basic_ostream { Returns true if per event checksum is enabled. */ bool is_checksum_enabled() { return have_checksum; } + + /** + Tell the writer which checksum algorithm the events about to + stream in were serialized with at their transaction's start. Set + before copying a cache (the Gtid event written before the cache + carries no checksum). + */ + void set_checksum_trx_start(enum_binlog_checksum_alg alg) { + m_checksum_trx_start = alg; + } }; /* @@ -1217,6 +713,7 @@ int binlog_cache_data::write_event(Log_event *ev) { DBUG_TRACE; if (ev != nullptr) { + if (is_trx_cache()) latch_large_trx_optimization(); DBUG_EXECUTE_IF("binlog_inject_incident", { // Set the incident status only once per session. Without this limitation, // it usually gets sets first for the transaction cache and then, when @@ -1231,6 +728,44 @@ int binlog_cache_data::write_event(Log_event *ev) { DBUG_EXECUTE_IF("simulate_disk_full_at_flush_pending", { DBUG_SET("+d,simulate_file_write_error"); }); + /* + Set the event's log_pos assuming this cache becomes the start of a new + binlog file: offset = reserved header region + bytes written so far. + - Small / non-bolt-promoted transaction: at commit the cache is copied + into the active binlog, and Binlog_event_writer overwrites log_pos with + the real destination offset. + - Large / bolt-promoted transaction: the spilled file becomes the next + binlog file with this transaction at its start, so this log_pos is already + correct and is used as is. + */ + ev->common_header->log_pos = m_cache.reserved_bytes() + m_cache.length(); + + /* + One checksum algorithm for the whole transaction, captured at its first + event, so a mid-transaction binlog_checksum change cannot mix algorithms + within one cache. Compressible transactions record + BINLOG_CHECKSUM_ALG_OFF, because events inside a compressed payload carry + no checksum. + + Only compute the checksum when writing to the cache when the binlog large + transaction optimization is enabled and this is the transaction cache + (m_large_trx_optimization_enabled is false for the statement cache). + The latch is per cache, so a transaction using both gets a checksummed + transaction cache and an unchecksummed statement cache. That is safe + because each cache is copied by its own Binlog_event_writer, which + knows whether to strip a stale checksum. + */ + if (m_large_trx_optimization_enabled) { + if (m_checksum_alg_in_cache == + mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF) + m_checksum_alg_in_cache = + (ev->thd != nullptr && ev->thd->variables.binlog_trx_compression) + ? mysql::binlog::event::BINLOG_CHECKSUM_ALG_OFF + : static_cast( + binlog_checksum_options); + ev->common_footer->checksum_alg = m_checksum_alg_in_cache; + } + if (binary_event_serialize(ev, &m_cache)) { DBUG_EXECUTE_IF("simulate_disk_full_at_flush_pending", { DBUG_SET("-d,simulate_file_write_error"); @@ -1351,116 +886,20 @@ bool MYSQL_BIN_LOG::write_transaction(THD *thd, binlog_cache_data *cache_data, assert(thd->owned_gtid.sidno == THD::OWNED_SIDNO_ANONYMOUS || thd->owned_gtid.sidno > 0); - int64 sequence_number, last_committed; - /* Generate logical timestamps for MTS */ - m_dependency_tracker.get_dependency(thd, parallelization_barrier, - sequence_number, last_committed); - - /* - In case both the transaction cache and the statement cache are - non-empty, both will be flushed in sequence and logged as - different transactions. Then the second transaction must only - be executed after the first one has committed. Therefore, we - need to set last_committed for the second transaction equal to - last_committed for the first transaction. This is done in - binlog_cache_data::flush. binlog_cache_data::flush uses the - condition trn_ctx->last_committed==SEQ_UNINIT to detect this - situation, hence the need to set it here. - */ - thd->get_transaction()->last_committed = SEQ_UNINIT; + Transaction_gtid_header metadata{thd, parallelization_barrier, + &m_dependency_tracker}; - /* - For delayed replication and also for the purpose of lag monitoring, - we assume that the commit timestamp of the transaction is the time of - executing this code (the time of writing the Gtid_log_event to the binary - log). - */ - ulonglong immediate_commit_timestamp = my_micro_time(); - - /* - When the original_commit_timestamp session variable is set to a value - other than UNDEFINED_COMMIT_TIMESTAMP, it means that either the timestamp - is known ( > 0 ) or the timestamp is not known ( == 0 ). - */ - ulonglong original_commit_timestamp = - thd->variables.original_commit_timestamp; - /* - When original_commit_timestamp == UNDEFINED_COMMIT_TIMESTAMP, we assume - that: - a) it is not known if this thread is a slave applier ( = 0 ); - b) this is a new transaction ( = immediate_commit_timestamp); - */ - if (original_commit_timestamp == UNDEFINED_COMMIT_TIMESTAMP) { - /* - When applying a transaction using replication, assume that the - original commit timestamp is not known (the transaction wasn't - originated on the current server). - */ - if (thd->slave_thread || thd->is_binlog_applier()) { - original_commit_timestamp = 0; - } else - /* Assume that this transaction is original from this server */ - { - DBUG_EXECUTE_IF("rpl_invalid_gtid_timestamp", - // add one our to the commit timestamps - immediate_commit_timestamp += 3600000000;); - original_commit_timestamp = immediate_commit_timestamp; - } - } else { - // Clear the session variable to have cleared states for next transaction. - thd->variables.original_commit_timestamp = UNDEFINED_COMMIT_TIMESTAMP; - } - - uint32_t trx_immediate_server_version = - do_server_version_int(::server_version); - // Clear the session variable to have cleared states for next transaction. - thd->variables.immediate_server_version = UNDEFINED_SERVER_VERSION; - DBUG_EXECUTE_IF("fixed_server_version", - trx_immediate_server_version = 888888;); - DBUG_EXECUTE_IF("gr_fixed_server_version", - trx_immediate_server_version = 777777;); - - /* - When the original_server_version session variable is set to a value - other than UNDEFINED_SERVER_VERSION, it means that either the - server version is known or the server_version is not known - (UNKNOWN_SERVER_VERSION). - */ - uint32_t trx_original_server_version = thd->variables.original_server_version; - - /* - When original_server_version == UNDEFINED_SERVER_VERSION, we assume - that: - a) it is not known if this thread is a slave applier ( = 0 ); - b) this is a new transaction ( = ::server_version); - */ - if (trx_original_server_version == UNDEFINED_SERVER_VERSION) { - /* - When applying a transaction using replication, assume that the - original server version is not known (the transaction wasn't - originated on the current server). - */ - if (thd->slave_thread || thd->is_binlog_applier()) { - trx_original_server_version = UNKNOWN_SERVER_VERSION; - } else - /* Assume that this transaction is original from this server */ - { - trx_original_server_version = trx_immediate_server_version; - } - } else { - // Clear the session variable to have cleared states for next transaction. - thd->variables.original_server_version = UNDEFINED_SERVER_VERSION; - } Gtid_log_event gtid_event( - thd, cache_data->is_trx_cache(), last_committed, sequence_number, - cache_data->may_have_sbr_stmts(), original_commit_timestamp, - immediate_commit_timestamp, trx_original_server_version, - trx_immediate_server_version); + thd, cache_data->is_trx_cache(), metadata.last_committed(), + metadata.sequence_number(), cache_data->may_have_sbr_stmts(), + metadata.original_commit_timestamp(), + metadata.immediate_commit_timestamp(), metadata.original_server_version(), + metadata.immediate_server_version()); // Set the transaction length, based on cache info - gtid_event.set_trx_length_by_cache_size(cache_data->get_byte_position(), - writer->is_checksum_enabled(), - cache_data->get_event_counter()); + gtid_event.set_trx_length_by_cache_size( + cache_data->get_byte_position(), writer->is_checksum_enabled(), + cache_data->is_checksum_computed(), cache_data->get_event_counter()); DBUG_PRINT("debug", ("cache_data->get_byte_position()= %llu", cache_data->get_byte_position())); @@ -1476,6 +915,9 @@ bool MYSQL_BIN_LOG::write_transaction(THD *thd, binlog_cache_data *cache_data, gtid_event.write(writer)); if (ret) goto end; + /* The Gtid event above carried no checksum; the cache's events may. */ + writer->set_checksum_trx_start(cache_data->checksum_alg_in_cache()); + /* finally write the transaction data, if it was not compressed and written as part of the gtid event already @@ -1831,6 +1273,16 @@ class Binlog_cache_compressor { DBUG_PRINT("info", ("fallback to uncompressed: may have SBR events")); return false; } + /* + Do not compress events that carry a checksum: events + inside a payload must not have one. This only happens when + binlog_transaction_compression was enabled after the + transaction's first event (see binlog_cache_data::write_event). + */ + if (m_cache.is_checksum_computed()) { + DBUG_PRINT("info", ("fallback to uncompressed: events have checksum")); + return false; + } // nothing can stop us now! return true; } @@ -1992,6 +1444,17 @@ int binlog_cache_data::finalize(THD *thd, Log_event *end_event) { if (!is_binlog_empty()) { assert(!flags.finalized); if (int error = flush_pending_event(thd)) return error; + /* + Record where the transaction's terminating event will sit in a promoted + file, so recovery can seek to it. Only a real end_event + (COMMIT / XID / XA_PREPARE) is a terminating event; immediately-logged + statements (e.g. CREATE TABLE) finalize with end_event == nullptr and + record no terminating metadata. + */ + if (end_event != nullptr) { + m_terminating_event_offset = m_cache.reserved_bytes() + m_cache.length(); + m_terminating_event_type = end_event->get_type_code(); + } if (int error = write_event(end_event)) return error; if (int error = this->compress(thd)) return error; DBUG_PRINT("debug", ("flags.finalized: %s", YESNO(flags.finalized))); @@ -2073,6 +1536,35 @@ int binlog_cache_mngr::handle_deferred_cache_write_incident(THD *thd) { return 0; } +/* + Cache reservation only needs an estimate. The exact header size is checked + again during promotion, so a stale value merely causes fallback to standard + commit rather than an unsafe promotion. +*/ +static std::atomic binlog_temp_file_previous_gtids_size{0}; + +my_off_t get_binlog_temp_file_reserved_bytes() { + const my_off_t previous_gtids_size = + binlog_temp_file_previous_gtids_size.load(std::memory_order_relaxed); + const my_off_t required_size = + previous_gtids_size + kBinlogTempFilePreviousGtidsHeadroomBytes; + const my_off_t remainder = required_size % kBinlogTempFileReservedBytes; + return remainder == 0 + ? required_size + : required_size + kBinlogTempFileReservedBytes - remainder; +} + +void update_binlog_temp_file_previous_gtids_size_estimate( + my_off_t previous_gtids_size) { + binlog_temp_file_previous_gtids_size.store(previous_gtids_size, + std::memory_order_relaxed); +} + +bool binlog_cache_data::open(my_off_t cache_size, my_off_t max_cache_size) { + return m_cache.open(cache_size, max_cache_size, + get_binlog_temp_file_reserved_bytes()); +} + /** Flush caches to the binary log. @@ -3509,14 +3001,22 @@ bool MYSQL_BIN_LOG::init_and_set_log_file_name(const char *log_name, @param new_index_number The binary log file index number to start from after the RESET BINARY LOGS AND GTIDS command is called. + @param mode kOpenExisting when the file already exists on disk + with its contents in place (a promoted spill file): + it is opened for append (positioned at its end) + rather than created fresh, and no new file header + or encryption header is written. kCreateNew for a + normal, newly created log file. @return true if error, false otherwise. */ bool MYSQL_BIN_LOG::open(PSI_file_key log_file_key, const char *log_name, - const char *new_name, uint32 new_index_number) { + const char *new_name, uint32 new_index_number, + Binlog_file_mode mode) { DBUG_TRACE; bool ret = false; + const bool existing = (mode == Binlog_file_mode::kOpenExisting); write_error = false; myf flags = MY_WME | MY_NABP | MY_WAIT_IF_FULL; @@ -3542,7 +3042,19 @@ bool MYSQL_BIN_LOG::open(PSI_file_key log_file_key, const char *log_name, */ if (!is_relay_log) mysql_mutex_lock(&LOCK_sync); - ret = m_binlog_file->open(log_file_key, log_file_name, flags); + ret = m_binlog_file->open(log_file_key, log_file_name, flags, existing); + + /* + A promoted binary log file already contains its header events and the + transaction; position at its end, so subsequent transactions append. + */ + if (!ret && existing) { + MY_STAT info; + if (mysql_file_stat(log_file_key, log_file_name, &info, MYF(MY_WME)) == + nullptr || + m_binlog_file->position_at(info.st_size)) + ret = true; + } if (!is_relay_log) mysql_mutex_unlock(&LOCK_sync); @@ -3904,6 +3416,8 @@ static enum_read_gtids_from_binlog_status read_gtids_from_binlog( enum_read_gtids_from_binlog_status ret = NO_GTIDS; bool done = false; bool seen_first_gtid = false; + my_off_t large_trx_end_offset = 0; + uint8_t large_trx_terminating_event_type = 0; while (!done && (ev = binlog_file_reader.read_event_object()) != nullptr) { #ifndef NDEBUG event_counter++; @@ -3914,6 +3428,23 @@ static enum_read_gtids_from_binlog_status read_gtids_from_binlog( case mysql::binlog::event::ROTATE_EVENT: // do nothing; just accept this event and go to next break; + case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT: { + /* + Large transaction optimization: record the offset and type of the + transaction's terminating event. After the Gtid event that follows + is read, the scan seeks past the transaction body straight to that + offset (see the seek logic further below). The offset is only + meaningful in a binary log; in a relay log it refers to the source's + file, so skip it there. + */ + if (!is_relay_log) { + const auto <h = + static_cast(*ev); + large_trx_end_offset = lth.get_terminating_event_offset(); + large_trx_terminating_event_type = lth.get_terminating_event_type(); + } + break; + } case mysql::binlog::event::PREVIOUS_GTIDS_LOG_EVENT: { ret = GOT_PREVIOUS_GTIDS; // add events to sets @@ -4043,7 +3574,66 @@ static enum_read_gtids_from_binlog_status read_gtids_from_binlog( if (ret == GOT_PREVIOUS_GTIDS && is_relay_log) done = true; break; } + const bool is_large_trx_header = + ev->get_type_code() == + mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT; + const bool is_gtid_event = + ev->get_type_code() == mysql::binlog::event::GTID_LOG_EVENT || + ev->get_type_code() == mysql::binlog::event::GTID_TAGGED_LOG_EVENT; + const my_off_t gtid_start = + is_gtid_event ? binlog_file_reader.event_start_pos() : 0; + const ulonglong trx_length = + is_gtid_event ? static_cast(ev)->get_trx_length() : 0; delete ev; + /* + Skip reading the transaction body when a large transaction header event + has been seen, using the offset of the terminating event that the header + recorded to seek to the transaction end. + + The header does not act on that offset when it is read: it waits until + reading the transaction's Gtid event, which carries the transaction + length the offset is checked against. So the header only records the + offset and it is consumed when a Gtid event is read, which is one event + after the header event. + + A large transaction in a promoted binlog file carries one header, + immediately followed by its Gtid. + */ + if (large_trx_end_offset != 0 && is_gtid_event && !done) { + const my_off_t target = large_trx_end_offset; + const uint8_t expected_type = large_trx_terminating_event_type; + large_trx_end_offset = 0; + large_trx_terminating_event_type = 0; + const bool trx_length_fits = + trx_length <= static_cast( + std::numeric_limits::max() - gtid_start); + if (trx_length_fits) { + const my_off_t expected_end = + gtid_start + static_cast(trx_length); + // seek() returns true on failure, so ERROR here means the seek itself + // failed. A rejected offset is not an error: the scan continues from + // where it is. + if (target > binlog_file_reader.position() /* must point forward */ && + is_valid_large_trx_terminating_event( + filename, target, binlog_file_reader.ifile()->length(), + expected_end, expected_type) && + binlog_file_reader.seek(target)) { + ret = ERROR; + done = true; + } + } + } else if (large_trx_end_offset != 0 && !is_large_trx_header) { + /* + Any event other than another large transaction header means the offset + does not describe what follows, so drop it. + + A large transaction in a promoted binlog file should carry only one + header. A second one is tolerated rather than treated as an error, + because every recorded offset is validated before use. + */ + large_trx_end_offset = 0; + large_trx_terminating_event_type = 0; + } DBUG_PRINT("info", ("done=%d", done)); } @@ -4505,15 +4095,33 @@ bool MYSQL_BIN_LOG::open_binlog( const char *log_name, const char *new_name, ulong max_size_arg, bool null_created_arg, bool need_lock_index, bool need_tsid_lock, Format_description_log_event *extra_description_event, - uint32 new_index_number) { + uint32 new_index_number, const char *promoted_log_name) { // lock_index must be acquired *before* tsid_lock. assert(need_tsid_lock || !need_lock_index); + /* + Only commit_large_transaction() supplies a promoted file name, and it + operates on the binary log. Rejected at runtime rather than asserted, + because publishing a spilled transaction file into the relay log sequence + would not be locally detectable. + */ + if (promoted_log_name != nullptr && is_relay_log) { + if (m_binlog_index_monitor.is_inited_purge_index_file()) { + (void)purge_index_entry(nullptr, nullptr, need_lock_index); + m_binlog_index_monitor.close_purge_index_file(); + } + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_RELAY_LOG_PROMOTION, promoted_log_name); + return true; + } DBUG_TRACE; DBUG_PRINT("enter", ("base filename: %s", log_name)); mysql_mutex_assert_owner(get_log_lock()); if (init_and_set_log_file_name(log_name, new_name, new_index_number)) { + if (promoted_log_name != nullptr) { + (void)purge_index_entry(nullptr, nullptr, need_lock_index); + m_binlog_index_monitor.close_purge_index_file(); + } LogErr(ERROR_LEVEL, ER_BINLOG_CANT_GENERATE_NEW_FILE_NAME); return true; } @@ -4522,10 +4130,15 @@ bool MYSQL_BIN_LOG::open_binlog( DEBUG_SYNC(current_thd, "after_log_file_name_initialized"); - if (m_binlog_index_monitor.open_purge_index_file(true) || - m_binlog_index_monitor.register_create_index_entry(log_file_name) || - m_binlog_index_monitor.sync_purge_index_file() || - DBUG_EVALUATE_IF("fault_injection_registering_index", 1, 0)) { + /* + For a promoted file, promote_spilled_file() creates the record in the purge + index file before it renames the temporary file. + */ + if (promoted_log_name == nullptr && + (m_binlog_index_monitor.open_purge_index_file(true) || + m_binlog_index_monitor.register_create_index_entry(log_file_name) || + m_binlog_index_monitor.sync_purge_index_file() || + DBUG_EVALUATE_IF("fault_injection_registering_index", 1, 0))) { /** @todo: although this was introduced to appease valgrind when injecting emulated faults using fault_injection_registering_index @@ -4541,14 +4154,21 @@ bool MYSQL_BIN_LOG::open_binlog( LogErr(ERROR_LEVEL, ER_BINLOG_FAILED_TO_SYNC_INDEX_FILE_IN_OPEN); return true; } + assert(promoted_log_name == nullptr || + m_binlog_index_monitor.is_inited_purge_index_file()); DBUG_EXECUTE_IF("crash_create_non_critical_before_update_index", DBUG_SUICIDE();); write_error = false; - /* open the main log file */ - if (open(m_key_file_log, log_name, new_name, new_index_number)) { + /* open the main log file. */ + if (open(m_key_file_log, log_name, new_name, new_index_number, + promoted_log_name != nullptr ? Binlog_file_mode::kOpenExisting + : Binlog_file_mode::kCreateNew)) { + if (m_binlog_index_monitor.is_inited_purge_index_file()) + (void)purge_index_entry(nullptr, nullptr, need_lock_index); m_binlog_index_monitor.close_purge_index_file(); + if (promoted_log_name != nullptr) my_delete(log_file_name, MYF(0)); return true; /* all warnings issued */ } @@ -4558,6 +4178,16 @@ bool MYSQL_BIN_LOG::open_binlog( Format_description_log_event s; + /* + A promoted binary log file already carries its header events (magic, + Format_description, Previous_gtids, Large_transaction_header, Gtid); + skip writing them and just register the file in the index. + */ + if (promoted_log_name != nullptr) { + write_file_name_to_index_file = true; + goto promoted_add_to_index; + } + if (m_binlog_file->is_empty()) { /* The binary log file was empty (probably newly created) @@ -4639,6 +4269,9 @@ bool MYSQL_BIN_LOG::open_binlog( if (is_relay_log) prev_gtids_ev.set_relay_log_event(); if (need_tsid_lock) tsid_lock->unlock(); if (write_event_to_binlog(&prev_gtids_ev)) goto err; + if (!is_relay_log) + update_binlog_temp_file_previous_gtids_size_estimate( + prev_gtids_ev.common_header->data_written); } else // !(current_thd) { /* @@ -4708,6 +4341,8 @@ bool MYSQL_BIN_LOG::open_binlog( goto err; bytes_written += extra_description_event->common_header->data_written; } + +promoted_add_to_index: if (m_binlog_file->flush_and_sync()) goto err; if (write_file_name_to_index_file) { @@ -4734,18 +4369,27 @@ bool MYSQL_BIN_LOG::open_binlog( } DBUG_EXECUTE_IF("crash_create_after_update_index", DBUG_SUICIDE();); + if (promoted_log_name != nullptr) + DBUG_EXECUTE_IF("crash_bolt_after_main_index_update", DBUG_SUICIDE();); } atomic_log_state = LOG_OPENED; /* At every rotate memorize the last transaction counter state to use it as - offset at logging the transaction logical timestamps. + offset at logging the transaction logical timestamps. For a promoted + file the tracker was rotated before its Gtid event was written. */ - m_dependency_tracker.rotate(); + if (promoted_log_name == nullptr) m_dependency_tracker.rotate(); m_binlog_index_monitor.close_purge_index_file(); + if (promoted_log_name != nullptr) + DBUG_EXECUTE_IF("crash_bolt_after_purge_index_remove", DBUG_SUICIDE();); - update_binlog_end_pos(); + /* + BOLT promotion defers this to promote_spilled_file(), which calls it after + the after_flush hook. + */ + if (promoted_log_name == nullptr) update_binlog_end_pos(); return false; err: @@ -4761,6 +4405,8 @@ bool MYSQL_BIN_LOG::open_binlog( LogErr(ERROR_LEVEL, ER_BINLOG_CANT_USE_FOR_LOGGING, (new_name) ? new_name : name, errno); close(LOG_CLOSE_INDEX, false, need_lock_index); + /* Undo the promotion rename before the caller relinquishes ownership. */ + if (promoted_log_name != nullptr) my_delete(log_file_name, MYF(0)); } return true; } @@ -5385,6 +5031,37 @@ int MYSQL_BIN_LOG::new_file_without_locking( return new_file_impl(false /*need_lock_log=false*/, extra_description_event); } +std::pair MYSQL_BIN_LOG::persist_gtids_on_rotate( + Gtid_persist_thd thd_mode) { + mysql_mutex_assert_owner(&LOCK_log); + m_binlog_index_monitor.assert_owner(); + + bool keep_current_binlog = false; + const bool use_client_thd_for_debug_readonly = + DBUG_EVALUATE_IF("gtid_executed_readonly", true, false); + THD *saved_current_thd = nullptr; + /* + Nulling current_thd is the mechanism that forces a dedicated THD: the + gtid_executed persister creates one when it finds none (see + Gtid_table_access_context::init). + */ + if (thd_mode == Gtid_persist_thd::kDedicated && + !use_client_thd_for_debug_readonly) { + saved_current_thd = current_thd; + current_thd = nullptr; + } + + const int error = gtid_state->save_gtids_of_last_binlog_into_table(); + if (saved_current_thd != nullptr) saved_current_thd->store_globals(); + + if (error == ER_RPL_GTID_TABLE_CANNOT_OPEN) { + keep_current_binlog = + m_binlog_file->get_real_file_size() < static_cast(max_size) && + !DBUG_EVALUATE_IF("simulate_max_binlog_size", true, false); + } + return {error, keep_current_binlog}; +} + /** Start writing to a new log file or reopen the old file. @@ -5443,15 +5120,17 @@ int MYSQL_BIN_LOG::new_file_impl( } if (!is_relay_log) { - /* Save set of GTIDs of the last binlog into table on binlog rotation */ - if ((error = gtid_state->save_gtids_of_last_binlog_into_table())) { + /* + Save the last binlog's GTID set before rotation. + */ + const auto [persist_error, keep_current_binlog] = + persist_gtids_on_rotate(Gtid_persist_thd::kCurrent); + error = persist_error; + if (error != 0) { if (error == ER_RPL_GTID_TABLE_CANNOT_OPEN) { - close_on_error = - m_binlog_file->get_real_file_size() >= - static_cast(max_size) || - DBUG_EVALUATE_IF("simulate_max_binlog_size", true, false); + close_on_error = !keep_current_binlog; - if (!close_on_error) { + if (keep_current_binlog) { LogErr(ERROR_LEVEL, ER_BINLOG_UNABLE_TO_ROTATE_GTID_TABLE_READONLY, "Current binlog file was flushed to disk and will be kept in " "use."); @@ -6775,7 +6454,10 @@ void MYSQL_BIN_LOG::close() {} */ int MYSQL_BIN_LOG::prepare(THD *thd, bool all) { DBUG_TRACE; - return m_tc_log_processing->prepare(this, thd, all); + DBUG_EXECUTE_IF("crash_before_tc_prepare", DBUG_SUICIDE();); + const int error = m_tc_log_processing->prepare(this, thd, all); + if (!error) DBUG_EXECUTE_IF("crash_after_tc_prepare", DBUG_SUICIDE();); + return error; } /** @@ -7047,14 +6729,22 @@ TC_LOG::enum_result MYSQL_BIN_LOG::commit(THD *thd, bool all) { return RESULT_ABORTED; } + binlog_cache_data *cache_to_promote = + get_cache_for_large_trx_commit(cache_mngr); + const bool used_bolt_promotion = cache_to_promote != nullptr; if (DBUG_EVALUATE_IF("simulate_xa_commit_log_inconsistency", true, false) || - ordered_commit(thd, all, skip_commit)) { + (used_bolt_promotion + ? commit_large_transaction(thd, all, skip_commit, cache_to_promote) + : ordered_commit(thd, all, skip_commit))) { thd_get_cache_mngr(thd)->reset(); if (thd->get_stmt_da()->is_ok()) thd->get_stmt_da()->reset_diagnostics_area(); return RESULT_INCONSISTENT; } + if (used_bolt_promotion) + DBUG_EXECUTE_IF("crash_bolt_after_binlog_commit", DBUG_SUICIDE();); + DBUG_EXECUTE_IF("ensure_binlog_cache_is_reset", { /* Assert that binlog cache is reset at commit time. */ assert(binlog_cache_is_reset); @@ -7124,41 +6814,6 @@ void MYSQL_BIN_LOG::reset_thread_caches(THD *thd) { return cache_mngr->reset(); } -void MYSQL_BIN_LOG::init_thd_variables(THD *thd, bool all, bool skip_commit) { - /* - These values are used while committing a transaction, so clear - everything. - - Notes: - - - It would be good if we could keep transaction coordinator - log-specific data out of the THD structure, but that is not the - case right now. - - - Everything in the transaction structure is reset when calling - ha_commit_low since that calls Transaction_ctx::cleanup. - */ - thd->tx_commit_pending = true; - thd->commit_error = THD::CE_NONE; - thd->next_to_commit = nullptr; - thd->durability_property = HA_IGNORE_DURABILITY; - thd->get_transaction()->m_flags.real_commit = all; - thd->get_transaction()->m_flags.xid_written = false; - thd->get_transaction()->m_flags.commit_low = !skip_commit; - thd->get_transaction()->m_flags.run_hooks = !skip_commit; -#ifndef NDEBUG - /* - The group commit Leader may have to wait for follower whose transaction - is not ready to be preempted. Initially the status is pessimistic. - Preemption guarding logics is necessary only when !NDEBUG is set. - It won't be required for the dbug-off case as long as the follower won't - execute any thread-specific write access code in this method, which is - the case as of current. - */ - thd->get_transaction()->m_flags.ready_preempt = false; -#endif -} - /** Commit a sequence of sessions. @@ -8086,9 +7741,9 @@ bool THD::binlog_configure_trx_cache_size(ulong new_size) { } // Close and reopen with new value - Binlog_cache_storage *const cache = cache_mngr->get_trx_cache(); - cache->close(); - return cache->open(new_size, max_binlog_cache_size); + binlog_cache_data *const cache_data = &cache_mngr->trx_cache; + cache_data->get_cache()->close(); + return cache_data->open(new_size, max_binlog_cache_size); } /** diff --git a/sql/binlog.h b/sql/binlog.h index 37885fa582fd..93a97f14cbab 100644 --- a/sql/binlog.h +++ b/sql/binlog.h @@ -47,8 +47,9 @@ #include "mysql/udf_registration_types.h" #include "mysql_com.h" // Item_result #include "sql/binlog/binlog_tc_log_processing.h" -#include "sql/binlog_index.h" // Log_info, Binlog_index -#include "sql/binlog_reader.h" // Binlog_file_reader +#include "sql/binlog/large_trx_commit.h" // Large_trx_fallback_reason +#include "sql/binlog_index.h" // Log_info, Binlog_index +#include "sql/binlog_reader.h" // Binlog_file_reader #include "sql/rpl_commit_stage_manager.h" #include "sql/rpl_trx_tracking.h" #include "sql/tc_log.h" // TC_LOG @@ -67,9 +68,12 @@ class Tsid_map; class THD; class Transaction_boundary_parser; class binlog_cache_data; +class binlog_cache_mngr; class user_var_entry; class Binlog_cache_storage; +binlog_cache_mngr *thd_get_cache_mngr(const THD *thd); + struct Gtid; typedef int64 query_id_t; @@ -112,6 +116,39 @@ class MYSQL_BIN_LOG : public TC_LOG { private: enum enum_log_state { LOG_OPENED, LOG_CLOSED, LOG_TO_BE_OPENED }; + /** + Which THD performs the mysql.gtid_executed write. + */ + enum class Gtid_persist_thd { + /** + A dedicated internal THD, created and dropped for the write. Required + when the caller is itself part-way through a commit: the write ends in + ha_commit_trans() on whichever THD performs it, which on the committing + session would commit that session's transaction early. + */ + kDedicated, + /** + The current THD. Only safe when no transaction is in flight on it, as + during an ordinary binary log rotation. + */ + kCurrent + }; + + /** + Whether open() creates a new binary log file or attaches to one that + already holds its header events. + */ + enum class Binlog_file_mode { + /** Create a fresh, empty file. */ + kCreateNew, + /** + The file already exists: open it and position at its end so that + subsequent transactions append. Used when a spilled temporary file is + promoted into the binary log sequence. + */ + kOpenExisting + }; + /* LOCK_log is inited by init_pthread_objects() */ mysql_mutex_t LOCK_log; char *name; @@ -249,8 +286,111 @@ class MYSQL_BIN_LOG : public TC_LOG { int new_file_impl(bool need_lock, Format_description_log_event *extra_description_event); + /** + Persist the current binary log's GTIDs before starting a new binary log. + The caller must hold LOCK_log and the binlog-index lock. + + @param thd_mode Which THD performs the mysql.gtid_executed write. A caller + that is itself inside a commit must pass kDedicated. + + @return {error, keep_current_binlog}. error is 0 on success, otherwise the + GTID persistence error. keep_current_binlog is meaningful only when + error is ER_RPL_GTID_TABLE_CANNOT_OPEN, where it reports that the + current binary log may stay in use. + */ + [[nodiscard]] std::pair persist_gtids_on_rotate( + Gtid_persist_thd thd_mode); + bool open(PSI_file_key log_file_key, const char *log_name, - const char *new_name, uint32 new_index_number); + const char *new_name, uint32 new_index_number, + Binlog_file_mode mode = Binlog_file_mode::kCreateNew); + + /** + Writes the file header of a promoted binary log file into the reserved + region at the head of a spilled temporary file: the binlog magic, a + Format_description event, a Previous_gtids event, a + Large_transaction_header event sized to fill the region exactly, and + the transaction's Gtid event, which ends precisely where the + transaction's first event was placed at spill time. + + Rotates the dependency tracker (the promoted file starts a new binlog + file) and assigns the transaction's GTID and logical timestamps, but + only after the reserved region is known to fit the header events. + Called with LOCK_log held. + + @param thd The committing session. + @param cache_data The transaction's (spilled) binlog cache. + @param file Descriptor of the temporary file. + + @return {error, fits}. error is true on serialization or I/O failure; fits + is false when the reserved region can no longer hold the header + events, which is a clean fallback rather than an error and leaves no + side effects. fits is meaningless when error is true. + */ + [[nodiscard]] std::pair write_promoted_binlog_header( + THD *thd, binlog_cache_data *cache_data, File file); + + /** + The possible outcome of promote_spilled_file(). + */ + enum class Promote_result { + kPromoted, ///< The file is in the binary log sequence and is active. + kFallback, ///< Nothing irreversible happened; use the standard path. + kError, ///< Fail the transaction per binlog_error_action. + }; + + struct Promote_outcome { + Promote_result result; + /// Valid only when result == kFallback; kNone otherwise. + Large_trx_fallback_reason reason; + /// True once the temporary file has been renamed to a binary log name, so + /// the cache must not delete it. + bool file_renamed; + }; + + /** + Promotes the transaction's spilled temporary file into the binary log + sequence as the next binary log file, so the transaction body is never + copied. + + In order: + + 1. Name the next binary log file. + 2. Take the binlog-index lock, and hold it for steps 3 to 8 so no rotation + or purge can interleave with them. + 3. Persist the current binary log's GTIDs into mysql.gtid_executed, as an + ordinary rotation does. + 4. Write the promoted file's header events -- Format_description, + Previous_gtids, Large_transaction_header and Gtid -- into the region + reserved at the front of the spilled file. The transaction body already + sits past that region, untouched. + 5. Record the new name in the purge index. That record owns the file until + step 8 puts it in the main index, including across a crash. + 6. Rename the spilled file to its binary log name and sync it. The cache + keeps its open descriptor across the rename. + 7. Write a Rotate event into the active binary log pointing at the + promoted file, sync it, then close the active binary log file. + 8. Open the promoted file as the active binary log, which adds it to the + main index. Ownership of the purge-index cleanup passes to open_binlog() + here, on success and on failure. + 9. Release the index lock, then publish the transaction's position and run + the after_flush hook. + + Steps 3 and 4 can decline rather than fail: neither has assigned a GTID or + written to the file yet, so the transaction can still commit through the + standard path. Step 5 onward is visible outside this session, so a failure + there deletes the renamed file and its purge-index entry instead. + + Called with LOCK_log and LOCK_commit held. Both are still held on return; + the caller releases them according to the outcome. + + @param thd The committing session. + @param cache_data The transaction's spilled binlog cache. + + @return the outcome; see Promote_outcome. + */ + [[nodiscard]] Promote_outcome promote_spilled_file( + THD *thd, binlog_cache_data *cache_data); bool init_and_set_log_file_name(const char *log_name, const char *new_name, uint32 new_index_number); int generate_new_name(char *new_name, const char *log_name, @@ -511,19 +651,6 @@ class MYSQL_BIN_LOG : public TC_LOG { bool change_stage(THD *thd, Commit_stage_manager::StageID stage, THD *queue, mysql_mutex_t *leave_mutex, mysql_mutex_t *enter_mutex); - /** - Set thread variables used while flushing a transaction. - - @param[in] thd thread whose variables need to be set - @param[in] all This is @c true if this is a real transaction commit, and - @c false otherwise. - @param[in] skip_commit - This is @c true if the call to @c ha_commit_low should - be skipped (it is handled by the caller somehow) and @c - false otherwise (the normal case). - */ - void init_thd_variables(THD *thd, bool all, bool skip_commit); - [[nodiscard]] int flush_cache_to_file(my_off_t *flush_end_pos); [[nodiscard]] std::pair flush_thread_caches(THD *thd); void handle_binlog_flush_or_sync_error(THD *thd, bool need_lock_log, @@ -716,12 +843,20 @@ class MYSQL_BIN_LOG : public TC_LOG { binary log files. @param new_index_number The binary log file index number to start from after the RESET BINARY LOGS AND GTIDS command is called. + @param promoted_log_name When a spilled large transaction commits by + promoting its temporary file into the binary log sequence, the name of + that file. It is opened at its end instead of a fresh file being created + because its header events are already in place. A non-NULL value also + means the caller has already renamed the file and registered it in the + durable purge index, so the recovery record stays in place until the + main-index update completes. This should be NULL otherwise. */ bool open_binlog(const char *log_name, const char *new_name, ulong max_size_arg, bool null_created_arg, bool need_lock_index, bool need_tsid_lock, Format_description_log_event *extra_description_event, - uint32 new_index_number = 0); + uint32 new_index_number = 0, + const char *promoted_log_name = nullptr); bool open_index_file(const char *index_file_name_arg, const char *log_name, bool need_lock_index); /* Use this to start writing a new log file */ @@ -742,6 +877,32 @@ class MYSQL_BIN_LOG : public TC_LOG { Binlog_event_writer *writer, bool parallelization_barrier); + /** + Commit a large transaction by promoting its spilled temporary file into + the binary log sequence: the file header events are written into the + reserved region at the head of the file, the file is renamed to become + the next binary log file, and the transaction is committed in the + engines. The commit work is constant regardless of the transaction + size: the transaction body, already in final binary log form in the + file, is never copied. + + If the reserved region cannot fit the header events, the transaction + falls back to ordered_commit(). + + @param thd The committing session. + @param all Is set in case of explicit commit + (COMMIT statement), or implicit commit issued by + a DDL. + @param skip_commit Is set in case of XA PREPARE, in which case the + commit in the engines is skipped. + @param cache_data The transaction's (spilled) binlog cache. + + @retval 0 Success. + @retval !=0 Error. + */ + int commit_large_transaction(THD *thd, bool all, bool skip_commit, + binlog_cache_data *cache_data); + /** Write a dml into statement cache and then flush it into binlog. It writes Gtid_log_event and BEGIN, COMMIT automatically. @@ -829,6 +990,15 @@ class MYSQL_BIN_LOG : public TC_LOG { const std::string &last); int rotate(bool force_rotate, bool *check_purge); + /** + Acquires LOCK_log and rotates the active binary log if it has grown past + max_binlog_size, then purges if the rotation asked for it. + + @return 0 on success, including when no rotation was needed; otherwise the + error from rotate(). + */ + int rotate_if_needed(); + /** @brief This function runs automatic purge if the conditions to meet automatic purge are met. Such conditions are: log is open, instance is not @@ -1036,6 +1206,16 @@ struct LOAD_FILE_INFO { extern MYSQL_PLUGIN_IMPORT MYSQL_BIN_LOG mysql_bin_log; +/** + Return the lock-free reservation based on the most recently serialized + Previous_gtids event size. +*/ +my_off_t get_binlog_temp_file_reserved_bytes(); + +/** Publish a newly serialized Previous_gtids event size for cache opens. */ +void update_binlog_temp_file_previous_gtids_size_estimate( + my_off_t previous_gtids_size); + /** Check if the the transaction is empty. diff --git a/sql/binlog/binlog_ofile.cc b/sql/binlog/binlog_ofile.cc index d76a426e8991..93aa096a42bf 100644 --- a/sql/binlog/binlog_ofile.cc +++ b/sql/binlog/binlog_ofile.cc @@ -153,6 +153,19 @@ bool MYSQL_BIN_LOG::Binlog_ofile::truncate(my_off_t offset) { return false; } +bool MYSQL_BIN_LOG::Binlog_ofile::position_at(my_off_t offset) { + assert(m_pipeline_head != nullptr); + /* + The caller passes a physical file offset (e.g. the promoted file's size) as + the logical position. Those two agree only for an unencrypted file. + */ + if (get_encrypted_header_size() != 0) return true; + + if (m_pipeline_head->seek(offset)) return true; + m_position = offset; + return false; +} + bool MYSQL_BIN_LOG::Binlog_ofile::flush() { return m_pipeline_head->flush(); } bool MYSQL_BIN_LOG::Binlog_ofile::sync() { return m_pipeline_head->sync(); } bool MYSQL_BIN_LOG::Binlog_ofile::flush_and_sync() { return flush() || sync(); } diff --git a/sql/binlog/binlog_ofile.h b/sql/binlog/binlog_ofile.h index f0680bb19970..62f9657c46be 100644 --- a/sql/binlog/binlog_ofile.h +++ b/sql/binlog/binlog_ofile.h @@ -109,6 +109,18 @@ class MYSQL_BIN_LOG::Binlog_ofile : public Basic_ostream { */ [[nodiscard]] virtual bool truncate(my_off_t offset); + /** + Seeks to an existing binlog offset so the next write appends there. + + An encrypted stream is rejected. + + @param[in] offset Logical offset for the next write. + + @retval false Success + @retval true Error, including when this stream is encrypted. + */ + [[nodiscard]] virtual bool position_at(my_off_t offset); + [[nodiscard]] virtual bool flush(); [[nodiscard]] virtual bool sync(); [[nodiscard]] virtual bool flush_and_sync(); diff --git a/sql/binlog/binlog_tc_log.cc b/sql/binlog/binlog_tc_log.cc index 3b0990d16e2e..b3b58e0d89ae 100644 --- a/sql/binlog/binlog_tc_log.cc +++ b/sql/binlog/binlog_tc_log.cc @@ -54,6 +54,11 @@ int Binlog_tc_log::prepare(MYSQL_BIN_LOG *binlog, THD *thd, bool all) { right before flushing them to binary log during binlog group commit flush stage. Reset to HA_REGULAR_DURABILITY at the beginning of parsing next command. + + This holds for every transaction, including one that goes on to commit + through the binlog large transaction optimization (BOLT). BOLT bypasses the + group-commit flush stage, it repays the same debt itself: see the + ha_flush_logs(true) call in MYSQL_BIN_LOG::commit_large_transaction(). */ thd->durability_property = HA_IGNORE_DURABILITY; diff --git a/sql/binlog/cache_data.cc b/sql/binlog/cache_data.cc new file mode 100644 index 000000000000..472e915dda44 --- /dev/null +++ b/sql/binlog/cache_data.cc @@ -0,0 +1,265 @@ +/** + @file + @brief Lifecycle and bookkeeping for the per-session binary log caches: the + transaction and statement caches a session accumulates events into before + commit. +*/ + +#include "sql/binlog/cache_data.h" +#include "sql/rpl_group_replication.h" // is_group_replication_running + +/* + binlog_cache_data +*/ + +void binlog_cache_data::latch_large_trx_optimization() { + if (!m_large_trx_optimization_latched) { + /* + The binlog large transaction optimization is unavailable while Group + Replication is running. This check is folded in here, rather than + reported as a fallback reason, because Group Replication status must be + known before the first event is serialized. + */ + m_large_trx_optimization_enabled = + opt_binlog_large_transaction_optimization_enabled && + !is_group_replication_running(); + m_large_trx_optimization_threshold = + opt_binlog_large_transaction_optimization_threshold; + m_large_trx_optimization_latched = true; + /* + Give the spill file a bolt_* format name only when the + optimization is enabled for this transaction, so a disabled knob leaves + no bolt_* files. This uses the same knob value captured here for + the promotion decision, so naming and eligibility always agree. The + reserved header region is applied regardless (see + IO_CACHE_binlog_cache_storage::open); only the naming is gated. + */ + m_cache.set_named_file(m_large_trx_optimization_enabled); + } +} + +void binlog_cache_data::cache_state_checkpoint(my_off_t pos_to_checkpoint) { + // We only need to store the cache state for pos > 0 + if (pos_to_checkpoint) { + cache_state state; + state.with_rbr = flags.with_rbr; + state.with_sbr = flags.with_sbr; + state.with_start = flags.with_start; + state.with_end = flags.with_end; + state.with_content = flags.with_content; + state.event_counter = m_event_counter; + cache_state_map[pos_to_checkpoint] = state; + } +} + +void binlog_cache_data::cache_state_rollback(my_off_t pos_to_rollback) { + if (pos_to_rollback) { + std::map::iterator it; + it = cache_state_map.find(pos_to_rollback); + if (it != cache_state_map.end()) { + flags.with_rbr = it->second.with_rbr; + flags.with_sbr = it->second.with_sbr; + flags.with_start = it->second.with_start; + flags.with_end = it->second.with_end; + flags.with_content = it->second.with_content; + m_event_counter = it->second.event_counter; + } else + assert(it == cache_state_map.end()); + } + // Rolling back to pos == 0 means cleaning up the cache. + else { + flags.with_rbr = false; + flags.with_sbr = false; + flags.with_start = false; + flags.with_end = false; + flags.with_content = false; + m_event_counter = 0; + m_large_trx_optimization_latched = false; + m_large_trx_optimization_enabled = false; + m_large_trx_optimization_threshold = 0; + m_checksum_alg_in_cache = mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF; + m_terminating_event_offset = 0; + m_terminating_event_type = mysql::binlog::event::UNKNOWN_EVENT; + } +} + +void binlog_cache_data::reset(bool preserve_spilled_file) { + compute_statistics(); + remove_pending_event(); + + if (m_cache.reset(preserve_spilled_file)) { + LogErr(WARNING_LEVEL, ER_BINLOG_CANT_RESIZE_CACHE); + } + + flags.with_xid = false; + flags.immediate = false; + flags.finalized = false; + flags.with_sbr = false; + flags.with_rbr = false; + flags.with_start = false; + flags.with_end = false; + flags.with_content = false; + + /* + The truncate function calls reinit_io_cache that calls my_b_flush_io_cache + which may increase disk_writes. This breaks the disk_writes use by the + binary log which aims to compute the ratio between in-memory cache usage + and disk cache usage. To avoid this undesirable behavior, we reset the + variable after truncating the cache. + */ + cache_state_map.clear(); + m_event_counter = 0; + m_checksum_alg_in_cache = mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF; + m_large_trx_optimization_latched = false; + m_large_trx_optimization_enabled = false; + m_large_trx_optimization_threshold = 0; + m_terminating_event_offset = 0; + m_terminating_event_type = mysql::binlog::event::UNKNOWN_EVENT; + m_compressed_size = 0; + m_decompressed_size = 0; + m_compression_type = mysql::binlog::event::compression::NONE; + assert(is_binlog_empty()); +} + +bool binlog_cache_data::has_empty_transaction() { + /* + The empty transaction has two events in trx/stmt binlog cache + and no changes: one is a transaction start and other is a transaction + end (there should be no SBR changing content and no RBR events). + */ + if (flags.with_start && // Has transaction start statement + flags.with_end && // Has transaction end statement + !flags.with_content) // Has no other content than START/END + { + assert(m_event_counter == 2); // Two events in the cache only + assert(!flags.with_sbr); // No statements changing content + assert(!flags.with_rbr); // No rows changing content + assert(!flags.immediate); // Not a DDL + assert(!flags.with_xid); // Not a XID trx and not an atomic DDL Query + return true; + } + return false; +} + +void binlog_cache_data::truncate(my_off_t pos) { + DBUG_PRINT("info", ("truncating to position %lu", (ulong)pos)); + remove_pending_event(); + + // TODO: check the return value. + (void)m_cache.truncate(pos); +} + +int binlog_cache_data::flush_pending_event(THD *thd) { + if (m_pending) { + m_pending->set_flags(Rows_log_event::STMT_END_F); + if (int error = write_event(m_pending)) return error; + thd->clear_binlog_table_maps(); + } + return 0; +} + +int binlog_cache_data::remove_pending_event() { + delete m_pending; + m_pending = nullptr; + return 0; +} + +void binlog_cache_data::compute_statistics() { + if (!is_binlog_empty()) { + (*ptr_binlog_cache_use)++; + if (m_cache.disk_writes() != 0) (*ptr_binlog_cache_disk_use)++; + } +} + +/* + binlog_trx_cache_data +*/ + +void binlog_trx_cache_data::reset(bool preserve_spilled_file) { + DBUG_TRACE; + DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + m_cannot_rollback = false; + before_stmt_pos = MY_OFF_T_UNDEF; + binlog_cache_data::reset(preserve_spilled_file); + DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + return; +} + +void binlog_trx_cache_data::set_prev_position(my_off_t pos) { + DBUG_TRACE; + DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + before_stmt_pos = pos; + cache_state_checkpoint(before_stmt_pos); + DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + return; +} + +void binlog_trx_cache_data::restore_prev_position() { + DBUG_TRACE; + DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + binlog_cache_data::truncate(before_stmt_pos); + cache_state_rollback(before_stmt_pos); + before_stmt_pos = MY_OFF_T_UNDEF; + /* + Binlog statement rollback clears with_xid now as the atomic DDL statement + marker which can be set as early as at event creation and caching. + */ + flags.with_xid = false; + DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + return; +} + +void binlog_trx_cache_data::restore_savepoint(my_off_t pos) { + DBUG_TRACE; + DBUG_PRINT("enter", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + binlog_cache_data::truncate(pos); + if (pos <= before_stmt_pos) before_stmt_pos = MY_OFF_T_UNDEF; + cache_state_rollback(pos); + DBUG_PRINT("return", ("before_stmt_pos: %llu", (ulonglong)before_stmt_pos)); + return; +} + +/* + binlog_cache_mngr +*/ + +bool binlog_cache_mngr::init() { + return stmt_cache.open(binlog_stmt_cache_size, max_binlog_stmt_cache_size) || + trx_cache.open(binlog_cache_size, max_binlog_cache_size); +} + +void binlog_cache_mngr::reset() { + if (!stmt_cache.is_binlog_empty()) stmt_cache.reset(); + if (!trx_cache.is_binlog_empty()) trx_cache.reset(); +} + +int binlog_cache_mngr::flush(THD *thd, my_off_t *bytes_written, + bool *wrote_xid) { + my_off_t stmt_bytes = 0; + my_off_t trx_bytes = 0; + assert(stmt_cache.has_xid() == 0); + + bool parallelization_barrier = false; + if (has_incident()) { + if (int error = handle_deferred_cache_write_incident(thd)) return error; + // Request force rotate + thd->rpl_thd_ctx.binlog_group_commit_ctx().set_force_rotate(); + // Set as parallelization_barrier so that dependency tracker marks all + // subsequent transactions to depend on it. + parallelization_barrier = true; + } + + int error = + stmt_cache.flush(thd, &stmt_bytes, wrote_xid, parallelization_barrier); + if (error) return error; + DEBUG_SYNC(thd, "after_flush_stm_cache_before_flush_trx_cache"); + error = trx_cache.flush(thd, &trx_bytes, wrote_xid, parallelization_barrier); + if (error) return error; + *bytes_written = stmt_bytes + trx_bytes; + return 0; +} + +bool binlog_cache_mngr::has_empty_transaction() { + return (trx_cache.is_empty_or_has_empty_transaction() && + stmt_cache.is_empty_or_has_empty_transaction() && !is_binlog_empty()); +} diff --git a/sql/binlog/cache_data.h b/sql/binlog/cache_data.h new file mode 100644 index 000000000000..288f680a3249 --- /dev/null +++ b/sql/binlog/cache_data.h @@ -0,0 +1,618 @@ +#ifndef BINLOG_CACHE_DATA_H_INCLUDED +#define BINLOG_CACHE_DATA_H_INCLUDED + +#include +#include +#include + +#include "my_dbug.h" +#include "my_inttypes.h" +#include "my_sys.h" +#include "mysql/binlog/event/binlog_event.h" +#include "mysql/components/services/log_builtins.h" +#include "mysqld_error.h" +#include "sql/binlog_ostream.h" +#include "sql/debug_sync.h" +#include "sql/log_event.h" +#include "sql/mysqld.h" // binlog_cache_size +#include "sql/sql_class.h" +#include "sql/xa.h" + +/** + @file + @brief The per-session binary log caches, where a session's binary log + events are buffered before they are written to a binary log file. + + binlog_cache_data + Base class. Serializes Log_events into its Binlog_cache_storage, which + buffers them in memory and spills to a temporary file once + binlog_cache_size is exceeded. + + binlog_stmt_cache_data + The statement cache, holding changes to non-transactional tables. + + binlog_trx_cache_data + The transaction cache, holding changes to transactional tables until + commit. + + binlog_cache_mngr + Owns one statement cache and one transaction cache per session, and + records a logging incident when events could not be cached. +*/ + +#define MY_OFF_T_UNDEF (~(my_off_t)0UL) + +/** + Caches for non-transactional and transactional data before writing + it to the binary log. + + @todo All the access functions for the flags suggest that the + encapsuling is not done correctly, so try to move any logic that + requires access to the flags into the cache. +*/ +class binlog_cache_data { + public: + binlog_cache_data(class binlog_cache_mngr &cache_mngr, bool trx_cache_arg, + ulong *ptr_binlog_cache_use_arg, + ulong *ptr_binlog_cache_disk_use_arg) + : m_cache_mngr(cache_mngr), + m_pending(nullptr), + ptr_binlog_cache_use(ptr_binlog_cache_use_arg), + ptr_binlog_cache_disk_use(ptr_binlog_cache_disk_use_arg) { + flags.transactional = trx_cache_arg; + } + + bool open(my_off_t cache_size, my_off_t max_cache_size); + + Binlog_cache_storage *get_cache() { return &m_cache; } + int finalize(THD *thd, Log_event *end_event); + int finalize(THD *thd, Log_event *end_event, XID_STATE *xs); + int flush(THD *thd, my_off_t *bytes, bool *wrote_xid, + bool parallelization_barrier); + int write_event(Log_event *event); + void set_event_counter(size_t event_counter) { + m_event_counter = event_counter; + } + size_t get_event_counter() const { return m_event_counter; } + size_t get_compressed_size() const { return m_compressed_size; } + size_t get_decompressed_size() const { return m_decompressed_size; } + mysql::binlog::event::compression::type get_compression_type() const { + return m_compression_type; + } + + void set_compressed_size(size_t s) { m_compressed_size = s; } + void set_decompressed_size(size_t s) { m_decompressed_size = s; } + void set_compression_type(mysql::binlog::event::compression::type t) { + m_compression_type = t; + } + + virtual ~binlog_cache_data() { + assert(is_binlog_empty()); + m_cache.close(); + } + + bool is_binlog_empty() const { + DBUG_PRINT("debug", ("%s_cache - pending: 0x%llx, bytes: %llu", + (flags.transactional ? "trx" : "stmt"), + (ulonglong)pending(), (ulonglong)m_cache.length())); + return pending() == nullptr && m_cache.is_empty(); + } + + bool is_finalized() const { return flags.finalized; } + + Rows_log_event *pending() const { return m_pending; } + + void set_pending(Rows_log_event *const pending) { m_pending = pending; } + + /// @see handle_deferred_cache_write_incident + void set_incident( + std::string_view incident_message = + "Non-transactional changes were not written to the binlog."); + + /// @see handle_deferred_cache_write_incident + bool has_incident(void) const; + + bool has_xid() const { + // There should only be an XID event if we are transactional + assert((flags.transactional && flags.with_xid) || !flags.with_xid); + return flags.with_xid; + } + + bool is_trx_cache() const { return flags.transactional; } + + /** + Returns the checksum algorithm the events of this cache are + serialized with, recorded at the transaction's first event (see + write_event). + + @return The algorithm, or BINLOG_CHECKSUM_ALG_UNDEF when no checksum is + written into this cache. + */ + mysql::binlog::event::enum_binlog_checksum_alg checksum_alg_in_cache() const { + return m_checksum_alg_in_cache; + } + + /** + Returns true when the events in this cache carry a checksum + (see write_event). + + @retval true The events carry their own checksum. + @retval false They do not, so a checksum is added when the cache is + copied into the binary log. + */ + bool is_checksum_computed() const { + return m_checksum_alg_in_cache != + mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF && + m_checksum_alg_in_cache != + mysql::binlog::event::BINLOG_CHECKSUM_ALG_OFF; + } + + /** + Captures the large transaction optimization's settings for the whole + transaction, on the first event written to the cache. + + Called on every write but effective only once, so a change to + binlog_large_transaction_optimization_enabled or + binlog_large_transaction_optimization_threshold mid-transaction + cannot change how that transaction commits. + + The captured values are read back through large_trx_optimization_enabled() + and large_trx_optimization_threshold(). + */ + void latch_large_trx_optimization(); + + /** + @return Whether this transaction may use the binlog large transaction + optimization, as captured by latch_large_trx_optimization() at the + transaction's first event. + */ + bool large_trx_optimization_enabled() const { + return m_large_trx_optimization_enabled; + } + + /** + @return The spilled size, in bytes, above which this transaction is a + promotion candidate, as captured by latch_large_trx_optimization() + at the transaction's first event. + */ + ulonglong large_trx_optimization_threshold() const { + return m_large_trx_optimization_threshold; + } + + /** + Returns where the transaction's terminating event begins in a promoted + binary log file, so recovery can seek to it. Recorded by finalize(), + together with the event's type. + + @return The byte offset from the start of the promoted file, or 0 when the + transaction has no terminating event. Only a real end event + (COMMIT / XID / XA_PREPARE) counts; an immediately-logged + statement finalizes without one. + */ + my_off_t terminating_event_offset() const { + return m_terminating_event_offset; + } + + /** + Returns the type of the transaction's terminating event in a promoted + binary log file. Recorded by finalize(), together with its offset. + + @return The event type, meaningful only when terminating_event_offset() is + nonzero. + */ + mysql::binlog::event::Log_event_type terminating_event_type() const { + return m_terminating_event_type; + } + + my_off_t get_byte_position() const { return m_cache.length(); } + + void cache_state_checkpoint(my_off_t pos_to_checkpoint); + + void cache_state_rollback(my_off_t pos_to_rollback); + + /** + Reset the cache to unused state when the transaction is finished. It + drops all data and clears the transaction flags. + + @param preserve_spilled_file When true, the spilled file is retained + rather than deleted, because the caller has promoted it into the + binary log sequence and now owns it. + */ + virtual void reset(bool preserve_spilled_file = false); + + /** + Returns information about the cache content with respect to + the binlog_format of the events. + + This will be used to set a flag on GTID_LOG_EVENT stating that the + transaction may have SBR statements or not, but the binlog dump + will show this flag as "rbr_only" when it is not set. That's why + an empty transaction should return true below, or else an empty + transaction would be assumed as "rbr_only" even not having RBR + events. + + When dumping a binary log content using mysqlbinlog client program, + for any transaction assumed as "rbr_only" it will be printed a + statement changing the transaction isolation level to READ COMMITTED. + It doesn't make sense to have an empty transaction "requiring" this + isolation level change. + + @return true The cache have SBR events or is empty. + @return false The cache contains a transaction with no SBR events. + */ + bool may_have_sbr_stmts() { return flags.with_sbr || !flags.with_rbr; } + + /** + Check if the binlog cache contains an empty transaction, which has + two binlog events "BEGIN" and "COMMIT". + + @return true The binlog cache contains an empty transaction. + @return false Otherwise. + */ + bool has_empty_transaction(); + + /** + Check if the binlog cache is empty or contains an empty transaction, + which has two binlog events "BEGIN" and "COMMIT". + + @return true The binlog cache is empty or contains an empty transaction. + @return false Otherwise. + */ + bool is_empty_or_has_empty_transaction() { + return is_binlog_empty() || has_empty_transaction(); + } + + protected: + /* + This structure should have all cache variables/flags that should be restored + when a ROLLBACK TO SAVEPOINT statement be executed. + */ + struct cache_state { + bool with_sbr; + bool with_rbr; + bool with_start; + bool with_end; + bool with_content; + size_t event_counter; + }; + /* + For every SAVEPOINT used, we will store a cache_state for the current + binlog cache position. So, if a ROLLBACK TO SAVEPOINT is used, we can + restore the cache_state values after truncating the binlog cache. + */ + std::map cache_state_map; + /* + In order to compute the transaction size (because of possible extra checksum + bytes), we need to keep track of how many events are in the binlog cache. + */ + size_t m_event_counter = 0; + + /** + Checksum algorithm the events of this cache are serialized with, + recorded at the transaction's first event (see write_event). Stays + BINLOG_CHECKSUM_ALG_UNDEF while no checksum is written into this cache, + which is the case for the statement cache and whenever the large + transaction optimization is disabled. + */ + mysql::binlog::event::enum_binlog_checksum_alg m_checksum_alg_in_cache = + mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF; + + /** + Whether the capture has happened, so that only the transaction's first + event decides and later events leave the values alone. + */ + bool m_large_trx_optimization_latched = false; + /** + Whether this transaction may use the large transaction optimization: the + value of binlog_large_transaction_optimization_enabled at the transaction's + first event. + + Group Replication check is folded in here rather than + reported as a fallback reason, because it must be known before the first + event is serialized. + */ + bool m_large_trx_optimization_enabled = false; + /** + Value of binlog_large_transaction_optimization_threshold at the + transaction's first event, in bytes. Compared against the spilled size at + commit. + */ + ulonglong m_large_trx_optimization_threshold = 0; + + /** + Offset of the transaction's terminating event in a promoted binary log + file, recorded at finalize() together with its type. Stays 0 when the + transaction has no terminating event. + */ + my_off_t m_terminating_event_offset = 0; + /** + Type of the transaction's terminating event in a promoted binary log file, + recorded at finalize() together with its offset. + */ + mysql::binlog::event::Log_event_type m_terminating_event_type = + mysql::binlog::event::UNKNOWN_EVENT; + + size_t m_compressed_size = 0; + size_t m_decompressed_size = 0; + mysql::binlog::event::compression::type m_compression_type = + mysql::binlog::event::compression::type::NONE; + /* + It truncates the cache to a certain position. This includes deleting the + pending event. It corresponds to rollback statement or rollback to + a savepoint. It doesn't change transaction state. + */ + void truncate(my_off_t pos); + + /** + Flush pending event to the cache buffer. + */ + int flush_pending_event(THD *thd); + + /** + Remove the pending event. + */ + int remove_pending_event(); + struct Flags { + /* + Defines if this is either a trx-cache or stmt-cache, respectively, a + transactional or non-transactional cache. + */ + bool transactional : 1; + + /* + This indicates that the cache should be written without BEGIN/END. + */ + bool immediate : 1; + + /* + This flag indicates that the buffer was finalized and has to be + flushed to disk. + */ + bool finalized : 1; + + /* + This indicates that either the cache contain an XID event, or it's + an atomic DDL Query-log-event. In the latter case the flag is set up + on the statement level, namely when the Query-log-event is cached + at time the DDL transaction is not committing. + The flag therefore gets reset when the cache is cleaned due to + the statement rollback, e.g in case of a DDL post-caching execution + error. + Any statement scope flag among other things must consider its + reset policy when the statement is rolled back. + */ + bool with_xid : 1; + + /* + This indicates that the cache contain statements changing content. + */ + bool with_sbr : 1; + + /* + This indicates that the cache contain RBR event changing content. + */ + bool with_rbr : 1; + + /* + This indicates that the cache contain s transaction start statement. + */ + bool with_start : 1; + + /* + This indicates that the cache contain a transaction end event. + */ + bool with_end : 1; + + /* + This indicates that the cache contain content other than START/END. + */ + bool with_content : 1; + } flags; + + /// Compress the current transaction "in-place", if possible + /// + /// This attempts to compress the transaction if it satisfies the + /// necessary pre-conditions. Otherwise it does nothing. + /// + /// @retval true Error: the cache has been corrupted and the + /// transaction must be aborted. + /// + /// @retval false Success: the transaction was either compressed + /// successfully, or compression was not attempted, or compression + /// failed and left the uncompressed transaction intact. + [[nodiscard]] bool compress(THD *thd); + + private: + /* + Reference to the cache_mngr which owns this cache. + */ + class binlog_cache_mngr &m_cache_mngr; + + /* + Storage for byte data. This binlog_cache_data will serialize + events into bytes and put them into m_cache. + */ + Binlog_cache_storage m_cache; + + /* + Pending binrows event. This event is the event where the rows are currently + written. + */ + Rows_log_event *m_pending; + + /** + This function computes binlog cache and disk usage. + */ + void compute_statistics(); + + /* + Stores a pointer to the status variable that keeps track of the in-memory + cache usage. This corresponds to either + . binlog_cache_use or binlog_stmt_cache_use. + */ + ulong *ptr_binlog_cache_use; + + /* + Stores a pointer to the status variable that keeps track of the disk + cache usage. This corresponds to either + . binlog_cache_disk_use or binlog_stmt_cache_disk_use. + */ + ulong *ptr_binlog_cache_disk_use; + + binlog_cache_data &operator=(const binlog_cache_data &info); + binlog_cache_data(const binlog_cache_data &info); +}; + +class binlog_stmt_cache_data : public binlog_cache_data { + public: + binlog_stmt_cache_data(binlog_cache_mngr &cache_mngr, bool trx_cache_arg, + ulong *ptr_binlog_cache_use_arg, + ulong *ptr_binlog_cache_disk_use_arg) + : binlog_cache_data(cache_mngr, trx_cache_arg, ptr_binlog_cache_use_arg, + ptr_binlog_cache_disk_use_arg) {} + + using binlog_cache_data::finalize; + + int finalize(THD *thd); +}; + +class binlog_trx_cache_data : public binlog_cache_data { + public: + binlog_trx_cache_data(binlog_cache_mngr &cache_mngr, bool trx_cache_arg, + ulong *ptr_binlog_cache_use_arg, + ulong *ptr_binlog_cache_disk_use_arg) + : binlog_cache_data(cache_mngr, trx_cache_arg, ptr_binlog_cache_use_arg, + ptr_binlog_cache_disk_use_arg), + m_cannot_rollback(false), + before_stmt_pos(MY_OFF_T_UNDEF) {} + + void reset(bool preserve_spilled_file = false) override; + + bool cannot_rollback() const { return m_cannot_rollback; } + + void set_cannot_rollback() { m_cannot_rollback = true; } + + my_off_t get_prev_position() const { return before_stmt_pos; } + + void set_prev_position(my_off_t pos); + + void restore_prev_position(); + + void restore_savepoint(my_off_t pos); + + using binlog_cache_data::truncate; + + void truncate(THD *thd, bool all); + + private: + /* + It will be set true if any statement which cannot be rolled back safely + is put in trx_cache. + */ + bool m_cannot_rollback; + + /* + Binlog position before the start of the current statement. + */ + my_off_t before_stmt_pos; + + binlog_trx_cache_data &operator=(const binlog_trx_cache_data &info); + binlog_trx_cache_data(const binlog_trx_cache_data &info); +}; + +class binlog_cache_mngr { + /// Indicates that some events did not get into the cache(s) and most + /// likely it is incomplete. @see handle_deferred_cache_write_incident + std::string m_incident; + + public: +#ifndef NDEBUG + /// The number of times that the incident status has been set due to the + /// debug symbol binlog_inject_incident. + int m_injected_incident_count{0}; +#endif + + binlog_cache_mngr(ulong *ptr_binlog_stmt_cache_use_arg, + ulong *ptr_binlog_stmt_cache_disk_use_arg, + ulong *ptr_binlog_cache_use_arg, + ulong *ptr_binlog_cache_disk_use_arg) + : stmt_cache(*this, false, ptr_binlog_stmt_cache_use_arg, + ptr_binlog_stmt_cache_disk_use_arg), + trx_cache(*this, true, ptr_binlog_cache_use_arg, + ptr_binlog_cache_disk_use_arg) {} + + bool init(); + + binlog_cache_data *get_binlog_cache_data(bool is_transactional) { + if (is_transactional) + return &trx_cache; + else + return &stmt_cache; + } + + Binlog_cache_storage *get_stmt_cache() { return stmt_cache.get_cache(); } + Binlog_cache_storage *get_trx_cache() { return trx_cache.get_cache(); } + /** + Convenience method to check if both caches are empty. + */ + bool is_binlog_empty() const { + return stmt_cache.is_binlog_empty() && trx_cache.is_binlog_empty(); + } + + int handle_deferred_cache_write_incident(THD *thd); + + /// Check if either of the caches have an incident + /// @see handle_deferred_cache_write_incident + bool has_incident() const { return !m_incident.empty(); } + + void set_incident(std::string_view incident_message) { + assert(!incident_message.empty()); + m_incident = incident_message; + } + + /* + clear stmt_cache and trx_cache if they are not empty + */ + void reset(); + +#ifndef NDEBUG + bool dbug_any_finalized() const { + return stmt_cache.is_finalized() || trx_cache.is_finalized(); + } +#endif + + /** + Convenience method to flush both caches to the binary log. + + @param thd The session owning the caches being flushed. + @param bytes_written Pointer to variable that will be set to the + number of bytes written for the flush. + @param wrote_xid Pointer to variable that will be set to @c + true if any XID event was written to the + binary log. Otherwise, the variable will not + be touched. + @return Error code on error, zero if no error. + */ + int flush(THD *thd, my_off_t *bytes_written, bool *wrote_xid); + + /** + Check if at least one of transactions and statement binlog caches + contains an empty transaction, other one is empty or contains an + empty transaction. + + @return true At least one of transactions and statement binlog + caches an empty transaction, other one is empty + or contains an empty transaction. + @return false Otherwise. + */ + bool has_empty_transaction(); + + binlog_stmt_cache_data stmt_cache; + binlog_trx_cache_data trx_cache; + + private: + binlog_cache_mngr &operator=(const binlog_cache_mngr &info); + binlog_cache_mngr(const binlog_cache_mngr &info); +}; + +#endif // BINLOG_CACHE_DATA_H_INCLUDED diff --git a/sql/binlog/large_trx_commit.cc b/sql/binlog/large_trx_commit.cc new file mode 100644 index 000000000000..f95c2fb62e6e --- /dev/null +++ b/sql/binlog/large_trx_commit.cc @@ -0,0 +1,726 @@ +#include "sql/binlog/large_trx_commit.h" +#include "sql/binlog/transaction_commit_helper.h" + +#include +#include +#include +#include + +#include "mutex_lock.h" // MUTEX_LOCK +#include "my_dbug.h" +#include "my_dir.h" +#include "my_sys.h" +#include "my_systime.h" // my_micro_time +#include "mysql/binlog/event/binlog_event.h" +#include "mysql/binlog/event/control_events.h" +#include "mysql/psi/mysql_file.h" +#include "mysqld_error.h" // ER_GNO_EXHAUSTED +#include "scope_guard.h" // create_scope_guard +#include "sql/basic_ostream.h" // StringBuffer_ostream +#include "sql/binlog.h" +#include "sql/binlog/binlog_ofile.h" // MYSQL_BIN_LOG::Binlog_ofile +#include "sql/binlog/cache_data.h" // binlog_cache_data +#include "sql/binlog_ostream.h" +#include "sql/current_thd.h" +#include "sql/derror.h" // ER_THD +#include "sql/handler.h" // ha_flush_logs +#include "sql/log_event.h" +#include "sql/mysqld.h" +#include "sql/rpl_group_replication.h" // is_group_replication_running +#include "sql/rpl_gtid.h" // gtid_state +#include "sql/rpl_handler.h" // RUN_HOOK +#include "sql/rpl_log_encryption.h" // rpl_encryption +#include "sql/rpl_replica_commit_order_manager.h" // Commit_order_manager +#include "sql/rpl_trx_tracking.h" // Transaction_dependency_tracker +#include "sql/sql_class.h" +#include "sql/transaction_info.h" + +/** + @file + @brief The commit path for the binary log large transaction optimization: + deciding whether a transaction may use it, promoting its spilled temporary + file into the binary log sequence, and reporting the reason when a candidate + transaction has to fall back to the standard commit path. +*/ + +using mysql::binlog::event::enum_binlog_checksum_alg; + +namespace { + +/** + Slot 0 corresponds to kNone and is never logged to the error log; + record_large_trx_fallback() rejects that value. Entries must be string + literals so .data() is NUL-terminated for the %s in + ER_BINLOG_BOLT_LARGE_TRX_FALLBACK. +*/ +constexpr std::array kFallbackDetails{ + // kNone (never logged, record_large_trx_fallback() rejects it) + "no fallback", + // kStatementCache + "the statement cache is nonempty", + // kNonRowFormat + "the transaction contains non-ROW events", + // kEncryption + "binary log encryption was enabled while the transaction was running", + // kCompression + "binary log transaction compression is enabled", + // kChecksumMismatch + "the transaction's checksum algorithm does not match binlog_checksum", + // kReservedHeaderSpace + "the reserved header region is too small", + // kGtidPersistence + "GTID persistence prevented promotion; attempting the standard " + "commit path", + // kIncident + "the transaction has a logging incident to report", + // kGroupReplication + "Group Replication started while the transaction was open", + // kLogClosed + "the binary log was closed while the transaction was running", +}; + +static_assert(kFallbackDetails.size() == + static_cast(Large_trx_fallback_reason::kLogClosed) + + 1, + "add a message when adding a fallback reason"); + +/** + Assess whether this transaction can use the optimized commit path. + The BOLT knob and threshold are captured at the + transaction's first event, so a mid-transaction change to either is ignored. + + @param cache_mngr The session's binlog cache manager. + + @retval true The transaction is a promotion candidate: BOLT was enabled at + its first event, its transaction cache spilled, and the + spilled size exceeds the captured threshold. + @retval false It is not, so it commits through the standard path without + being counted as a missed optimization. +*/ +bool is_large_trx_commit_candidate(binlog_cache_mngr *cache_mngr) { + binlog_cache_data *trx_cache = &cache_mngr->trx_cache; + return trx_cache->large_trx_optimization_enabled() && + trx_cache->get_cache()->is_spilled() && + /* + Only a named spilled file can be promoted. + */ + trx_cache->get_cache()->tmp_file_name() != nullptr && + trx_cache->get_byte_position() > + trx_cache->large_trx_optimization_threshold(); +} + +/** + For a candidate transaction, the first condition that prevents it from + committing through the BOLT path, or kNone when none does. When several + conditions apply only the first one checked is returned; the reason is a + diagnostic hint, not an exhaustive list. + + @param cache_mngr The session's binlog cache manager. + + @return The blocking reason, or Large_trx_fallback_reason::kNone when the + transaction may be promoted. +*/ +Large_trx_fallback_reason large_trx_commit_blocker( + binlog_cache_mngr *cache_mngr) { + binlog_cache_data *trx_cache = &cache_mngr->trx_cache; + if (!cache_mngr->stmt_cache.is_binlog_empty()) + return Large_trx_fallback_reason::kStatementCache; + /* + A pending incident is normally materialized and force-rotated by the + group-commit flush, which the optimized path bypasses. Fall back to the + standard commit path so the incident is written and the replica is notified. + */ + if (cache_mngr->has_incident()) return Large_trx_fallback_reason::kIncident; + if (trx_cache->may_have_sbr_stmts()) + return Large_trx_fallback_reason::kNonRowFormat; + /* + Only promote when the spilled file is not encrypted and binlog_encryption is + OFF. binlog_encryption can be turned on between this check and when the + promotion actually happens. promote_spilled_file() checks it again under + LOCK_log to double confirm the encryption is not enabled. + */ + if (trx_cache->get_cache()->is_encrypted() || rpl_encryption.is_enabled()) + return Large_trx_fallback_reason::kEncryption; + if (trx_cache->get_compression_type() != + mysql::binlog::event::compression::NONE) + return Large_trx_fallback_reason::kCompression; + if (trx_cache->checksum_alg_in_cache() != + static_cast(binlog_checksum_options)) + return Large_trx_fallback_reason::kChecksumMismatch; + /* + Reached only when Group Replication started after this transaction's first + event. Falling back prevents the local promotion, but it does not make the + transaction safe for Group Replication to send: its cached events already + carry checksums, and the before_commit observer copies the cache exactly + as it is into the message it broadcasts, so those checksums travel with it. + */ + if (is_group_replication_running()) + return Large_trx_fallback_reason::kGroupReplication; + return Large_trx_fallback_reason::kNone; +} + +} // namespace + +void record_large_trx_fallback(Large_trx_fallback_reason reason) { + assert(reason != Large_trx_fallback_reason::kNone); + if (reason == Large_trx_fallback_reason::kNone) return; + + /* Atomic so concurrent sessions can increment this counter. */ + binlog_large_transaction_optimization_missed_count.fetch_add( + 1, std::memory_order_relaxed); + LogErr(WARNING_LEVEL, ER_BINLOG_BOLT_LARGE_TRX_FALLBACK, + kFallbackDetails[static_cast(reason)].data()); +} + +binlog_cache_data *get_cache_for_large_trx_commit( + binlog_cache_mngr *cache_mngr) { + /* + This is the fast path out for small transactions. Such a transaction was + never a candidate, so it is not a missed optimization and nothing is + recorded here, unlike the checks below. + */ + if (!is_large_trx_commit_candidate(cache_mngr)) return nullptr; + + DBUG_EXECUTE_IF("force_large_trx_compression_fallback", { + record_large_trx_fallback(Large_trx_fallback_reason::kCompression); + return nullptr; + }); + + const Large_trx_fallback_reason reason = large_trx_commit_blocker(cache_mngr); + if (reason != Large_trx_fallback_reason::kNone) { + record_large_trx_fallback(reason); + return nullptr; + } + return &cache_mngr->trx_cache; +} + +my_off_t large_trx_header_event_min_length(my_off_t checksum_len) { + return LOG_EVENT_HEADER_LEN + + mysql::binlog::event::Large_transaction_header_event:: + kFixedBodyLength + + checksum_len; +} + +bool large_trx_header_events_fit(my_off_t prefix_length, my_off_t checksum_len, + my_off_t reserved) { + return prefix_length + large_trx_header_event_min_length(checksum_len) + + mysql::binlog::event::Gtid_event::get_max_event_length() + + checksum_len <= + reserved; +} + +my_off_t large_trx_header_event_padding(my_off_t reserved, + my_off_t gtid_event_length, + my_off_t prefix_length, + my_off_t checksum_len) { + const my_off_t min_length = large_trx_header_event_min_length(checksum_len); + const my_off_t lth_length = + reserved - (gtid_event_length + checksum_len) - prefix_length; + /* + Cannot underflow: large_trx_header_events_fit() budgeted + Gtid_event::get_max_event_length(), which is a compile-time upper bound on + gtid_event_length, so lth_length is at least min_length. + */ + assert(lth_length >= min_length); + return lth_length - min_length; +} + +std::pair MYSQL_BIN_LOG::write_promoted_binlog_header( + THD *thd, binlog_cache_data *cache_data, File file) { + DBUG_TRACE; + mysql_mutex_assert_owner(&LOCK_log); + + const enum_binlog_checksum_alg checksum_alg = + cache_data->checksum_alg_in_cache(); + assert(checksum_alg != mysql::binlog::event::BINLOG_CHECKSUM_ALG_UNDEF); + const my_off_t checksum_len = + checksum_alg != mysql::binlog::event::BINLOG_CHECKSUM_ALG_OFF + ? BINLOG_CHECKSUM_LEN + : 0; + const my_off_t reserved = cache_data->get_cache()->reserved_bytes(); + + /* + The header events are serialized into memory first. + */ + constexpr int kPromotedHeaderBlockInitialSize = 1024; + StringBuffer_ostream block; + + if (block.write(pointer_cast(BINLOG_MAGIC), + BIN_LOG_HEADER_SIZE)) + return {true, false}; + + /* The Format_description event, with rotation semantics (created = 0), + so replicas do not treat the promoted file as a server restart. */ + Format_description_log_event fde; + fde.common_header->flags |= LOG_EVENT_BINLOG_IN_USE_F; + fde.dont_set_created = true; + if (!fde.is_valid()) return {true, false}; + fde.common_footer->checksum_alg = checksum_alg; + fde.common_header->log_pos = block.length(); + if (binary_event_serialize(&fde, &block)) return {true, false}; + + /* The Previous_gtids event. The snapshot is stable: transactions commit + under LOCK_commit, which the caller holds. */ + { + Gtid_set logged_gtids_binlog(global_tsid_map, global_tsid_lock); + global_tsid_lock->wrlock(); + const Gtid_set *executed_gtids = gtid_state->get_executed_gtids(); + const Gtid_set *gtids_only_in_table = gtid_state->get_gtids_only_in_table(); + if (logged_gtids_binlog.add_gtid_set(executed_gtids) != RETURN_STATUS_OK) { + global_tsid_lock->unlock(); + return {true, false}; + } + logged_gtids_binlog.remove_gtid_set(gtids_only_in_table); + Previous_gtids_log_event prev_gtids_ev(&logged_gtids_binlog); + global_tsid_lock->unlock(); + const my_off_t previous_gtids_start = block.length(); + prev_gtids_ev.common_footer->checksum_alg = checksum_alg; + prev_gtids_ev.common_header->log_pos = previous_gtids_start; + if (binary_event_serialize(&prev_gtids_ev, &block)) return {true, false}; + update_binlog_temp_file_previous_gtids_size_estimate(block.length() - + previous_gtids_start); + } + + /* + Check that the reserved region fits the remaining header events: the + Large_transaction_header event at its minimum size and the Gtid event + at its maximum. If not, nothing has been assigned or written yet and + the transaction can still fall back to the standard commit path. + */ + bool header_events_fit = + large_trx_header_events_fit(block.length(), checksum_len, reserved); + /* + Debug hook to force the "reserved region too small" fallback in tests. + Placed here, before any GTID assignment or file I/O, so it takes the same + no-side-effects fallback path as the real check just below. + */ + DBUG_EXECUTE_IF("force_large_trx_reserved_header_fallback", + header_events_fit = false;); + if (!header_events_fit) return {false, false}; + + /* + Commit point of the promotion. The promoted file starts a new binary + log file: rotate the dependency tracker before generating the + transaction's logical timestamps, so they restart for the new file + (open_binlog() must then not rotate it again). The transaction is a + parallelization barrier for the replica's parallel applier. + */ + m_dependency_tracker.rotate(); + thd->get_transaction()->sequence_number = m_dependency_tracker.step(); + + assert(thd->next_to_commit == nullptr); + if (assign_automatic_gtids_to_flush_group(thd)) return {true, false}; + + Transaction_gtid_header metadata{thd, true /*parallelization_barrier*/, + &m_dependency_tracker}; + + Gtid_log_event gtid_event( + thd, cache_data->is_trx_cache(), metadata.last_committed(), + metadata.sequence_number(), cache_data->may_have_sbr_stmts(), + metadata.original_commit_timestamp(), + metadata.immediate_commit_timestamp(), metadata.original_server_version(), + metadata.immediate_server_version()); + gtid_event.set_trx_length_by_cache_size( + cache_data->get_byte_position(), checksum_len != 0, + cache_data->is_checksum_computed(), cache_data->get_event_counter()); + + /* + The Large_transaction_header event's padding is sized so the Gtid + event ends exactly at the reserved offset, where the transaction's + first event was placed at spill time. + */ + Large_transaction_header_log_event lth_event( + thd, cache_data->terminating_event_offset(), + cache_data->terminating_event_type(), + large_trx_header_event_padding(reserved, gtid_event.get_event_length(), + block.length(), + checksum_len) /*padding_size*/); + lth_event.common_footer->checksum_alg = checksum_alg; + lth_event.common_header->log_pos = block.length(); + if (binary_event_serialize(<h_event, &block)) return {true, false}; + + gtid_event.common_footer->checksum_alg = checksum_alg; + gtid_event.common_header->log_pos = block.length(); + if (binary_event_serialize(>id_event, &block)) return {true, false}; + assert(block.length() == reserved); + + /* Fill the reserved region and make the complete file durable. */ + if (mysql_file_pwrite(file, pointer_cast(block.ptr()), + block.length(), 0, MYF(MY_WME + MY_NABP)) != 0) + return {true, false}; + if (mysql_file_sync(file, MYF(MY_WME)) != 0) return {true, false}; + DBUG_EXECUTE_IF("crash_bolt_after_header_sync", DBUG_SUICIDE();); + return {false, true}; +} + +MYSQL_BIN_LOG::Promote_outcome MYSQL_BIN_LOG::promote_spilled_file( + THD *thd, binlog_cache_data *cache_data) { + mysql_mutex_assert_owner(&LOCK_log); + mysql_mutex_assert_owner(&LOCK_commit); + + char new_name[FN_REFLEN]; + /* + file_renamed is captured by reference, so both helpers always report its + current value rather than the value at the point they were defined. + */ + bool file_renamed = false; + bool purge_index_registered = false; + + auto error_outcome = [&] { + return Promote_outcome{Promote_result::kError, + Large_trx_fallback_reason::kNone, file_renamed}; + }; + auto fallback_outcome = [&](Large_trx_fallback_reason reason) { + return Promote_outcome{Promote_result::kFallback, reason, file_renamed}; + }; + + /* + Re-check the encryption policy under LOCK_log. + */ + if (rpl_encryption.is_enabled()) + return fallback_outcome(Large_trx_fallback_reason::kEncryption); + + /* + Another session may have turned binary logging off since this transaction + started, which closes the log and leaves MYSQL_BIN_LOG::name null. + Fall back so the transaction reaches ordered_commit(), which handles a + closed log by skipping the flush and sync stages. + */ + if (!is_open()) + return fallback_outcome(Large_trx_fallback_reason::kLogClosed); + + /* Ensure new binlog file is complete and previous binlog file is durable. */ + if (generate_new_name(new_name, name)) return error_outcome(); + + /* + The binlog-index lock covers this block: the purge-index record, the rename + and the main-index update must not interleave with another rotation, nor + with a purge, which runs without LOCK_log by design (see the note in + ordered_commit) and takes this lock itself. + */ + { + MUTEX_LOCK(index_guard, get_index_lock()); + + /* + Persist the current binary log's GTIDs into mysql.gtid_executed, exactly + as a normal rotation does. If that table is temporarily read-only, + keep_current_binlog is set and we fall back to the standard commit path. + + Using a dedicated thread (Gtid_persist_thd::kDedicated) is required + because this runs inside the committing session's own commit; the + gtid_executed write ends in ha_commit_trans() on whichever THD performs + it. + */ + const auto [persist_error, keep_current_binlog] = + persist_gtids_on_rotate(Gtid_persist_thd::kDedicated); + if (persist_error != 0) { + if (keep_current_binlog) { + DBUG_EXECUTE_IF("gtid_executed_readonly", + { DBUG_SET("-d,gtid_executed_readonly"); }); + return fallback_outcome(Large_trx_fallback_reason::kGtidPersistence); + } + thd->commit_error = THD::CE_FLUSH_ERROR; + return error_outcome(); + } + + /* + Write the promoted file's header events into the reserved region at the + front of the spilled file. The transaction body lives past that region, so + this does not disturb the cache's data. A header that no longer fits is a + clean fallback: no GTID has been assigned and nothing has been written. + */ + const auto [header_error, fits] = write_promoted_binlog_header( + thd, cache_data, cache_data->get_cache()->spilled_file()); + if (header_error) return error_outcome(); + if (!fits) + return fallback_outcome(Large_trx_fallback_reason::kReservedHeaderSpace); + + /* + From here on the promotion is visible outside this session, so failure + must undo it. Register the new name in the purge index first: that record + is the file's crash-recovery owner until open_binlog() adds it to the + main index. + */ + if (m_binlog_index_monitor.open_purge_index_file(true)) + return error_outcome(); + + auto rollback_promotion = create_scope_guard([&] { + if (purge_index_registered && file_renamed) + (void)purge_index_entry(nullptr, nullptr, false /*need_lock_index*/); + m_binlog_index_monitor.close_purge_index_file(); + }); + + if (m_binlog_index_monitor.register_create_index_entry(new_name) || + m_binlog_index_monitor.sync_purge_index_file()) { + LogErr(ERROR_LEVEL, ER_BINLOG_FAILED_TO_SYNC_INDEX_FILE_IN_OPEN); + return error_outcome(); + } + purge_index_registered = true; + DBUG_EXECUTE_IF("crash_bolt_after_purge_index_sync", DBUG_SUICIDE();); + + /* + Promote the file. The cache keeps its open descriptor across the rename, + so the sync below makes the file durable under its new binary-log name. + */ + const char *temp_file_name = cache_data->get_cache()->tmp_file_name(); + if (temp_file_name == nullptr) return error_outcome(); + if (my_rename(temp_file_name, new_name, MYF(MY_WME))) { + LogErr(ERROR_LEVEL, ER_BINLOG_CANT_USE_FOR_LOGGING, new_name, errno); + return error_outcome(); + } + file_renamed = true; + if (mysql_file_sync(cache_data->get_cache()->spilled_file(), MYF(MY_WME)) != + 0) + return error_outcome(); + DBUG_EXECUTE_IF("crash_bolt_after_promote_rename", DBUG_SUICIDE();); + + /* + Chain the current binary log to the promoted file and sync + that Rotate event before opening the promoted file as active. + */ + { + Rotate_log_event r(new_name + dirname_length(new_name), 0, + LOG_EVENT_OFFSET, 0 /*flags*/); + if (write_event_to_binlog(&r)) return error_outcome(); + } + if (m_binlog_file->flush_and_sync()) return error_outcome(); + + DBUG_EXECUTE_IF("crash_bolt_after_rotate_event_sync", DBUG_SUICIDE();); + + /* + The promoted file is durable and the old log's Rotate event points to it. + Close the old log file but keep the index open, so startup recovery cannot + act on the purge record before open_binlog() adds the promoted file to the + main index just below. + */ + char *old_name = name; + name = nullptr; // open_binlog() reassigns it; do not free here. + close(LOG_CLOSE_TO_BE_OPENED, false /*need_lock_log*/, + false /*need_lock_index*/); + + const bool open_failed = + open_binlog(old_name, new_name, max_size, true /*null_created_arg*/, + false /*need_lock_index*/, true /*need_tsid_lock*/, + nullptr /*extra_description_event*/, 0 /*new_index_number*/, + new_name /*promoted_log_name*/); + my_free(old_name); + + /* open_binlog() owns purge-index cleanup from here, on success and + failure. */ + rollback_promotion.release(); + if (open_failed) return error_outcome(); + } // index_guard destroyed: the binlog-index lock is released here + + { + const my_off_t end_pos = m_binlog_file->position(); + thd->set_trans_pos(log_file_name, end_pos); + thd->set_next_event_pos(log_file_name, end_pos); + } + if (cache_data->has_xid()) inc_prep_xids(thd); + + bool after_flush_failed = false; + if (RUN_HOOK(binlog_storage, after_flush, + (thd, log_file_name + dirname_length(log_file_name), + m_binlog_file->position()))) { + LogErr(ERROR_LEVEL, ER_BINLOG_FAILED_TO_RUN_AFTER_FLUSH_HOOK); + after_flush_failed = true; + } + + /* + Publish the end position after the after_flush hook. + */ + update_binlog_end_pos(); + + /* LOCK_log is held; the binlog-index lock was released above. */ + if (after_flush_failed) + handle_binlog_flush_or_sync_error(thd, false /*need_lock_log*/, nullptr); + + /* Counted once the promotion is complete. Atomic for consistency with the + missed counter; not strictly required here since this always runs under + LOCK_log. */ + binlog_large_transaction_optimization_count.fetch_add( + 1, std::memory_order_relaxed); + + return {Promote_result::kPromoted, Large_trx_fallback_reason::kNone, true}; +} + +/** + @return 0 on success, non-zero on error. +*/ +int MYSQL_BIN_LOG::commit_large_transaction(THD *thd, bool all, + bool skip_commit, + binlog_cache_data *cache_data) { + DBUG_TRACE; + + init_thd_variables(thd, all, skip_commit, true /*ready_preempt*/); + + /* Make the spilled file durable before entering the critical section. */ + if (cache_data->get_cache()->flush_and_sync_spilled_file() || + DBUG_EVALUATE_IF("fail_bolt_spill_file_sync", true, false)) { + thd->commit_error = THD::CE_FLUSH_ERROR; + cache_data->reset(); + handle_binlog_flush_or_sync_error(thd, true /*need_lock_log*/, nullptr); + return finish_commit(thd); + } + + /* + We flush prepared records of the large transaction to the log of storage + engine (for example, InnoDB redo log) right before promoting the temp file + into a binlog file. + */ + (void)ha_flush_logs(true); + + /* + When a replication worker thread commits a large transaction with log_bin, + log_replica_updates, replica_preserve_commit_order, and BOLT all enabled, + it should wait for its turn before writing anything into the binary log. + + A transaction that falls back to the standard commit path waits again inside + ordered_commit(), which is harmless: the first wait leaves the worker at the + head of the commit-order queue, so the second one returns without waiting. + + On a non replica session, Commit_order_manager::wait() returns immediately, + so this costs one call per commit, exactly as ordered_commit() already does. + */ + if (is_persistence_enabled() && + (Commit_order_manager::wait_for_its_turn_before_flush_stage(thd) || + ending_trans(thd, all) || + Commit_order_manager::get_rollback_status(thd))) { + if (Commit_order_manager::wait(thd)) return thd->commit_error; + } + + /* Critical section */ + mysql_mutex_lock(&LOCK_log); + wait_for_prep_xids(); + mysql_mutex_lock(&LOCK_commit); + + const Promote_outcome outcome = promote_spilled_file(thd, cache_data); + + switch (outcome.result) { + case Promote_result::kPromoted: + /* + In BOLT we let the next worker in the commit order proceed only after + the promotion has succeeded, rather than at enqueue time as the + standard path does. Releasing earlier would be wrong: a transaction + that releases the next worker and then falls back could re-enter the + flush queue behind the worker it just released, inverting the order. + */ + Commit_order_manager::finish_one(thd); + + /* + The binary log is durable and visible, so commit in + the storage engines. LOCK_log is released first so the next + transaction's flush can overlap this engine commit, exactly as the + standard pipeline does; LOCK_commit is held across it to preserve commit + order. + */ + DBUG_EXECUTE_IF("crash_bolt_before_engine_commit", DBUG_SUICIDE();); + mysql_mutex_unlock(&LOCK_log); + + { + /* + Run the after_sync hook after leaving the sync stage and before + committing the engines, holding the commit lock only (see + ordered_commit()). + */ + const int sync_error = call_after_sync_hook(thd); + + /* The cache keeps the renamed file while resetting its state. */ + cache_data->reset(true /*preserve_spilled_file*/); + + /* + finish_commit() is called with LOCK_commit held, so the after_commit + hook that finish_commit() runs also happens under LOCK_commit, whereas + ordered_commit() swaps to LOCK_after_commit first. This is fine, since + it only adds extra lock hold time when rpl_semi_sync_source_wait_point + is set to AFTER_COMMIT, where that hook waits for a replica + acknowledgement. + */ + (void)finish_commit(thd); + mysql_mutex_unlock(&LOCK_commit); + + /* + Handled only after both locks are released, as the ordered_commit + does. + */ + if (sync_error) + handle_binlog_flush_or_sync_error(thd, true /*need_lock_log*/, + nullptr); + } + + /* + Post-commit: rotate when the promoted active file exceeds max_size. + + Skipped once the commit has failed, as ordered_commit() does. + */ + if (thd->commit_error == THD::CE_NONE && rotate_if_needed()) + thd->commit_error = THD::CE_COMMIT_ERROR; + return thd->commit_error == THD::CE_COMMIT_ERROR ? 1 : 0; + + case Promote_result::kFallback: + /* + Nothing irreversible happened; commit through the standard path. No + commit-order release is needed: ordered_commit()'s own wait() is a no-op + because this session's stage is not REGISTERED, and its enroll_for() + then calls finish_one(). + */ + mysql_mutex_unlock(&LOCK_commit); + mysql_mutex_unlock(&LOCK_log); + record_large_trx_fallback(outcome.reason); + return ordered_commit(thd, all, skip_commit); + + case Promote_result::kError: + if (thd->commit_error == THD::CE_NONE) + thd->commit_error = THD::CE_FLUSH_ERROR; + cache_data->reset(outcome.file_renamed /*preserve_spilled_file*/); + /* + Apply binlog_error_action semantics: either the server aborts, or binary + logging is disabled and the commit proceeds in the engines. Called with + LOCK_log still held (need_lock_log = false). + + The next worker in commit order is released by the unconditional + end-of-group wait_and_finish() in + Slave_worker::slave_worker_ends_group(), as for any transaction that + binlogs nothing. + */ + /* + Name the GNO-exhausted error explicitly, as ordered_commit() does. + */ + handle_binlog_flush_or_sync_error( + thd, false /*need_lock_log*/, + (thd->commit_error == THD::CE_FLUSH_GNO_EXHAUSTED_ERROR) + ? ER_THD(thd, ER_GNO_EXHAUSTED) + : nullptr); + mysql_mutex_unlock(&LOCK_commit); + mysql_mutex_unlock(&LOCK_log); + return finish_commit(thd); + } + + /* + Unreachable: every Promote_result value returns above. + */ + assert(false); + mysql_mutex_unlock(&LOCK_commit); + mysql_mutex_unlock(&LOCK_log); + return finish_commit(thd); +} + +int MYSQL_BIN_LOG::rotate_if_needed() { + if (m_binlog_file->get_real_file_size() < static_cast(max_size)) + return 0; + + bool check_purge = false; + mysql_mutex_lock(&LOCK_log); + + DBUG_EXECUTE_IF("crash_bolt_before_max_size_rotate", DBUG_SUICIDE();); + + int error = rotate(false /*force_rotate*/, &check_purge); + /* Match the normal group-commit rotation boundary. */ + if (!error) + DBUG_EXECUTE_IF("crash_bolt_after_max_size_rotate", DBUG_SUICIDE();); + + mysql_mutex_unlock(&LOCK_log); + + if (!error && check_purge) auto_purge(); + return error; +} diff --git a/sql/binlog/large_trx_commit.h b/sql/binlog/large_trx_commit.h new file mode 100644 index 000000000000..eb50af36864c --- /dev/null +++ b/sql/binlog/large_trx_commit.h @@ -0,0 +1,123 @@ +#ifndef BINLOG_LARGE_TRX_COMMIT_H_INCLUDED +#define BINLOG_LARGE_TRX_COMMIT_H_INCLUDED + +/** + @file + + The binlog large transaction optimization's commit path: a transaction + whose spilled binlog cache exceeds + binlog_large_transaction_optimization_threshold commits by promoting + its temporary file into the binary log sequence as the next binary log + file (MYSQL_BIN_LOG::commit_large_transaction), instead of copying + the cache into the active binary log. +*/ + +#include "my_inttypes.h" // my_off_t + +class THD; +class binlog_cache_data; +class binlog_cache_mngr; + +/** + Why a transaction cannot commit through the large transaction + optimization's commit path. +*/ +enum class Large_trx_fallback_reason { + kNone, + kStatementCache, + kNonRowFormat, + kEncryption, + kCompression, + kChecksumMismatch, + kReservedHeaderSpace, + kGtidPersistence, + kIncident, + kGroupReplication, + kLogClosed, +}; + +/** + Returns the transaction cache to commit through the large transaction + optimization, that is, by promoting its spilled temporary file into the + binary log sequence, or nullptr when the transaction commits through the + standard path. When a candidate transaction is blocked, the blocking + condition is recorded (see record_large_trx_fallback). + + @param cache_mngr The session's binlog cache manager. + + @return the cache whose spilled file is to be promoted, or nullptr when the + transaction commits through the standard path. +*/ +binlog_cache_data *get_cache_for_large_trx_commit( + binlog_cache_mngr *cache_mngr); + +/** + Records that a candidate transaction had to use the standard commit path: + increments binlog_large_transaction_optimization_missed_count and writes a + diagnostic naming the blocking condition. + + @param reason The blocking condition. Must not be + Large_trx_fallback_reason::kNone. +*/ +void record_large_trx_fallback(Large_trx_fallback_reason reason); + +/** + The promoted file's header events are written into a region reserved at the + front of the spilled file. The three functions below are that region's + boundary arithmetic. + + They are declared here, rather than kept local to + write_promoted_binlog_header(), so a unit test can exercise the boundary. No + MTR test can: the region is sized from the serialized Previous_gtids estimate + at spill time, so no configuration makes it too small while leaving the + transaction otherwise promotable. + + @param checksum_len BINLOG_CHECKSUM_LEN when the cache carries per-event + checksums, otherwise 0. + + @return the smallest on-disk length a Large_transaction_header event can have, + that is, with no padding. +*/ +my_off_t large_trx_header_event_min_length(my_off_t checksum_len); + +/** + Whether the reserved region can hold the promoted file's header events: the + bytes already serialized, a minimal Large_transaction_header event, and a + Gtid event at its maximum possible size. + + @param prefix_length Bytes already serialized into the region, that is the + magic, the Format_description event and the + Previous_gtids event. + @param checksum_len As for large_trx_header_event_min_length(). + @param reserved The reserved region's size. + + @retval true The header events fit and the transaction may be promoted. + @retval false They do not, so the transaction falls back with + Large_trx_fallback_reason::kReservedHeaderSpace. +*/ +bool large_trx_header_events_fit(my_off_t prefix_length, my_off_t checksum_len, + my_off_t reserved); + +/** + The padding the Large_transaction_header event needs so that the Gtid event + after it ends exactly at the reserved offset, where the transaction's first + event was placed at spill time. + + Only meaningful when large_trx_header_events_fit() returned true for the same + reserved region and prefix; that call is what makes the subtraction below + safe, since it budgets the Gtid event's maximum length. + + @param prefix_length As for large_trx_header_events_fit(). + @param gtid_event_length The Gtid event's actual length, excluding its + checksum. + @param checksum_len As for large_trx_header_event_min_length(). + @param reserved The reserved region's size. + + @return the padding size, in bytes. +*/ +my_off_t large_trx_header_event_padding(my_off_t reserved, + my_off_t gtid_event_length, + my_off_t prefix_length, + my_off_t checksum_len); + +#endif // BINLOG_LARGE_TRX_COMMIT_H_INCLUDED diff --git a/sql/binlog/log_sanitizer.cc b/sql/binlog/log_sanitizer.cc index 848bbea59e8c..e790d1805ca5 100644 --- a/sql/binlog/log_sanitizer.cc +++ b/sql/binlog/log_sanitizer.cc @@ -22,6 +22,8 @@ // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. #include "sql/binlog/log_sanitizer.h" +#include "mysql/components/services/log_builtins.h" // LogErr +#include "mysqld_error.h" #include "sql/binlog.h" #include "sql/binlog/decompressing_event_object_istream.h" // Decompressing_event_object_istream #include "sql/psi_memory_key.h" @@ -62,6 +64,78 @@ bool Log_sanitizer::is_log_truncation_needed() const { return m_is_log_truncation_needed; } +void Log_sanitizer::process_large_trx_header_event( + Large_transaction_header_log_event const &ev, + IBasic_binlog_file_reader &reader) { + const my_off_t xid_offset = + static_cast(ev.get_terminating_event_offset()); + const uint8_t xid_type = ev.get_terminating_event_type(); + + const bool valid_xid_type = + xid_type == mysql::binlog::event::QUERY_EVENT || + xid_type == mysql::binlog::event::XID_EVENT || + xid_type == mysql::binlog::event::XA_PREPARE_LOG_EVENT; + + const bool valid_xid_offset = + xid_offset != 0 && xid_offset >= BIN_LOG_HEADER_SIZE && + xid_offset <= m_last_file_size && xid_offset > reader.position() && + m_last_file_size - xid_offset >= LOG_EVENT_HEADER_LEN; + + if (!valid_xid_offset || !valid_xid_type) { + m_is_malformed = true; + m_failure_message.assign( + "Large_transaction_header_log_event holds an invalid terminal event"); + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_INVALID_LARGE_TRX_HEADER, + m_valid_file.c_str()); + return; + } + + if (reader.is_checksum_verification_enabled()) { + // The binlog large transaction optimization's recovery shortcut, which + // skips the transaction body, is not applicable when source checksum + // verification is enabled. + LogErr(WARNING_LEVEL, + ER_BINLOG_BOLT_RECOVERY_LARGE_TRX_CHECKSUM_VERIFICATION); + return; + } + + m_large_trx_xid_offset = xid_offset; + m_large_trx_xid_type = xid_type; + if (reader.seek(xid_offset)) { + m_is_malformed = true; + m_failure_message.assign( + "Large_transaction_header_log_event holds an invalid terminal event"); + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_INVALID_LARGE_TRX_HEADER, + m_valid_file.c_str()); + return; + } + + LogErr(INFORMATION_LEVEL, ER_BINLOG_BOLT_RECOVERY_LARGE_TRX_SKIP, + static_cast(xid_offset), m_valid_file.c_str()); + m_in_transaction = true; +} + +bool Log_sanitizer::validate_large_trx_terminal_event(Log_event const &ev) { + const my_off_t event_start_pos = static_cast( + ev.common_header->log_pos - ev.common_header->data_written); + if (event_start_pos != m_large_trx_xid_offset) return true; + + if (static_cast(ev.get_type_code()) == m_large_trx_xid_type) { + // Terminal event validated. Clear the recorded metadata so a later event + // can never be re-matched against this (already-consumed) transaction. + m_large_trx_xid_offset = 0; + m_large_trx_xid_type = 0; + return true; + } + + m_is_malformed = true; + m_failure_message.assign( + "Large_transaction_header_log_event holds an invalid terminal event"); + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_INVALID_LARGE_TRX_HEADER, + m_valid_file.c_str()); + return false; +} + void Log_sanitizer::process_query_event(Query_log_event const &ev) { std::string query{ev.query}; diff --git a/sql/binlog/log_sanitizer.h b/sql/binlog/log_sanitizer.h index 5cf6e9e21791..bd5f7917cfa6 100644 --- a/sql/binlog/log_sanitizer.h +++ b/sql/binlog/log_sanitizer.h @@ -115,6 +115,11 @@ class Log_sanitizer { /// @returns Reference to a memory key virtual PSI_memory_key &get_memory_key() const = 0; + /// @brief Whether this sanitizer is recovering a relay log rather than a + /// binary log. + /// @returns true for relay-log recovery, false for binary-log recovery. + virtual bool is_relay_log_recovery() const { return false; } + /// @brief This function goes through the opened file and searches for /// a valid position in a binary log file. It also gathers /// information about XA transactions which will be used during the @@ -229,6 +234,22 @@ class Log_sanitizer { /// Last opened file size my_off_t m_last_file_size{0}; + /// Metadata for the large transaction's terminal event. + my_off_t m_large_trx_xid_offset{0}; + uint8_t m_large_trx_xid_type{0}; + + /// @brief Invoked when a `Large_transaction_header_log_event` is read from + /// the reader. + /// @details Validates the terminal event's offset and type, seeks to that + /// event, and marks the transaction as open. + /// @param ev The `Large_transaction_header_log_event` to process. + /// @param reader Reader for the current binary log. + void process_large_trx_header_event( + Large_transaction_header_log_event const &ev, + IBasic_binlog_file_reader &reader); + + bool validate_large_trx_terminal_event(Log_event const &ev); + /// @brief Invoked when a `Query_log_event` is read from the binary log file /// reader. /// @details The underlying query string is inspected to determine if the diff --git a/sql/binlog/log_sanitizer_impl.hpp b/sql/binlog/log_sanitizer_impl.hpp index 26aaffde906c..97a9ac8112de 100644 --- a/sql/binlog/log_sanitizer_impl.hpp +++ b/sql/binlog/log_sanitizer_impl.hpp @@ -123,6 +123,8 @@ bool Log_sanitizer::process_one_log(Type_reader &reader, this->m_is_malformed = false; while (istream >> ev) { + if (!this->validate_large_trx_terminal_event(*ev)) break; + bool is_source_event = !ev->is_relay_log_event() || (ev->server_id && ::server_id != ev->server_id); switch (ev->get_type_code()) { @@ -139,6 +141,18 @@ bool Log_sanitizer::process_one_log(Type_reader &reader, dynamic_cast(*ev)); break; } + case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT: { + // The header's offset refers to the source's binary log, so it is only + // actionable during binary-log recovery. Relay-log recovery must skip + // it: a header relayed from the source does not carry + // LOG_EVENT_RELAY_LOG_F, so is_relay_log_recovery() (not the event + // flag) is the correct discriminator. + if (!this->is_relay_log_recovery()) { + this->process_large_trx_header_event( + dynamic_cast(*ev), reader); + } + break; + } case mysql::binlog::event::ROTATE_EVENT: { if (is_source_event) { m_validation_started = true; diff --git a/sql/binlog/transaction_commit_helper.cc b/sql/binlog/transaction_commit_helper.cc new file mode 100644 index 000000000000..a1691b152bcb --- /dev/null +++ b/sql/binlog/transaction_commit_helper.cc @@ -0,0 +1,72 @@ +#include "sql/binlog/transaction_commit_helper.h" + +#include "dur_prop.h" +#include "my_dbug.h" +#include "my_systime.h" +#include "sql/mysqld.h" +#include "sql/rpl_gtid.h" +#include "sql/rpl_trx_tracking.h" +#include "sql/sql_class.h" +#include "sql/transaction_info.h" + +/** + @file + @brief Implementations for sql/binlog/transaction_commit_helper.h. +*/ + +void init_thd_variables(THD *thd, bool all, bool skip_commit, + [[maybe_unused]] bool ready_preempt) { + /* These values are reset before a transaction enters commit processing. */ + thd->tx_commit_pending = true; + thd->commit_error = THD::CE_NONE; + thd->next_to_commit = nullptr; + thd->durability_property = HA_IGNORE_DURABILITY; + thd->get_transaction()->m_flags.real_commit = all; + thd->get_transaction()->m_flags.xid_written = false; + thd->get_transaction()->m_flags.commit_low = !skip_commit; + thd->get_transaction()->m_flags.run_hooks = !skip_commit; +#ifndef NDEBUG + thd->get_transaction()->m_flags.ready_preempt = ready_preempt; +#endif +} + +Transaction_gtid_header::Transaction_gtid_header( + THD *thd, bool parallelization_barrier, + Transaction_dependency_tracker *dependency_tracker) { + dependency_tracker->get_dependency(thd, parallelization_barrier, + m_sequence_number, m_last_committed); + + /* Preserve commit ordering when the statement cache follows this cache. */ + thd->get_transaction()->last_committed = SEQ_UNINIT; + + m_immediate_commit_timestamp = my_micro_time(); + m_original_commit_timestamp = thd->variables.original_commit_timestamp; + if (m_original_commit_timestamp == UNDEFINED_COMMIT_TIMESTAMP) { + if (thd->slave_thread || thd->is_binlog_applier()) { + m_original_commit_timestamp = 0; + } else { + DBUG_EXECUTE_IF("rpl_invalid_gtid_timestamp", + m_immediate_commit_timestamp += 3600000000;); + m_original_commit_timestamp = m_immediate_commit_timestamp; + } + } else { + thd->variables.original_commit_timestamp = UNDEFINED_COMMIT_TIMESTAMP; + } + + m_immediate_server_version = do_server_version_int(::server_version); + thd->variables.immediate_server_version = UNDEFINED_SERVER_VERSION; + DBUG_EXECUTE_IF("fixed_server_version", m_immediate_server_version = 888888;); + DBUG_EXECUTE_IF("gr_fixed_server_version", + m_immediate_server_version = 777777;); + + m_original_server_version = thd->variables.original_server_version; + if (m_original_server_version == UNDEFINED_SERVER_VERSION) { + if (thd->slave_thread || thd->is_binlog_applier()) { + m_original_server_version = UNKNOWN_SERVER_VERSION; + } else { + m_original_server_version = m_immediate_server_version; + } + } else { + thd->variables.original_server_version = UNDEFINED_SERVER_VERSION; + } +} diff --git a/sql/binlog/transaction_commit_helper.h b/sql/binlog/transaction_commit_helper.h new file mode 100644 index 000000000000..fd44331bd164 --- /dev/null +++ b/sql/binlog/transaction_commit_helper.h @@ -0,0 +1,108 @@ +#ifndef BINLOG_TRANSACTION_COMMIT_HELPER_H_INCLUDED +#define BINLOG_TRANSACTION_COMMIT_HELPER_H_INCLUDED + +#include + +#include "my_inttypes.h" + +/** + @file + @brief State shared by the group commit path and the large transaction + optimization's commit path: the per-transaction THD setup both perform, and + the fields both use to build a transaction's Gtid event. Extracted so the two + paths derive them from the same code rather than from two copies. +*/ + +class THD; +class Transaction_dependency_tracker; + +/** + Initializes THD state shared by group commit and the binlog large transaction + optimization. Resets the per-commit error and queue fields, and records on + the transaction which follow-up work the commit still owes. + + @param thd Session about to enter commit processing. + @param all True when committing a whole transaction, false for a + single statement. Stored as the real_commit flag. + @param skip_commit True when the caller commits in the engines itself, so + the commit_low and run_hooks flags are cleared. + @param ready_preempt Debug-only flag used by the group commit preemption + tests. Ignored in release builds. +*/ +void init_thd_variables(THD *thd, bool all, bool skip_commit, + bool ready_preempt = false); + +/** + This class collects the fields used to create a transaction's GTID event. + It is used by both group and large transaction commit codepaths. +*/ +class Transaction_gtid_header { + public: + /** + Collects every field of the transaction's Gtid event. Constructing this + consumes state from the session: it takes logical timestamps from the + dependency tracker, and clears the session's original commit timestamp and + server version overrides so a later transaction does not inherit them. It + is therefore built once per transaction, immediately before the Gtid event + is serialized. + + @param thd Session whose transaction is committing. + @param parallelization_barrier True when the transaction must not be + applied in parallel with its neighbours, which makes the tracker + assign it a last_committed equal to its own sequence_number. A + promoted transaction sets this. + @param dependency_tracker Supplies the sequence_number and + last_committed pair that lets a replica decide what may be applied + in parallel. + */ + Transaction_gtid_header(THD *thd, bool parallelization_barrier, + Transaction_dependency_tracker *dependency_tracker); + + /// @return Sequence number of the last transaction this one depends on. + int64 last_committed() const { return m_last_committed; } + + /// @return This transaction's own logical commit sequence number. + int64 sequence_number() const { return m_sequence_number; } + + /** + @return When the transaction committed on the server that originated it, in + microseconds. Zero for a replica-applied transaction whose source + did not supply one. + */ + ulonglong original_commit_timestamp() const { + return m_original_commit_timestamp; + } + + /// @return When the transaction committed on this server, in microseconds. + ulonglong immediate_commit_timestamp() const { + return m_immediate_commit_timestamp; + } + + /** + @return Version of the server that originated the transaction, or + UNKNOWN_SERVER_VERSION when a replica applies one that carried + none. + */ + uint32_t original_server_version() const { return m_original_server_version; } + + /// @return Version of this server. + uint32_t immediate_server_version() const { + return m_immediate_server_version; + } + + private: + /// Sequence number of the last transaction this one depends on. + int64 m_last_committed; + /// This transaction's own logical commit sequence number. + int64 m_sequence_number; + /// Commit time on the originating server, in microseconds. + ulonglong m_original_commit_timestamp; + /// Commit time on this server, in microseconds. + ulonglong m_immediate_commit_timestamp; + /// Version of the server that originated the transaction. + uint32_t m_original_server_version; + /// Version of this server. + uint32_t m_immediate_server_version; +}; + +#endif // BINLOG_TRANSACTION_COMMIT_HELPER_H_INCLUDED diff --git a/sql/binlog_ostream.cc b/sql/binlog_ostream.cc index 9c6b4680a93b..4455f2879368 100644 --- a/sql/binlog_ostream.cc +++ b/sql/binlog_ostream.cc @@ -23,10 +23,20 @@ #include "sql/binlog_ostream.h" #include +#include +#include +#include +#include +#include +#include +#include +#include #include "my_aes.h" +#include "my_dir.h" #include "my_inttypes.h" #include "my_rnd.h" #include "my_sys.h" +#include "my_thread_local.h" // my_errno #include "mysql/components/services/log_builtins.h" #include "mysql/psi/mysql_file.h" #include "mysqld_error.h" @@ -38,21 +48,95 @@ bool binlog_cache_is_reset = false; #endif +bool is_bolt_temp_file(const char *name) { + constexpr std::string_view prefix{kBinlogTempFilePrefix}; + const std::string_view file_name{name}; + + return file_name.starts_with(prefix) && file_name.size() > prefix.size() && + std::ranges::all_of(file_name.substr(prefix.size()), [](char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_'; + }); +} + +namespace { + +/* + A spilled temp file starts as an anonymous mkstemp() file, which is created + 0600 with my_umask ignored. It may get promoted directly into a binary log + through the binlog large transaction optimization code path, so edit the + permissions to match what a binlog file created by the server would have. +*/ +ulong binlog_temp_file_permissions() { + ulong permissions = 0; + if (my_umask & 0400) permissions |= USER_READ; + if (my_umask & 0200) permissions |= USER_WRITE; + if (my_umask & 0100) permissions |= USER_EXECUTE; + if (my_umask & 0040) permissions |= GROUP_READ; + if (my_umask & 0020) permissions |= GROUP_WRITE; + if (my_umask & 0010) permissions |= GROUP_EXECUTE; + if (my_umask & 0004) permissions |= OTHERS_READ; + if (my_umask & 0002) permissions |= OTHERS_WRITE; + if (my_umask & 0001) permissions |= OTHERS_EXECUTE; + return permissions; +} + +} // namespace + IO_CACHE_binlog_cache_storage::IO_CACHE_binlog_cache_storage() = default; IO_CACHE_binlog_cache_storage::~IO_CACHE_binlog_cache_storage() { close(); } bool IO_CACHE_binlog_cache_storage::open(const char *dir, const char *prefix, my_off_t cache_size, - my_off_t max_cache_size) { + my_off_t max_cache_size, + my_off_t reserved_bytes) { DBUG_TRACE; if (open_cached_file(&m_io_cache, dir, prefix, cache_size, MYF(MY_WME))) return true; + m_spilled_file_is_managed = false; + + /* + Default to an anonymous spill file. Whether it is instead a named file that + can be promoted into the binary log sequence is decided per transaction from + binlog_large_transaction_optimization_enabled, at transaction start, via + set_named_file(). + A named file is kept in the filesystem namespace (promotable, and cleaned + up at server startup if left behind); an anonymous file is unlinked at + creation and is never visible. So when the binlog large transaction + optimization is disabled for a transaction, its spill file leaves no + visible bolt_ file. + */ + m_io_cache.named_file = false; + /* Keep the arguments: reset re-opens the cache after a spill. */ + m_dir = dir; + m_prefix = prefix; + m_cache_size = cache_size; + m_max_cache_size_arg = max_cache_size; + + /* + The cache's content is placed after the reserved bytes: physical + positions start there, and the first flush into the lazily created + temporary file seeks there (the file's offset is 0 at creation). + + The reserved region is applied in all cases, even when the binlog large + transaction optimization is disabled for this transaction. A non-promoted + transaction never fills it with header events and never copies it into the + binary log (begin() starts the read cursor past it), so it costs only + transient temp-file space and is invisible in the binary log. Only the file + naming is gated on binlog_large_transaction_optimization_enabled. + */ + m_reserved_bytes = reserved_bytes; + m_io_cache.pos_in_file = reserved_bytes; + m_io_cache.seek_not_done = true; if (rpl_encryption.is_enabled()) enable_encryption(); m_max_cache_size = max_cache_size; + /* The max cache size caps physical positions: shift it too. */ + if (m_max_cache_size <= ~(my_off_t)0 - reserved_bytes) + m_max_cache_size += reserved_bytes; /* Set the max cache size for IO_CACHE */ - m_io_cache.end_of_file = max_cache_size; + m_io_cache.end_of_file = m_max_cache_size; return false; } @@ -87,10 +171,25 @@ bool IO_CACHE_binlog_cache_storage::write(const unsigned char *buffer, } } - return my_b_safe_write(&m_io_cache, buffer, length); + if (my_b_safe_write(&m_io_cache, buffer, length)) return true; + + /* + open_cached_file creates the physical backing file lazily. For a named + (promotable) spill file, immediately replace its generic mkstemp name with + the managed bolt_ form before callers can observe or promote it. An + anonymous spill file (optimization disabled for this transaction) has no + name and is left as-is. + */ + if (is_spilled() && m_io_cache.named_file && !m_spilled_file_is_managed) { + if (rename_spilled_file()) return true; + m_spilled_file_is_managed = true; + } + return false; } bool IO_CACHE_binlog_cache_storage::truncate(my_off_t offset) { + /* Translate the zero-based data offset to a physical position. */ + offset += m_reserved_bytes; /* It is not really necessary to flush the data will be truncated into temporary file before truncating . And it may cause write failure. So set @@ -105,20 +204,75 @@ bool IO_CACHE_binlog_cache_storage::truncate(my_off_t offset) { return false; } -bool IO_CACHE_binlog_cache_storage::reset() { - if (truncate(0)) return true; +bool IO_CACHE_binlog_cache_storage::rename_spilled_file() { + DBUG_TRACE; + assert(is_spilled()); + if (m_io_cache.file_name == nullptr) return true; + + const char *const old_name = m_io_cache.file_name; + + /* + The name is bolt__, both in hex. It is unique + within a server run: server_start_time is constant for the run and the + serial is a monotonic atomic counter, so no probe or retry is needed. + Leftover files from earlier runs are removed at startup; if that cleanup + fails the binlog large transaction optimization is disabled and no new files + are created, so a name cannot collide with an earlier run's file either. + */ + static std::atomic serial_counter{0}; + const uint64_t serial = + serial_counter.fetch_add(1, std::memory_order_relaxed); + + char new_name[FN_REFLEN]; + const char *const dir = (m_dir != nullptr) ? m_dir : ""; + const char separator[2] = {(m_dir != nullptr) ? FN_LIBCHAR : '\0', '\0'}; + const int length = snprintf( + new_name, sizeof(new_name), "%s%s%s%llx_%llx", dir, separator, + kBinlogTempFilePrefix, static_cast(server_start_time), + static_cast(serial)); + if (length < 0 || static_cast(length) >= sizeof(new_name)) + return true; + + char *replacement = my_strdup(PSI_NOT_INSTRUMENTED, new_name, MYF(MY_WME)); + if (replacement == nullptr) return true; + if (mysql_file_rename(m_io_cache.file_key, old_name, new_name, MYF(MY_WME))) { + my_free(replacement); + return true; + } - /* Truncate the temporary file if there is one. */ - if (m_io_cache.file != -1) { - if (my_chsize(m_io_cache.file, 0, 0, MYF(MY_WME))) return true; + my_free(m_io_cache.file_name); + m_io_cache.file_name = replacement; + return my_chmod(new_name, binlog_temp_file_permissions(), MYF(MY_WME)); +} + +bool IO_CACHE_binlog_cache_storage::reset(bool preserve_spilled_file) { + assert(!preserve_spilled_file || is_spilled()); + if (is_spilled()) { + disable_encryption(); - DBUG_EXECUTE_IF("show_io_cache_size", { - my_off_t file_size = - my_seek(m_io_cache.file, 0L, MY_SEEK_END, MYF(MY_WME + MY_FAE)); - assert(file_size == 0); - }); + /* + For a temp file that has been promoted to a binlog file, we reset the + file_name before close() to prevent close_cached_file() from deleting + the promoted file. For the same reason we close and re-open instead of + truncating: the cache keeps its descriptor across the promotion rename, + so a truncate would zero the promoted binlog file. + */ + if (preserve_spilled_file) { + my_free(m_io_cache.file_name); + m_io_cache.file_name = nullptr; + } + + close(); + if (open(m_dir, m_prefix, m_cache_size, m_max_cache_size_arg, + m_reserved_bytes)) + return true; + } else if (truncate(0)) { /* Never spilled: rewind to the reserved boundary */ + return true; } + assert(!is_spilled()); + assert(length() == 0); + DBUG_EXECUTE_IF("ensure_binlog_cache_temporary_file_is_encrypted", { /* Reset the binlog_cache_temporary_file_is_encrypted at resetting @@ -144,8 +298,26 @@ size_t IO_CACHE_binlog_cache_storage::disk_writes() const { return m_io_cache.disk_writes; } +bool IO_CACHE_binlog_cache_storage::is_spilled() const { + return m_io_cache.file != -1; +} + +bool IO_CACHE_binlog_cache_storage::flush_and_sync_spilled_file() { + DBUG_TRACE; + assert(is_spilled()); + if (flush_io_cache(&m_io_cache)) return true; + /* + A ROLLBACK TO SAVEPOINT truncation repositions the cache but does not + shrink the file: cut any stale bytes past the logical end, so the + promoted file ends exactly at the transaction's terminating event. + */ + if (my_chsize(m_io_cache.file, my_b_tell(&m_io_cache), 0, MYF(MY_WME))) + return true; + return mysql_file_sync(m_io_cache.file, MYF(MY_WME)) != 0; +} + const char *IO_CACHE_binlog_cache_storage::tmp_file_name() const { - return my_filename(m_io_cache.file); + return m_io_cache.file_name; } bool IO_CACHE_binlog_cache_storage::begin(unsigned char **buffer, @@ -166,7 +338,9 @@ bool IO_CACHE_binlog_cache_storage::begin(unsigned char **buffer, m_io_cache.m_decryptor == nullptr); };); - if (reinit_io_cache(&m_io_cache, READ_CACHE, 0, false, false)) { + /* The data starts after the reserved bytes. */ + if (reinit_io_cache(&m_io_cache, READ_CACHE, m_reserved_bytes, false, + false)) { DBUG_EXECUTE_IF("simulate_tmpdir_partition_full", { DBUG_SET("-d,simulate_file_write_error"); }); @@ -194,8 +368,11 @@ bool IO_CACHE_binlog_cache_storage::next(unsigned char **buffer, } my_off_t IO_CACHE_binlog_cache_storage::length() const { - if (m_io_cache.type == WRITE_CACHE) return my_b_tell(&m_io_cache); - return m_io_cache.end_of_file; + /* Physical positions include the reserved bytes; report the data + length. */ + if (m_io_cache.type == WRITE_CACHE) + return my_b_tell(&m_io_cache) - m_reserved_bytes; + return m_io_cache.end_of_file - m_reserved_bytes; } bool IO_CACHE_binlog_cache_storage::enable_encryption() { @@ -250,10 +427,10 @@ bool IO_CACHE_binlog_cache_storage::setup_ciphers_password() { return false; } -bool Binlog_cache_storage::open(my_off_t cache_size, my_off_t max_cache_size) { - const char *LOG_PREFIX = "ML"; - - if (m_file.open(mysql_tmpdir, LOG_PREFIX, cache_size, max_cache_size)) +bool Binlog_cache_storage::open(my_off_t cache_size, my_off_t max_cache_size, + my_off_t reserved_bytes) { + if (m_file.open(binlog_temp_files_dir.path(), kBinlogTempFilePrefix, + cache_size, max_cache_size, reserved_bytes)) return true; m_pipeline_head = &m_file; return false; @@ -411,3 +588,105 @@ bool Binlog_encryption_ostream::sync() { return m_down_ostream->sync(); } int Binlog_encryption_ostream::get_header_size() { return m_header->get_header_size(); } + +Binlog_temp_files_dir binlog_temp_files_dir; + +/* + Deletes the bolt_* spill files left in the binlog temp files directory by a + previous run. The directory belongs to the server and should only hold bolt_* + files; Binlog_temp_files_dir::init() calls this during startup. + + Every entry other than "." and ".." must be a regular file, must not be a + symlink, and must match the bolt_* name. Anything else means this is not the + directory we think it is, so the error reason is logged and the cleanup + fails rather than being skipped. Since init() runs from mysqld startup, that + failure aborts server start. + + Returns true on failure, having logged the reason. +*/ +static bool temp_files_dir_clear_files(const char *path) { + MY_DIR *dir_info = my_dir(path, MYF(MY_WANT_STAT)); + if (dir_info == nullptr) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_FAILED, path, my_errno()); + return true; + } + + uint removed = 0; + bool failed = false; + for (uint i = 0; i < dir_info->number_off_files && !failed; i++) { + const fileinfo *file = dir_info->dir_entry + i; + /* Skip "." and "..". */ + if (file->name[0] == '.' && + (!file->name[1] || (file->name[1] == '.' && !file->name[2]))) + continue; + char file_path[FN_REFLEN]; + if (snprintf(file_path, sizeof(file_path), "%s%c%s", path, FN_LIBCHAR, + file->name) >= static_cast(sizeof(file_path))) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_INVALID, path); + failed = true; + break; + } + /* Do not follow links or delete entries the server did not create. */ + if (file->mystat == nullptr || !MY_S_ISREG(file->mystat->st_mode) || + my_is_symlink(file_path, nullptr) || !is_bolt_temp_file(file->name)) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_UNSAFE_ENTRY, path, + file->name); + failed = true; + break; + } + if (my_delete(file_path, MYF(0))) { + LogErr(ERROR_LEVEL, ER_BINLOG_CANT_DELETE_FILE, file_path); + failed = true; + break; + } + removed++; + } + my_dirend(dir_info); + if (failed) return true; + + if (removed > 0) + LogErr(INFORMATION_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_CLEANED, removed, + path); + return false; +} + +bool Binlog_temp_files_dir::init(const char *log_basename) { + DBUG_TRACE; + assert(log_basename != nullptr && !m_initialized); + + // Build /#binlog_temp_files + char dir_part[FN_REFLEN]; + size_t dir_len; // Required out-param of dirname_part(); value unused. + dirname_part(dir_part, log_basename, &dir_len); + + const int path_len = snprintf(m_path, sizeof(m_path), "%s%s", dir_part, + kBinlogTempFilesDirName); + if (path_len < 0 || static_cast(path_len) >= sizeof(m_path)) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_FAILED, log_basename, + ENAMETOOLONG); + return true; + } + const char *path = m_path; + + /* A symlink is rejected even if it points to a directory: files in + this directory must be on the same filesystem as the binlog files. */ + if (my_is_symlink(path, nullptr)) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_INVALID, path); + return true; + } + + MY_STAT stat_area; + if (my_stat(path, &stat_area, MYF(0)) != nullptr) { + if (!MY_S_ISDIR(stat_area.st_mode)) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_INVALID, path); + return true; + } + if (temp_files_dir_clear_files(path)) return true; + } else if (my_mkdir(path, my_umask_dir, MYF(0)) != 0) { + LogErr(ERROR_LEVEL, ER_BINLOG_BOLT_TEMP_FILES_DIR_FAILED, path, my_errno()); + return true; + } + + m_initialized = true; + return false; +} diff --git a/sql/binlog_ostream.h b/sql/binlog_ostream.h index 41dba074fd3a..198174b693a0 100644 --- a/sql/binlog_ostream.h +++ b/sql/binlog_ostream.h @@ -25,6 +25,8 @@ #define BINLOG_OSTREAM_INCLUDED #include +#include +#include "my_io.h" // FN_REFLEN #include "sql/basic_ostream.h" #include "sql/rpl_log_encryption.h" @@ -84,11 +86,13 @@ class IO_CACHE_binlog_cache_storage : public Truncatable_ostream { @param[in] prefix Prefix of the temporary file name @param[in] cache_size Size of the memory buffer. @param[in] max_cache_size Maximum size of the memory buffer + @param[in] reserved_bytes Bytes reserved for header events + @retval false Success @retval true Error */ bool open(const char *dir, const char *prefix, my_off_t cache_size, - my_off_t max_cache_size); + my_off_t max_cache_size, my_off_t reserved_bytes); void close(); bool write(const unsigned char *buffer, my_off_t length) override; @@ -97,10 +101,11 @@ class IO_CACHE_binlog_cache_storage : public Truncatable_ostream { /* binlog cache doesn't need seek operation. Setting true to return error */ bool seek(my_off_t offset [[maybe_unused]]) override { return true; } /** - Reset status and drop all data. It looks like a cache never was used after - reset. + Reset status and drop all data. When preserve_spilled_file is true, the + caller has promoted the spilled file and reset closes the cache without + deleting that file. */ - bool reset(); + bool reset(bool preserve_spilled_file = false); /** Returns the file name if a temporary file is opened, otherwise nullptr is returned. @@ -135,12 +140,54 @@ class IO_CACHE_binlog_cache_storage : public Truncatable_ostream { */ bool next(unsigned char **buffer, my_off_t *length); my_off_t length() const; + my_off_t reserved_bytes() const { return m_reserved_bytes; } bool flush() override { return false; } bool sync() override { return false; } + /** + Returns true once the cache overflowed into its temporary file. + */ + bool is_spilled() const; + /** + Returns the spilled temporary file's descriptor (-1 when not + spilled). The cache retains ownership until reset. + */ + File spilled_file() const { return m_io_cache.file; } + /** + Returns true if the cache's events are encrypted. This reflects the + temporary file's actual encryption, which is fixed at spill time, and is + independent of the current global binlog_encryption setting (which may + have changed since the transaction spilled). + */ + bool is_encrypted() const { return m_io_cache.m_encryptor != nullptr; } + /** + Flushes buffered bytes into the spilled temporary file and syncs the + file to disk. The cache must be spilled. + + @retval false Success + @retval true Error + */ + bool flush_and_sync_spilled_file(); + /** + Selects whether the next lazily-created spill file is a named file (kept + in the filesystem namespace, and therefore promotable) or an anonymous + file. Set per transaction, before the first spill. + */ + void set_named_file(bool named) { m_io_cache.named_file = named; } private: + /** Rename a newly spilled generic cache file to a managed bolt_ name. */ + bool rename_spilled_file(); + IO_CACHE m_io_cache; my_off_t m_max_cache_size = 0; + my_off_t m_reserved_bytes = 0; + /* True after the lazily-created spill file has a generated bolt_ name. */ + bool m_spilled_file_is_managed = false; + /* The open() arguments, kept for re-opening after reset. */ + const char *m_dir = nullptr; + const char *m_prefix = nullptr; + my_off_t m_cache_size = 0; + my_off_t m_max_cache_size_arg = 0; /** Enable IO Cache temporary file encryption. @@ -175,7 +222,8 @@ class Binlog_cache_storage : public Basic_ostream { public: ~Binlog_cache_storage() override; - bool open(my_off_t cache_size, my_off_t max_cache_size); + bool open(my_off_t cache_size, my_off_t max_cache_size, + my_off_t reserved_bytes); void close(); bool write(const unsigned char *buffer, my_off_t length) override { @@ -192,19 +240,37 @@ class Binlog_cache_storage : public Basic_ostream { bool truncate(my_off_t offset) { return m_pipeline_head->truncate(offset); } /** - Reset status and drop all data. It looks like a cache was never used - after reset. + Reset status and drop all data. When preserve_spilled_file is true, the + cache closes without deleting a file the binlog large transaction + optimization has already promoted. */ - bool reset() { return m_file.reset(); } + bool reset(bool preserve_spilled_file = false) { + return m_file.reset(preserve_spilled_file); + } /** Returns the count of disk writes */ size_t disk_writes() const { return m_file.disk_writes(); } + /** + Returns the bytes reserved at the beginning of the temp file. + */ + my_off_t reserved_bytes() const { return m_file.reserved_bytes(); } /** Returns the name of the temporary file. */ const char *tmp_file_name() const { return m_file.tmp_file_name(); } - + /// @see IO_CACHE_binlog_cache_storage::is_spilled + bool is_spilled() const { return m_file.is_spilled(); } + /// @see IO_CACHE_binlog_cache_storage::spilled_file + File spilled_file() const { return m_file.spilled_file(); } + /// @see IO_CACHE_binlog_cache_storage::is_encrypted + bool is_encrypted() const { return m_file.is_encrypted(); } + /// @see IO_CACHE_binlog_cache_storage::flush_and_sync_spilled_file + bool flush_and_sync_spilled_file() { + return m_file.flush_and_sync_spilled_file(); + } + /// @see IO_CACHE_binlog_cache_storage::set_named_file + void set_named_file(bool named) { m_file.set_named_file(named); } /** Copy all data to a output stream. This function hides the internal implementation of storage detail. So it will not disturb the callers @@ -301,4 +367,73 @@ class Binlog_encryption_ostream : public Truncatable_ostream { std::unique_ptr m_header; std::unique_ptr m_encryptor; }; + +// Directory where the temp binlog files (spilled from cache), live. +// This directory lives in the binlog directory. +inline constexpr const char *kBinlogTempFilesDirName = "#binlog_temp_files"; + +// Allocation quantum for the header reservation at the beginning of every +// binlog temp file. +inline constexpr my_off_t kBinlogTempFileReservedBytes = 64 * 1024; + +// Minimum space left after the Previous_gtids payload in a temp-file header. +inline constexpr my_off_t kBinlogTempFilePreviousGtidsHeadroomBytes = 32 * 1024; + +// Managed large-transaction spill files are named +// bolt__ (both hex), which is unique within a +// server run and lets startup cleanup recognize only files created by this +// feature. +inline constexpr char kBinlogTempFilePrefix[] = "bolt_"; + +/** + Returns true if 'name' is a binary log cache spill file, i.e. matches the + bolt_ pattern that IO_CACHE_binlog_cache_storage gives its spill files. + + A true result does NOT mean the file was, or will be, promoted into the + binary log sequence. Every binlog cache spill file uses this name, including + transactions that commit through the standard path (below the threshold, + encrypted, compressed, and so on). This is purely an ownership check, so that + startup cleanup of \#binlog_temp_files only deletes files the binary log cache + created. + + Declared here, rather than kept local to binlog_ostream.cc, so a unit test can + exercise the whole name space cheaply. Getting this predicate wrong is + expensive: a name it wrongly rejects becomes an "unsafe entry" that aborts + startup, which is how the missing A-Z range in the original character class + behaved, since mkstemp() also produces upper case. + + @param name Base name of a directory entry, without any directory part. + + @retval true The binary log cache created this file. + @retval false It did not, so cleanup must leave the file alone. +*/ +bool is_bolt_temp_file(const char *name); + +class Binlog_temp_files_dir { + public: + /** + Init directory at server startup. + + @param log_basename The log basename; the directory is created in + its directory part + + @retval false Success. + @retval true Failure; an error has been logged. + */ + bool init(const char *log_basename); + + // @return full path of the directory. + const char *path() const { + assert(m_initialized); + return m_path; + } + + private: + char m_path[FN_REFLEN]; + bool m_initialized{false}; +}; + +// The binary log's temp files directory (#binlog_temp_files). +extern Binlog_temp_files_dir binlog_temp_files_dir; + #endif // BINLOG_OSTREAM_INCLUDED diff --git a/sql/binlog_reader.cc b/sql/binlog_reader.cc index 82fe6bd84f8c..d758673857ae 100644 --- a/sql/binlog_reader.cc +++ b/sql/binlog_reader.cc @@ -306,6 +306,9 @@ Binlog_read_error::Error_type binlog_event_deserialize( case mysql::binlog::event::TRANSACTION_PAYLOAD_EVENT: ev = new Transaction_payload_log_event(buf, fde); break; + case mysql::binlog::event::LARGE_TRANSACTION_HEADER_EVENT: + ev = new Large_transaction_header_log_event(buf, fde); + break; default: /* Create an object of Ignorable_log_event for unrecognized sub-class. diff --git a/sql/binlog_reader.h b/sql/binlog_reader.h index eb61dfe3d92d..f4d1407cc4ff 100644 --- a/sql/binlog_reader.h +++ b/sql/binlog_reader.h @@ -396,6 +396,9 @@ class IBasic_binlog_file_reader { /// The return value is static memory that is never deallocated. virtual const char *get_error_str() const = 0; + /// Return whether checksum verification is enabled. + virtual bool is_checksum_verification_enabled() const = 0; + /// Return the current position in bytes, relative to the beginning /// of the file. virtual my_off_t position() const = 0; @@ -529,6 +532,9 @@ class Basic_binlog_file_reader : public IBasic_binlog_file_reader { } bool is_open() const { return m_ifile.is_open(); } + bool is_checksum_verification_enabled() const override { + return m_verify_checksum; + } my_off_t position() const override { return m_ifile.position(); } bool seek(my_off_t pos) override { return m_ifile.seek(pos); } diff --git a/sql/log_event.cc b/sql/log_event.cc index d02d47b5d9e1..67f2c7835cd1 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -1154,6 +1154,13 @@ bool Log_event::need_checksum() { assert(!ret || ((common_footer->checksum_alg == static_cast(binlog_checksum_options) || + /* + Cached events may carry the algorithm recorded at their first + event write to the transaction cache when the binlog large + transaction optimization is enabled. A recorded algorithm may + differ from the server's binlog_checksum_options. + */ + event_cache_type != Log_event::EVENT_NO_CACHE || /* Stop event closes the relay-log and its checksum alg preference is set by the caller can be different @@ -12985,6 +12992,75 @@ void Ignorable_log_event::print(FILE *, } #endif +Large_transaction_header_log_event::Large_transaction_header_log_event( + const char *buf, const Format_description_event *descr_event) + : mysql::binlog::event::Large_transaction_header_event(buf, descr_event), + Log_event(header(), footer()) { + DBUG_TRACE; +} + +void Large_transaction_header_log_event::claim_memory_ownership(bool claim) { + my_claim(temp_buf, claim); + my_claim(this, claim); +} + +#ifdef MYSQL_SERVER +int Large_transaction_header_log_event::pack_info(Protocol *protocol) { + char buf[256]; + const size_t bytes = + snprintf(buf, sizeof(buf), + "# Large transaction header " + "(terminating event offset %llu, type %u)", + static_cast(m_terminating_event_offset), + static_cast(m_terminating_event_type)); + protocol->store_string(buf, bytes, &my_charset_bin); + return 0; +} + +bool Large_transaction_header_log_event::write_data_body( + Basic_ostream *ostream) { + DBUG_TRACE; + uchar fixed[Large_transaction_header_event::kFixedBodyLength]; + fixed[0] = m_version; + int8store(fixed + 1, m_terminating_event_offset); + fixed[1 + sizeof(m_terminating_event_offset)] = m_terminating_event_type; + if (wrapper_my_b_safe_write(ostream, fixed, sizeof(fixed))) return true; + + /* Write the padding in bounded chunks; its contents are undefined and + ignored on read, zeros keep the file deterministic. */ + const uchar zeros[4096] = {0}; + for (uint64_t left = m_padding_size; left > 0;) { + const size_t chunk = std::min(left, sizeof(zeros)); + if (wrapper_my_b_safe_write(ostream, zeros, chunk)) return true; + left -= chunk; + } + return false; +} + +int Large_transaction_header_log_event::do_apply_event(Relay_log_info const *) { + DBUG_TRACE; + /* Nothing to apply: the event only carries recovery metadata for the + file it was written into. */ + return 0; +} +#endif + +#ifndef MYSQL_SERVER +void Large_transaction_header_log_event::print( + FILE *, PRINT_EVENT_INFO *print_event_info) const { + if (print_event_info->short_form) return; + + print_header(&print_event_info->head_cache, print_event_info, false); + my_b_printf(&print_event_info->head_cache, + "\tLarge transaction header\tIgnorable\n"); + my_b_printf(&print_event_info->head_cache, + "# Terminating event offset %llu, type %u, padding %llu bytes\n", + static_cast(m_terminating_event_offset), + static_cast(m_terminating_event_type), + static_cast(m_padding_size)); +} +#endif + Rows_query_log_event::Rows_query_log_event( const char *buf, const Format_description_event *descr_event) : mysql::binlog::event::Ignorable_event(buf, descr_event), @@ -13713,32 +13789,55 @@ Log_event::enum_skip_reason Gtid_log_event::do_shall_skip(Relay_log_info *rli) { } #endif // MYSQL_SERVER -void Gtid_log_event::set_trx_length_by_cache_size_tagged( - ulonglong cache_size, bool is_checksum_enabled, int event_counter) { - auto transaction_length_overhead = cache_size; +/** + Compute the transaction's on-disk length from its cache size, correcting for + any checksum bytes that will be added or removed relative to the cache. + + @param cache_size Byte size of the transaction's cached events. + @param is_checksum_enabled Whether events will carry a checksum on disk. + @param is_checksum_computed Whether the cached events already include a + checksum (so it is not added again per event). + @param event_counter Number of events in the transaction. + @return The adjusted transaction length. +*/ +static ulonglong adjust_trx_length_to_checksum_changes( + ulonglong cache_size, bool is_checksum_enabled, bool is_checksum_computed, + int event_counter) { + ulonglong length = cache_size; if (is_checksum_enabled) { - transaction_length_overhead += (event_counter + 1) * BINLOG_CHECKSUM_LEN; + length += BINLOG_CHECKSUM_LEN; + if (!is_checksum_computed) length += event_counter * BINLOG_CHECKSUM_LEN; + } else if (is_checksum_computed) { + length -= event_counter * BINLOG_CHECKSUM_LEN; } - transaction_length_overhead += LOG_EVENT_HEADER_LEN; + return length; +} + +void Gtid_log_event::set_trx_length_by_cache_size_tagged( + ulonglong cache_size, bool is_checksum_enabled, bool is_checksum_computed, + int event_counter) { + auto transaction_length_overhead = adjust_trx_length_to_checksum_changes( + cache_size, is_checksum_enabled, + is_checksum_computed, event_counter) + + LOG_EVENT_HEADER_LEN; update_tagged_transaction_length(transaction_length_overhead); } void Gtid_log_event::set_trx_length_by_cache_size(ulonglong cache_size, bool is_checksum_enabled, + bool is_checksum_computed, int event_counter) { if (is_tagged()) { - return set_trx_length_by_cache_size_tagged(cache_size, is_checksum_enabled, - event_counter); + return set_trx_length_by_cache_size_tagged( + cache_size, is_checksum_enabled, is_checksum_computed, event_counter); } - // Transaction content length - transaction_length = cache_size; - if (is_checksum_enabled) - transaction_length += event_counter * BINLOG_CHECKSUM_LEN; + // Transaction content length, including all checksums + transaction_length = adjust_trx_length_to_checksum_changes( + cache_size, is_checksum_enabled, is_checksum_computed, event_counter); // GTID length transaction_length += LOG_EVENT_HEADER_LEN; transaction_length += POST_HEADER_LENGTH; - transaction_length += is_checksum_enabled ? BINLOG_CHECKSUM_LEN : 0; transaction_length += get_commit_timestamp_length(); transaction_length += get_server_version_length(); return update_untagged_transaction_length(); diff --git a/sql/log_event.h b/sql/log_event.h index f8715432ac02..0c452cd7b478 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -55,6 +55,7 @@ #include "my_thread_local.h" #include "mysql/binlog/event/binlog_event.h" #include "mysql/binlog/event/control_events.h" +#include "mysql/binlog/event/large_transaction_header_event.h" #include "mysql/binlog/event/load_data_events.h" #include "mysql/binlog/event/rows_event.h" #include "mysql/binlog/event/statement_events.h" @@ -3763,6 +3764,93 @@ class Ignorable_log_event } }; +/** + @class Large_transaction_header_log_event + + Server class of the binlog large transaction optimization header event + (see mysql::binlog::event::Large_transaction_header_event + for the wire format and purpose). Written by the server into the + reserved header region of a promoted binary log file; on the applier + side it is a no-op (and, being flagged ignorable, servers that do not + know the type skip it entirely). + + @internal + The inheritance structure is as follows + + Binary_log_event + ^ + | + B_l:Large_transaction_header_event Log_event + \ / + \ / + Large_transaction_header_log_event + + B_l: namespace mysql::binlog::event + @endinternal +*/ +class Large_transaction_header_log_event + : public mysql::binlog::event::Large_transaction_header_event, + public Log_event { + public: + // disable copy-move semantics + Large_transaction_header_log_event( + Large_transaction_header_log_event &&) noexcept = delete; + Large_transaction_header_log_event &operator=( + Large_transaction_header_log_event &&) noexcept = delete; + Large_transaction_header_log_event( + const Large_transaction_header_log_event &) = delete; + Large_transaction_header_log_event &operator=( + const Large_transaction_header_log_event &) = delete; + +#ifdef MYSQL_SERVER + /** + Creates the event for writing into a promoted binary log file's + reserved header region. + + @param thd_arg THD of the committing session. + @param terminating_event_offset Offset of the transaction's + terminating event in the file. + @param terminating_event_type Type of the transaction's terminating + event in the file. + @param padding_size Filler bytes occupying the remainder + of the reserved region. + */ + Large_transaction_header_log_event( + THD *thd_arg, uint64_t terminating_event_offset, + mysql::binlog::event::Log_event_type terminating_event_type, + uint64_t padding_size) + : mysql::binlog::event::Large_transaction_header_event( + terminating_event_offset, + static_cast(terminating_event_type), padding_size), + Log_event(thd_arg, LOG_EVENT_IGNORABLE_F, Log_event::EVENT_STMT_CACHE, + Log_event::EVENT_NORMAL_LOGGING, header(), footer()) { + DBUG_TRACE; + common_header->set_is_valid(true); + } + + int pack_info(Protocol *protocol) override; + bool write_data_body(Basic_ostream *ostream) override; +#endif + + Large_transaction_header_log_event( + const char *buf, + const mysql::binlog::event::Format_description_event *descr_event); + + ~Large_transaction_header_log_event() override = default; + + void claim_memory_ownership(bool claim) override; + + size_t get_data_size() override { return kFixedBodyLength + m_padding_size; } + +#ifndef MYSQL_SERVER + void print(FILE *file, PRINT_EVENT_INFO *print_event_info) const override; +#endif + +#if defined(MYSQL_SERVER) + int do_apply_event(Relay_log_info const *rli) override; +#endif +}; + /** @class Rows_query_log_event It is used to record the original query for the rows @@ -4189,16 +4277,20 @@ class Gtid_log_event : public mysql::binlog::event::Gtid_event, @param cache_size The size of the binlog cache in bytes. @param is_checksum_enabled If checksum will be added to events on flush. + @param is_checksum_computed If the events in the cache already carry + their checksum. @param event_counter The amount of events in the cache. */ void set_trx_length_by_cache_size(ulonglong cache_size, bool is_checksum_enabled = false, + bool is_checksum_computed = false, int event_counter = 0); /// @copydoc set_trx_length_by_cache_size /// @detail tagged version of event void set_trx_length_by_cache_size_tagged(ulonglong cache_size, bool is_checksum_enabled = false, + bool is_checksum_computed = false, int event_counter = 0); }; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index e60484796785..a659ad05d781 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -794,9 +794,10 @@ MySQL clients support the protocol: #include "sql/auth/authentication_policy.h" #include "sql/auth/sql_authentication.h" // init_rsa_keys #include "sql/auth/sql_security_ctx.h" -#include "sql/auto_thd.h" // Auto_THD -#include "sql/binlog.h" // mysql_bin_log -#include "sql/bootstrap.h" // bootstrap +#include "sql/auto_thd.h" // Auto_THD +#include "sql/binlog.h" // mysql_bin_log +#include "sql/binlog_ostream.h" // binlog_temp_files_dir +#include "sql/bootstrap.h" // bootstrap #include "sql/check_stack.h" #include "sql/conn_handler/connection_acceptor.h" // Connection_acceptor #include "sql/conn_handler/connection_handler_impl.h" // Per_thread_connection_handler @@ -1400,6 +1401,8 @@ ulong binlog_stmt_cache_size = 0; int32 opt_binlog_max_flush_queue_time = 0; long opt_binlog_group_commit_sync_delay = 0; ulong opt_binlog_group_commit_sync_no_delay_count = 0; +bool opt_binlog_large_transaction_optimization_enabled = true; +ulonglong opt_binlog_large_transaction_optimization_threshold = 0; ulonglong max_binlog_stmt_cache_size = 0; ulong refresh_version; /* Increments on each reload */ std::atomic atomic_global_query_id{1}; @@ -1412,6 +1415,8 @@ ulong binlog_cache_use = 0, binlog_cache_disk_use = 0; ulong binlog_stmt_cache_use = 0, binlog_stmt_cache_disk_use = 0; ulong max_connections, max_connect_errors; ulong rpl_stop_replica_timeout = LONG_TIMEOUT; +std::atomic binlog_large_transaction_optimization_count{0}; +std::atomic binlog_large_transaction_optimization_missed_count{0}; bool thread_cache_size_specified = false; bool host_cache_size_specified = false; bool table_definition_cache_specified = false; @@ -6849,6 +6854,7 @@ int init_common_variables() { } } update_parser_max_mem_size(); + update_binlog_large_transaction_optimization_threshold(); update_optimizer_switch(); set_server_version(); @@ -8387,6 +8393,14 @@ static int init_server_components() { unireg_abort(MYSQLD_ABORT_EXIT); } + /* + Initialize the #binlog_temp_files directory for spilled files; if the + directory already exists, clear it. + */ + if (opt_bin_log && !is_help_or_validate_option() && + binlog_temp_files_dir.init(log_bin_basename)) + unireg_abort(MYSQLD_ABORT_EXIT); + if (global_system_variables.binlog_row_value_options != 0) { const char *msg = nullptr; longlong err = ER_BINLOG_ROW_VALUE_OPTION_IGNORED; @@ -9964,6 +9978,8 @@ int mysqld_main(int argc, char **argv) if (mysql_bin_log.write_event_to_binlog_and_sync(&prev_gtids_ev)) unireg_abort(MYSQLD_ABORT_EXIT); + update_binlog_temp_file_previous_gtids_size_estimate( + prev_gtids_ev.common_header->data_written); // run auto purge member function. It will evaluate auto purge controls // and configuration, calculate which log files are to be purged, and @@ -11460,6 +11476,26 @@ static int show_count_hit_query_past_global_conn_mem_status_limit(THD *, return 0; } +static int show_binlog_large_transaction_optimization_count(THD *, + SHOW_VAR *var, + char *buf) { + var->type = SHOW_LONG; + var->value = buf; + *((long *)buf) = (long)(binlog_large_transaction_optimization_count.load( + std::memory_order_relaxed)); + return 0; +} + +static int show_binlog_large_transaction_optimization_missed_count( + THD *, SHOW_VAR *var, char *buf) { + var->type = SHOW_LONG; + var->value = buf; + *((long *)buf) = + (long)(binlog_large_transaction_optimization_missed_count.load( + std::memory_order_relaxed)); + return 0; +} + static int show_count_hit_query_past_conn_mem_status_limit(THD *, SHOW_VAR *var, char *buf) { var->type = SHOW_LONG; @@ -11734,6 +11770,12 @@ SHOW_VAR status_vars[] = { SHOW_SCOPE_GLOBAL}, {"Binlog_cache_use", (char *)&binlog_cache_use, SHOW_LONG, SHOW_SCOPE_GLOBAL}, + {"Binlog_large_transaction_optimization_count", + (char *)&show_binlog_large_transaction_optimization_count, SHOW_FUNC, + SHOW_SCOPE_GLOBAL}, + {"Binlog_large_transaction_optimization_missed_count", + (char *)&show_binlog_large_transaction_optimization_missed_count, + SHOW_FUNC, SHOW_SCOPE_GLOBAL}, {"Binlog_stmt_cache_disk_use", (char *)&binlog_stmt_cache_disk_use, SHOW_LONG, SHOW_SCOPE_GLOBAL}, {"Binlog_stmt_cache_use", (char *)&binlog_stmt_cache_use, SHOW_LONG, diff --git a/sql/mysqld.h b/sql/mysqld.h index ed6d9e6019df..00712881f3ce 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -293,6 +293,8 @@ extern const char *server_build_id_ptr; #endif extern const double log_10[309]; extern ulong binlog_cache_use, binlog_cache_disk_use; +extern std::atomic binlog_large_transaction_optimization_count; +extern std::atomic binlog_large_transaction_optimization_missed_count; extern ulong binlog_stmt_cache_use, binlog_stmt_cache_disk_use; extern ulong aborted_threads; extern ulong delayed_insert_timeout; @@ -323,6 +325,8 @@ extern ulonglong max_binlog_cache_size, max_binlog_stmt_cache_size; extern int32 opt_binlog_max_flush_queue_time; extern long opt_binlog_group_commit_sync_delay; extern ulong opt_binlog_group_commit_sync_no_delay_count; +extern bool opt_binlog_large_transaction_optimization_enabled; +extern ulonglong opt_binlog_large_transaction_optimization_threshold; extern ulong max_binlog_size, max_relay_log_size; extern ulong replica_max_allowed_packet; extern ulong binlog_row_event_max_size; diff --git a/sql/rpl_relay_log_sanitizer.h b/sql/rpl_relay_log_sanitizer.h index da1d7bd5ded4..bf8f9c4caac7 100644 --- a/sql/rpl_relay_log_sanitizer.h +++ b/sql/rpl_relay_log_sanitizer.h @@ -80,6 +80,8 @@ class Relay_log_sanitizer : public binlog::Log_sanitizer { PSI_memory_key &get_memory_key() const override { return key_memory_relaylog_recovery; } + + bool is_relay_log_recovery() const override { return true; } }; } // namespace rpl diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 60ada26b275b..e4d510f1cbfe 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -1283,8 +1283,64 @@ static Sys_var_bool Sys_partial_revokes( ON_CHECK(check_partial_revokes), ON_UPDATE(partial_revokes_update), nullptr, sys_var::PARSE_EARLY); +static void warn_binlog_large_transaction_optimization_threshold_adjusted( + THD *thd, ulonglong previous_threshold, ulonglong adjusted_threshold) { + LogErr(WARNING_LEVEL, ER_BINLOG_BOLT_THRESHOLD_ADJUSTED, previous_threshold, + adjusted_threshold); + if (thd != nullptr) + push_warning_printf( + thd, Sql_condition::SL_WARNING, + ER_BINLOG_BOLT_THRESHOLD_ADJUSTED_SQL_WARNING, + ER_THD(thd, ER_BINLOG_BOLT_THRESHOLD_ADJUSTED_SQL_WARNING), + previous_threshold, adjusted_threshold); +} + +/* + The threshold must not be smaller than binlog_cache_size: a transaction only + qualifies large transaction optimization once its cache has spilled, + which happens at binlog_cache_size, so a lower threshold would never + be the deciding limit. +*/ +static bool adjust_binlog_large_transaction_optimization_threshold(THD *thd) { + if (opt_binlog_large_transaction_optimization_threshold == 0 || + opt_binlog_large_transaction_optimization_threshold >= + static_cast(binlog_cache_size)) + return false; + + const ulonglong previous_threshold = + opt_binlog_large_transaction_optimization_threshold; + opt_binlog_large_transaction_optimization_threshold = binlog_cache_size; + if (opt_binlog_large_transaction_optimization_enabled) + warn_binlog_large_transaction_optimization_threshold_adjusted( + thd, previous_threshold, + opt_binlog_large_transaction_optimization_threshold); + return true; +} + +static bool check_binlog_large_transaction_optimization_threshold( + sys_var *, THD *thd, set_var *var) { + if (var->save_result.ulonglong_value >= + static_cast(binlog_cache_size)) + return false; + + const ulonglong requested_threshold = var->save_result.ulonglong_value; + var->save_result.ulonglong_value = binlog_cache_size; + if (opt_binlog_large_transaction_optimization_enabled) + warn_binlog_large_transaction_optimization_threshold_adjusted( + thd, requested_threshold, var->save_result.ulonglong_value); + return false; +} + +static bool fix_binlog_large_transaction_optimization_enabled(sys_var *, + THD *thd, + enum_var_type) { + adjust_binlog_large_transaction_optimization_threshold(thd); + return false; +} + static bool fix_binlog_cache_size(sys_var *, THD *thd, enum_var_type) { check_binlog_cache_size(thd); + adjust_binlog_large_transaction_optimization_threshold(thd); return false; } @@ -1315,6 +1371,38 @@ static Sys_var_ulong Sys_binlog_stmt_cache_size( NO_MUTEX_GUARD, NOT_IN_BINLOG, ON_CHECK(nullptr), ON_UPDATE(fix_binlog_stmt_cache_size)); +static Sys_var_bool Sys_binlog_large_transaction_optimization_enabled( + "binlog_large_transaction_optimization_enabled", + "Enables the large transaction optimization, which keeps " + "large-transaction commit latency low, avoids stalling concurrent " + "commits, and keeps binary log crash recovery fast regardless of " + "transaction size. When ON (the default), a transaction whose spilled " + "size exceeds binlog_large_transaction_optimization_threshold is " + "committed by promoting its temporary file into the binary log " + "sequence. When OFF, all transactions commit through the standard " + "code path.", + GLOBAL_VAR(opt_binlog_large_transaction_optimization_enabled), + CMD_LINE(OPT_ARG), DEFAULT(true), NO_MUTEX_GUARD, NOT_IN_BINLOG, + ON_CHECK(nullptr), + ON_UPDATE(fix_binlog_large_transaction_optimization_enabled)); + +static Sys_var_ulonglong Sys_binlog_large_transaction_optimization_threshold( + "binlog_large_transaction_optimization_threshold", + "The spilled size in bytes above which a transaction qualifies for the " + "large transaction optimization. Has no effect while " + "binlog_large_transaction_optimization_enabled is OFF.", + GLOBAL_VAR(opt_binlog_large_transaction_optimization_threshold), + CMD_LINE(REQUIRED_ARG), VALID_RANGE(10 * 1024 * 1024, ULLONG_MAX), + DEFAULT(128 * 1024 * 1024), BLOCK_SIZE(1), NO_MUTEX_GUARD, NOT_IN_BINLOG, + ON_CHECK(check_binlog_large_transaction_optimization_threshold), + ON_UPDATE(nullptr)); + +void update_binlog_large_transaction_optimization_threshold() { + if (adjust_binlog_large_transaction_optimization_threshold(nullptr)) + Sys_binlog_large_transaction_optimization_threshold.update_default( + opt_binlog_large_transaction_optimization_threshold); +} + static Sys_var_int32 Sys_binlog_max_flush_queue_time( "binlog_max_flush_queue_time", "The maximum time that the binary log group commit will keep reading" diff --git a/sql/sys_vars.h b/sql/sys_vars.h index f129aee4cf94..68f7ab7e5b11 100644 --- a/sql/sys_vars.h +++ b/sql/sys_vars.h @@ -2841,6 +2841,7 @@ class Sys_var_binlog_encryption : public Sys_var_bool { void update_temptable_max_ram_default(); void update_parser_max_mem_size(); +void update_binlog_large_transaction_optimization_threshold(); void update_optimizer_switch(); #endif /* SYS_VARS_H_INCLUDED */ diff --git a/unittest/gunit/CMakeLists.txt b/unittest/gunit/CMakeLists.txt index aa127a0a53b7..8a3722017f6f 100644 --- a/unittest/gunit/CMakeLists.txt +++ b/unittest/gunit/CMakeLists.txt @@ -307,6 +307,7 @@ SET(SERVER_TESTS ha_info_iterator xid_extract log_event_status_size + binlog_bolt ) # Hangs forever. IF(LINUX_ARM AND CMAKE_BUILD_TYPE_UPPER STREQUAL "RELWITHDEBINFO") diff --git a/unittest/gunit/binlog_bolt-t.cc b/unittest/gunit/binlog_bolt-t.cc new file mode 100644 index 000000000000..d8a3afeb0527 --- /dev/null +++ b/unittest/gunit/binlog_bolt-t.cc @@ -0,0 +1,291 @@ +/* Copyright (c) 2026, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is designed to work with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have either included with + the program or referenced in the documentation. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/* + Unit tests for the pure logic of the binlog large transaction optimization + (BOLT). Everything here is arithmetic or a predicate: no server, no THD, no + locks and no files. + + These three subjects are covered by no MTR test, or by an MTR test that cannot + reach the interesting values: + + - the reserved-region size quantization. Nothing in the MTR suite asserts the + reservation or its rounding at all. + - the reserved-region fit and padding arithmetic. The MTR test for the + "region too small" fallback forces the outcome with a debug symbol rather + than driving the predicate, because the region is sized from the serialized + Previous_gtids estimate at spill time and no configuration makes it too + small while leaving the transaction otherwise promotable. + - is_bolt_temp_file(). MTR covers roughly seven literal names, each at the + cost of a full server start, and cannot practically cover the boundaries. + This predicate has already regressed once: its character class omitted + A-Z while mkstemp() produces upper case, and a name it wrongly rejects is + treated as an unsafe entry, which aborts startup. +*/ + +#include + +#include +#include + +#include "my_inttypes.h" +#include "mysql/binlog/event/binlog_event.h" +#include "mysql/binlog/event/control_events.h" +#include "mysql/binlog/event/large_transaction_header_event.h" +#include "sql/binlog.h" // reserved-bytes helpers +#include "sql/binlog/large_trx_commit.h" // fit and padding arithmetic +#include "sql/binlog_ostream.h" // is_bolt_temp_file, constants + +namespace mysql::binlog::event::unittests { + +namespace { + +/// The quantum the reservation is rounded up to. +constexpr my_off_t kQuantum = kBinlogTempFileReservedBytes; + +/// The headroom always added on top of the Previous_gtids estimate. +constexpr my_off_t kHeadroom = kBinlogTempFilePreviousGtidsHeadroomBytes; + +} // namespace + +// +// The reserved-region size: previous_gtids + headroom, rounded up to the +// quantum. +// + +class BoltReservedBytesTest : public ::testing::Test { + protected: + /// Publish an estimate and read back the reservation it produces. + static my_off_t reservation_for(my_off_t previous_gtids_size) { + update_binlog_temp_file_previous_gtids_size_estimate(previous_gtids_size); + return get_binlog_temp_file_reserved_bytes(); + } +}; + +TEST_F(BoltReservedBytesTest, RoundsUpToTheQuantum) { + // An empty Previous_gtids still needs the headroom, which rounds to one + // quantum. + EXPECT_EQ(reservation_for(0), kQuantum); + EXPECT_EQ(reservation_for(1), kQuantum); + + // Exactly one quantum once the headroom is added: the early return, with no + // rounding applied. + ASSERT_LT(kHeadroom, kQuantum) << "the headroom must be inside one quantum"; + EXPECT_EQ(reservation_for(kQuantum - kHeadroom), kQuantum); + + // One byte past it must cost a whole further quantum. + EXPECT_EQ(reservation_for(kQuantum - kHeadroom + 1), 2 * kQuantum); + + // And the same shape one quantum further out. + EXPECT_EQ(reservation_for(2 * kQuantum - kHeadroom), 2 * kQuantum); + EXPECT_EQ(reservation_for(2 * kQuantum - kHeadroom + 1), 3 * kQuantum); +} + +TEST_F(BoltReservedBytesTest, IsAlwaysAQuantumMultipleAndLeavesHeadroom) { + // The two properties the callers depend on, over a range that crosses + // several quantum boundaries: the spill file's reserved region is a whole + // number of quanta, and it can always hold the estimate plus the headroom. + for (my_off_t estimate = 0; estimate <= 4 * kQuantum; estimate += 997) { + const my_off_t reserved = reservation_for(estimate); + EXPECT_EQ(reserved % kQuantum, my_off_t{0}) << "estimate " << estimate; + EXPECT_GE(reserved, estimate + kHeadroom) << "estimate " << estimate; + // Never more than one quantum of slack beyond what was required. + EXPECT_LT(reserved, estimate + kHeadroom + kQuantum) + << "estimate " << estimate; + } +} + +TEST_F(BoltReservedBytesTest, IsMonotonic) { + my_off_t previous = 0; + for (my_off_t estimate = 0; estimate <= 4 * kQuantum; estimate += 1024) { + const my_off_t reserved = reservation_for(estimate); + EXPECT_GE(reserved, previous) << "estimate " << estimate; + previous = reserved; + } +} + +// +// The reserved-region fit and padding arithmetic. +// + +class BoltHeaderArithmeticTest : public ::testing::Test { + protected: + /// The largest a Gtid event can be, which is what the fit check budgets. + static my_off_t max_gtid_length() { + return static_cast(Gtid_event::get_max_event_length()); + } + + /** + The region is filled exactly when the prefix, the padded + Large_transaction_header event and the Gtid event add up to its size. This + is the invariant write_promoted_binlog_header() asserts after serializing + them, expressed as arithmetic. + */ + static my_off_t region_used(my_off_t reserved, my_off_t gtid_event_length, + my_off_t prefix_length, my_off_t checksum_len) { + const my_off_t padding = large_trx_header_event_padding( + reserved, gtid_event_length, prefix_length, checksum_len); + const my_off_t lth_total = + large_trx_header_event_min_length(checksum_len) + padding; + return prefix_length + lth_total + gtid_event_length + checksum_len; + } +}; + +TEST_F(BoltHeaderArithmeticTest, MinLengthAccountsForTheChecksum) { + const my_off_t without = large_trx_header_event_min_length(0); + const my_off_t with = large_trx_header_event_min_length(BINLOG_CHECKSUM_LEN); + EXPECT_EQ(with, without + BINLOG_CHECKSUM_LEN); + + // A minimal event is its header plus the fixed body, and nothing else. + EXPECT_EQ(without, static_cast( + LOG_EVENT_HEADER_LEN + + Large_transaction_header_event::kFixedBodyLength)); +} + +TEST_F(BoltHeaderArithmeticTest, FitIsExactAtTheBoundary) { + for (const my_off_t checksum_len : + {my_off_t{0}, my_off_t{BINLOG_CHECKSUM_LEN}}) { + const my_off_t prefix = 1024; + // The smallest region that can hold everything the check budgets. + const my_off_t needed = prefix + + large_trx_header_event_min_length(checksum_len) + + max_gtid_length() + checksum_len; + + EXPECT_TRUE(large_trx_header_events_fit(prefix, checksum_len, needed)) + << "checksum_len " << checksum_len; + // One byte short must not fit. This is the boundary MTR cannot reach. + EXPECT_FALSE(large_trx_header_events_fit(prefix, checksum_len, needed - 1)) + << "checksum_len " << checksum_len; + // One byte spare must still fit. + EXPECT_TRUE(large_trx_header_events_fit(prefix, checksum_len, needed + 1)) + << "checksum_len " << checksum_len; + } +} + +TEST_F(BoltHeaderArithmeticTest, PaddingFillsTheRegionExactly) { + // Whatever the actual Gtid event turns out to be, the padding must make the + // header events end precisely at the reserved offset, because that offset is + // where the transaction's first event was written at spill time. + const my_off_t checksum_len = BINLOG_CHECKSUM_LEN; + const my_off_t prefix = 1024; + const my_off_t reserved = 4 * kQuantum; + + ASSERT_TRUE(large_trx_header_events_fit(prefix, checksum_len, reserved)); + + for (my_off_t gtid_length = 1; gtid_length <= max_gtid_length(); + gtid_length += 17) { + EXPECT_EQ(region_used(reserved, gtid_length, prefix, checksum_len), + reserved) + << "gtid_length " << gtid_length; + } + // And at the largest Gtid event the fit check budgets for. + EXPECT_EQ(region_used(reserved, max_gtid_length(), prefix, checksum_len), + reserved); +} + +TEST_F(BoltHeaderArithmeticTest, PaddingDoesNotUnderflowAtTheTightestFit) { + // The fit check budgets Gtid_event::get_max_event_length() while the padding + // subtracts the event's actual length, so the two disagree by construction. + // At the tightest region that still fits, and with the largest possible Gtid + // event, the padding must come out at exactly zero rather than wrapping + // around: my_off_t is unsigned, so an underflow here would become an enormous + // padding size rather than a negative one. + const my_off_t checksum_len = BINLOG_CHECKSUM_LEN; + const my_off_t prefix = 1024; + const my_off_t needed = prefix + + large_trx_header_event_min_length(checksum_len) + + max_gtid_length() + checksum_len; + + ASSERT_TRUE(large_trx_header_events_fit(prefix, checksum_len, needed)); + EXPECT_EQ(large_trx_header_event_padding(needed, max_gtid_length(), prefix, + checksum_len), + my_off_t{0}); + + // A shorter Gtid event in the same region leaves exactly the difference as + // padding. + const my_off_t shorter = max_gtid_length() - 40; + EXPECT_EQ( + large_trx_header_event_padding(needed, shorter, prefix, checksum_len), + my_off_t{40}); +} + +// +// The spill-file name predicate. +// + +class BoltTempFileNameTest : public ::testing::Test { + protected: + static std::string with_prefix(const std::string &suffix) { + return std::string{kBinlogTempFilePrefix} + suffix; + } +}; + +TEST_F(BoltTempFileNameTest, AcceptsTheNamesTheCacheCreates) { + // mkstemp() draws from [A-Za-z0-9], so upper case must be accepted. This is + // the case the original character class got wrong. + EXPECT_TRUE(is_bolt_temp_file(with_prefix("aB3xY9").c_str())); + EXPECT_TRUE(is_bolt_temp_file(with_prefix("ZZZZZZ").c_str())); + EXPECT_TRUE(is_bolt_temp_file(with_prefix("0123456789abcdef").c_str())); + + // The promoted-name form is _, so an embedded + // underscore is part of the alphabet. + EXPECT_TRUE(is_bolt_temp_file(with_prefix("18f2c4a1b_0").c_str())); + EXPECT_TRUE(is_bolt_temp_file(with_prefix("_").c_str())); + + // A single character is enough of a suffix. + EXPECT_TRUE(is_bolt_temp_file(with_prefix("a").c_str())); + EXPECT_TRUE(is_bolt_temp_file(with_prefix("0").c_str())); + EXPECT_TRUE(is_bolt_temp_file(with_prefix("Z").c_str())); +} + +TEST_F(BoltTempFileNameTest, RejectsThePrefixWithNoSuffix) { + // The bare prefix is not a name the cache ever creates, and deleting it + // would mean deleting something the binary log does not own. + EXPECT_FALSE(is_bolt_temp_file(kBinlogTempFilePrefix)); +} + +TEST_F(BoltTempFileNameTest, RejectsNamesOutsideTheAlphabet) { + EXPECT_FALSE(is_bolt_temp_file(with_prefix("a-b").c_str())); + EXPECT_FALSE(is_bolt_temp_file(with_prefix("a.b").c_str())); + EXPECT_FALSE(is_bolt_temp_file(with_prefix("a b").c_str())); + EXPECT_FALSE(is_bolt_temp_file(with_prefix("a/b").c_str())); + EXPECT_FALSE(is_bolt_temp_file(with_prefix("a\tb").c_str())); + // A rejected character anywhere in the suffix is enough, including last. + EXPECT_FALSE(is_bolt_temp_file(with_prefix("abc!").c_str())); +} + +TEST_F(BoltTempFileNameTest, RejectsNamesThatDoNotCarryThePrefix) { + EXPECT_FALSE(is_bolt_temp_file("")); + EXPECT_FALSE(is_bolt_temp_file("bolt")); + EXPECT_FALSE(is_bolt_temp_file("boltx")); + EXPECT_FALSE(is_bolt_temp_file("xbolt_a")); + EXPECT_FALSE(is_bolt_temp_file("not_a_managed_bolt_file")); + EXPECT_FALSE(is_bolt_temp_file("binlog.000001")); + EXPECT_FALSE(is_bolt_temp_file("binlog.index")); + + // The prefix is matched case sensitively. + EXPECT_FALSE(is_bolt_temp_file("BOLT_abc")); + EXPECT_FALSE(is_bolt_temp_file("Bolt_abc")); +} + +} // namespace mysql::binlog::event::unittests diff --git a/unittest/gunit/binlogevents/CMakeLists.txt b/unittest/gunit/binlogevents/CMakeLists.txt index 097ea5047a0d..12b472d6d2c3 100644 --- a/unittest/gunit/binlogevents/CMakeLists.txt +++ b/unittest/gunit/binlogevents/CMakeLists.txt @@ -36,6 +36,7 @@ SET(TESTS grow_calculator gtids heartbeat_codec + large_transaction_header payload_event_buffer_istream transaction_compression transaction_payload_codec diff --git a/unittest/gunit/binlogevents/large_transaction_header-t.cc b/unittest/gunit/binlogevents/large_transaction_header-t.cc new file mode 100644 index 000000000000..bbc1ab333ffd --- /dev/null +++ b/unittest/gunit/binlogevents/large_transaction_header-t.cc @@ -0,0 +1,114 @@ +/* Copyright (c) 2026, Oracle and/or its affiliates. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2.0, + as published by the Free Software Foundation. + + This program is designed to work with certain software (including + but not limited to OpenSSL) that is licensed under separate terms, + as designated in a particular file or component or in included license + documentation. The authors of MySQL hereby grant you an additional + permission to link the program and your derivative works with the + separately licensed software that they have either included with + the program or referenced in the documentation. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License, version 2.0, for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include +#include +#include + +#include "my_byteorder.h" +#include "mysql/binlog/event/binlog_event.h" +#include "mysql/binlog/event/control_events.h" +#include "mysql/binlog/event/large_transaction_header_event.h" + +namespace mysql::binlog::event::unittests { + +class LargeTrxHeaderTest : public ::testing::Test { + protected: + LargeTrxHeaderTest() : m_fde(BINLOG_VERSION, "9.7.0") {} + + /** + Build a serialized LTH event with a full fixed body (version, offset, + terminal event type). The version byte is caller-supplied so that + invalid-version cases can be exercised without truncating the body. + */ + std::vector build_event(uint8_t version, uint64_t offset, + uint8_t terminal_type, size_t padding_size) { + const size_t event_size = LOG_EVENT_MINIMAL_HEADER_LEN + + Large_transaction_header_event::kFixedBodyLength + + padding_size; + std::vector buf(event_size, '\0'); + uchar *p = reinterpret_cast(buf.data()); + int4store(p, 1755000000); // timestamp + p[EVENT_TYPE_OFFSET] = LARGE_TRANSACTION_HEADER_EVENT; + int4store(p + SERVER_ID_OFFSET, 1); + int4store(p + EVENT_LEN_OFFSET, static_cast(event_size)); + int4store(p + LOG_POS_OFFSET, 0); + int2store(p + FLAGS_OFFSET, LOG_EVENT_IGNORABLE_F); + uchar *body = p + LOG_EVENT_MINIMAL_HEADER_LEN; + body[0] = version; + int8store(body + 1, offset); + body[Large_transaction_header_event::kFixedBodyLength - 1] = terminal_type; + return buf; + } + + Format_description_event m_fde; +}; + +TEST_F(LargeTrxHeaderTest, DecodeRoundtrip) { + const uint64_t offsets[] = {0, 1, UINT32_MAX, 0x1122334455667788ULL, + UINT64_MAX}; + const size_t paddings[] = {0, 1, 4096, 128 * 1024}; + for (uint64_t offset : offsets) { + for (size_t padding : paddings) { + auto buf = build_event(Large_transaction_header_event::kVersion, offset, + static_cast(XID_EVENT), padding); + Large_transaction_header_event ev(buf.data(), &m_fde); + ASSERT_TRUE(ev.header()->get_is_valid()); + EXPECT_EQ(ev.get_version(), Large_transaction_header_event::kVersion); + EXPECT_EQ(ev.get_terminating_event_offset(), offset); + EXPECT_EQ(ev.get_terminating_event_type(), + static_cast(XID_EVENT)); + EXPECT_EQ(ev.get_padding_size(), padding); + } + } +} + +TEST_F(LargeTrxHeaderTest, IgnorableFlagIsPreserved) { + auto buf = build_event(Large_transaction_header_event::kVersion, 42, + static_cast(XID_EVENT), 16); + Large_transaction_header_event ev(buf.data(), &m_fde); + ASSERT_TRUE(ev.header()->get_is_valid()); + EXPECT_NE(ev.header()->flags & LOG_EVENT_IGNORABLE_F, 0); +} + +TEST_F(LargeTrxHeaderTest, RejectsUnknownVersion) { + for (uint8_t bad_version : {uint8_t{0}, uint8_t{2}, uint8_t{255}}) { + auto buf = + build_event(bad_version, 42, static_cast(XID_EVENT), 16); + Large_transaction_header_event ev(buf.data(), &m_fde); + EXPECT_FALSE(ev.header()->get_is_valid()); + } +} + +TEST_F(LargeTrxHeaderTest, RejectsTruncatedBody) { + /* Body shorter than the fixed fields: version byte only. */ + auto buf = build_event(Large_transaction_header_event::kVersion, 42, + static_cast(XID_EVENT), 0); + buf.resize(LOG_EVENT_MINIMAL_HEADER_LEN + 1); + uchar *p = reinterpret_cast(buf.data()); + int4store(p + EVENT_LEN_OFFSET, static_cast(buf.size())); + Large_transaction_header_event ev(buf.data(), &m_fde); + EXPECT_FALSE(ev.header()->get_is_valid()); +} + +} // namespace mysql::binlog::event::unittests diff --git a/unittest/gunit/log_event_status_size-t.cc b/unittest/gunit/log_event_status_size-t.cc index d9023397cf90..7894c6bcfe26 100644 --- a/unittest/gunit/log_event_status_size-t.cc +++ b/unittest/gunit/log_event_status_size-t.cc @@ -75,8 +75,13 @@ class LogEventStatusSizeTest : public ::testing::Test { Query_log_event qe(srv.thd(), query.c_str(), query.length(), using_trans, immediate, suppress_use, error, ignore_command); + /* The cache creates its temporary file in #binlog_temp_files, + prepared at server startup; create it here. */ + ASSERT_FALSE(binlog_temp_files_dir.init("./gunit_binlog")); + Binlog_cache_storage os; - os.open(50000, 90000); // random values, bigger than maximal packet size + // random values, bigger than maximal packet size + os.open(50000, 90000, kBinlogTempFileReservedBytes); // set qe values to simulate maximal size of the status variables // artificial data