From 478220ce7963b62328a27a5044d77bc90d58f35d Mon Sep 17 00:00:00 2001 From: lipu Date: Sun, 6 Sep 2026 09:31:25 +0800 Subject: [PATCH] [bugfix] decrypt pages outside the AIO array mutex In the Linux native AIO completion path, find_completed_slot() returned the completed slot while still holding the AIO array mutex, so the whole completion processing - check_state(), post_io_processing(), io_complete() and, for encrypted or compressed tablespaces, the page decryption and decompression - ran under that mutex. Every I/O handler thread and every submitter (AIO::reserve_slot() needs the same mutex) serialized behind it, capping completion throughput at one page-processing time regardless of the number of handler threads. With AES-NI a 16 KiB page decrypts in ~16 us and the damage stays invisible, but with a slower software cipher implementation (~40 us per page) aggregate throughput saturates at ~40 MB/s no matter how many handler threads run, and page cleaner write-out stalls behind read completions as well. Fix: find_completed_slot() now claims the slot under the mutex by clearing io_already_done - the kernel event was already reaped by collect(), so the slot cannot be marked again, and other handler threads skip slots with io_already_done == false - and releases the mutex before returning. The caller then exclusively owns the slot: completion processing, including page decryption, runs without the mutex held. poll() re-acquires the mutex only to resubmit a partially completed request and to free the slot via AIO::release(). This matches the existing patterns: the write path already performs page compression and encryption outside the array mutex (AIO::reserve_slot()), and the simulated AIO handler performs its completion processing without holding the mutex. Measured on the same code base (MySQL 8.0.25 lineage) with a cold full-table-scan workload over encrypted tablespaces on a RAM disk: throughput with a software cipher went from 0.038 GB/s to 1.18 GB/s at 8 handler threads (31x), AES improved 3x, and unencrypted throughput was unchanged. Old and new servers read identical data (full-table CRC32 over an encrypted tablespace), and the MTR encryption suite passes fully on a debug build. --- storage/innobase/os/os0file.cc | 38 +++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 7618d6d0265..bb2057e1914 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2219,13 +2219,14 @@ dberr_t LinuxAIOHandler::resubmit(Slot *slot) { return (ret < 0 ? DB_IO_PARTIAL_FAILED : DB_SUCCESS); } -/** Check if the AIO succeeded +/** Check if the AIO succeeded. The caller must have exclusive ownership +of the slot (see find_completed_slot()) but must not hold the array +mutex: the completion processing includes page decryption, which can +take long enough to stall all submitters and handler threads. @param[in,out] slot The slot to check @return DB_SUCCESS, DB_FAIL if the operation should be retried or DB_IO_ERROR on all other errors */ dberr_t LinuxAIOHandler::check_state(Slot *slot) { - ut_ad(m_array->is_mutex_owned()); - /* Note that it may be that there is more then one completed IO requests. We process them one at a time. We may have a case here to improve the performance slightly by dealing with all @@ -2234,7 +2235,7 @@ dberr_t LinuxAIOHandler::check_state(Slot *slot) { srv_set_io_thread_op_info(m_global_segment, "processing completed aio requests"); - ut_ad(slot->io_already_done); + ut_ad(!slot->io_already_done); dberr_t err; @@ -2259,9 +2260,12 @@ dberr_t LinuxAIOHandler::check_state(Slot *slot) { return (err); } -/** If no slot was found then the m_array->m_mutex will be released. +/** Find a slot with a completed IO request and claim exclusive ownership +of it. The mutex is always released before returning; when a completed +slot is found, the caller takes over the slot: it must process it and +free it (AIO::release) without holding the mutex. @param[out] n_pending The number of pending IOs -@return NULL or a slot that has completed IO */ +@return NULL or a claimed slot that has completed IO */ Slot *LinuxAIOHandler::find_completed_slot(ulint *n_pending) { ulint offset = m_n_slots * m_segment; @@ -2276,8 +2280,15 @@ Slot *LinuxAIOHandler::find_completed_slot(ulint *n_pending) { ++*n_pending; if (slot->io_already_done) { - /* Something for us to work on. - Note: We don't release the mutex. */ + /* Something for us to work on. Claim the slot before + releasing the mutex: the kernel event was already reaped + by collect(), so the slot will not be marked again, and + other handler threads skip slots with io_already_done + == false. Completion processing (page decryption et al.) + must not run while holding the array mutex, otherwise + all submitters and handler threads serialize behind it. */ + slot->io_already_done = false; + m_array->release(); return (slot); } } @@ -2437,7 +2448,10 @@ dberr_t LinuxAIOHandler::poll(fil_node_t **m1, void **m2, IORequest *request) { slot = find_completed_slot(&n_pending); if (slot != nullptr) { - ut_ad(m_array->is_mutex_owned()); + /* We now exclusively own the slot and the mutex is + released: the completion processing below includes page + decryption, which must not hold up the other submitters + and handler threads. */ err = check_state(slot); @@ -2448,14 +2462,14 @@ dberr_t LinuxAIOHandler::poll(fil_node_t **m1, void **m2, IORequest *request) { /* Partial IO, resubmit request for remaining bytes to read/write */ + m_array->acquire(); err = resubmit(slot); + m_array->release(); if (err != DB_SUCCESS) { break; } - m_array->release(); - } else if (is_shutdown() && n_pending == 0) { /* There is no completed request. If there is no pending request at all, and the system is @@ -2492,8 +2506,8 @@ dberr_t LinuxAIOHandler::poll(fil_node_t **m1, void **m2, IORequest *request) { *request = slot->type; + m_array->acquire(); m_array->release(slot); - m_array->release(); return (err);