diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 7618d6d0265..bb2057e1914 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -2219,13 +2219,14 @@ dberr_t LinuxAIOHandler::resubmit(Slot *slot) { return (ret < 0 ? DB_IO_PARTIAL_FAILED : DB_SUCCESS); } -/** Check if the AIO succeeded +/** Check if the AIO succeeded. The caller must have exclusive ownership +of the slot (see find_completed_slot()) but must not hold the array +mutex: the completion processing includes page decryption, which can +take long enough to stall all submitters and handler threads. @param[in,out] slot The slot to check @return DB_SUCCESS, DB_FAIL if the operation should be retried or DB_IO_ERROR on all other errors */ dberr_t LinuxAIOHandler::check_state(Slot *slot) { - ut_ad(m_array->is_mutex_owned()); - /* Note that it may be that there is more then one completed IO requests. We process them one at a time. We may have a case here to improve the performance slightly by dealing with all @@ -2234,7 +2235,7 @@ dberr_t LinuxAIOHandler::check_state(Slot *slot) { srv_set_io_thread_op_info(m_global_segment, "processing completed aio requests"); - ut_ad(slot->io_already_done); + ut_ad(!slot->io_already_done); dberr_t err; @@ -2259,9 +2260,12 @@ dberr_t LinuxAIOHandler::check_state(Slot *slot) { return (err); } -/** If no slot was found then the m_array->m_mutex will be released. +/** Find a slot with a completed IO request and claim exclusive ownership +of it. The mutex is always released before returning; when a completed +slot is found, the caller takes over the slot: it must process it and +free it (AIO::release) without holding the mutex. @param[out] n_pending The number of pending IOs -@return NULL or a slot that has completed IO */ +@return NULL or a claimed slot that has completed IO */ Slot *LinuxAIOHandler::find_completed_slot(ulint *n_pending) { ulint offset = m_n_slots * m_segment; @@ -2276,8 +2280,15 @@ Slot *LinuxAIOHandler::find_completed_slot(ulint *n_pending) { ++*n_pending; if (slot->io_already_done) { - /* Something for us to work on. - Note: We don't release the mutex. */ + /* Something for us to work on. Claim the slot before + releasing the mutex: the kernel event was already reaped + by collect(), so the slot will not be marked again, and + other handler threads skip slots with io_already_done + == false. Completion processing (page decryption et al.) + must not run while holding the array mutex, otherwise + all submitters and handler threads serialize behind it. */ + slot->io_already_done = false; + m_array->release(); return (slot); } } @@ -2437,7 +2448,10 @@ dberr_t LinuxAIOHandler::poll(fil_node_t **m1, void **m2, IORequest *request) { slot = find_completed_slot(&n_pending); if (slot != nullptr) { - ut_ad(m_array->is_mutex_owned()); + /* We now exclusively own the slot and the mutex is + released: the completion processing below includes page + decryption, which must not hold up the other submitters + and handler threads. */ err = check_state(slot); @@ -2448,14 +2462,14 @@ dberr_t LinuxAIOHandler::poll(fil_node_t **m1, void **m2, IORequest *request) { /* Partial IO, resubmit request for remaining bytes to read/write */ + m_array->acquire(); err = resubmit(slot); + m_array->release(); if (err != DB_SUCCESS) { break; } - m_array->release(); - } else if (is_shutdown() && n_pending == 0) { /* There is no completed request. If there is no pending request at all, and the system is @@ -2492,8 +2506,8 @@ dberr_t LinuxAIOHandler::poll(fil_node_t **m1, void **m2, IORequest *request) { *request = slot->type; + m_array->acquire(); m_array->release(slot); - m_array->release(); return (err);