From 4d3ae6fa756bbc81f0aadf1488255b6e4a4394c9 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 21:51:23 +0000 Subject: [PATCH 1/4] ipc: split the kernel part of ipc-common.c into src/kernel/ipc ipc-common.c mixes code that only runs in kernel mode with code that also runs in the user-space IPC thread. Move the kernel part to src/kernel/ipc/ipc-common-kernel.c: - the IPC context and ipc_get() of user-space LL builds; - the IPC task, command completion and IDC forwarding to other cores; - the outgoing message queue and its work queue, and the kernel half of the ipc_msg_send() and ipc_msg_list_remove() system calls; - the setup of the user-space IPC threads, their kernel-only object copies, their registration as SOF system call callers and command forwarding to them; - the cross-core request slot semaphores and the access grants between the IPC and LL threads of the cores; - ipc_init(). ipc-common.c keeps message allocation, the component lookups and, for user-space LL builds, the user-space IPC thread function, the default command dispatch and the cross-core request helpers, which the user-space IPC and LL threads use instead of IDC. The user thread entry point and its event bits are now shared through sof/ipc/common.h. The unit tests that build ipc-common.c also build the kernel part. The code is unchanged. Only the Xtensa assembler's alignment padding and its choice between narrow and wide encodings of the same instructions differ in a few functions, as do the line numbers the moved functions pass to the assertion handler in builds with assertions (qemu_xtensa). In user-space LL builds the cross-core request helpers call ipc_get() instead of inlining it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/ipc/common.h | 14 + src/ipc/ipc-common.c | 843 +---------------- src/kernel/ipc/CMakeLists.txt | 2 +- src/kernel/ipc/ipc-common-kernel.c | 895 ++++++++++++++++++ test/cmocka/src/audio/buffer/CMakeLists.txt | 4 + .../cmocka/src/audio/component/CMakeLists.txt | 1 + test/cmocka/src/audio/dcblock/CMakeLists.txt | 1 + test/cmocka/src/audio/eq_fir/CMakeLists.txt | 1 + test/cmocka/src/audio/eq_iir/CMakeLists.txt | 1 + test/cmocka/src/audio/mixer/CMakeLists.txt | 1 + .../src/audio/pcm_converter/CMakeLists.txt | 1 + test/cmocka/src/audio/pipeline/CMakeLists.txt | 3 + test/cmocka/src/audio/selector/CMakeLists.txt | 1 + test/cmocka/src/audio/volume/CMakeLists.txt | 1 + test/cmocka/src/math/fft/CMakeLists.txt | 1 + test/ztest/unit/audio/drc/CMakeLists.txt | 1 + test/ztest/unit/math/fft/CMakeLists.txt | 1 + 17 files changed, 931 insertions(+), 841 deletions(-) create mode 100644 src/kernel/ipc/ipc-common-kernel.c diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 64839766dfb3..d0bb5fca9d6e 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -356,6 +356,20 @@ void ipc_complete_cmd(struct ipc *ipc); extern bool ipc_enter_gdb; #ifdef CONFIG_SOF_USERSPACE_LL +/* Events that wake the user-space IPC thread */ +#define IPC_USER_EVENT_CMD BIT(0) +#define IPC_USER_EVENT_STOP BIT(1) +#define IPC_USER_EVENT_XCORE BIT(2) + +/** + * @brief User-space IPC thread entry point. + * + * @param p1 struct ipc_user shared with the kernel + * @param p2 Core ID + * @param p3 Unused + */ +void ipc_user_thread_fn(void *p1, void *p2, void *p3); + /** * @brief Forward an IPC command to the user-space thread and wait for it. * diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index b74b4a679d7e..aaa3d6d98123 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -50,25 +50,10 @@ #endif #include -#include -LOG_MODULE_REGISTER(ipc, CONFIG_SOF_LOG_LEVEL); +/* The IPC code that runs in kernel mode is in src/kernel/ipc/ipc-common-kernel.c */ -SOF_DEFINE_REG_UUID(ipc); - -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(ipc_tr, SOF_UUID(ipc_uuid), LOG_LEVEL_INFO); - -#ifdef CONFIG_SOF_USERSPACE_LL -K_APPMEM_PARTITION_DEFINE(ipc_context_part); - -K_APP_BMEM(ipc_context_part) static struct ipc ipc_context; - -struct ipc *ipc_get(void) -{ - return &ipc_context; -} -#endif +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, uint32_t extension, uint32_t size) @@ -101,46 +86,6 @@ struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, return msg; } -int ipc_process_on_core(uint32_t core, bool blocking) -{ - struct ipc *ipc = ipc_get(); - struct idc_msg msg = { .header = IDC_MSG_IPC, .core = core, }; - int ret; - - /* check if requested core is enabled */ - if (!cpu_is_core_enabled(core)) { - tr_err(&ipc_tr, "core #%d is disabled", core); - return -EACCES; - } - -#if CONFIG_IPC_MAJOR_3 - /* The other core will write there its response */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - ((struct sof_ipc_cmd_hdr *)ipc->comp_data)->size); -#endif - - /* - * If the primary core is waiting for secondary cores to complete, it - * will also reply to the host - */ - if (!blocking) { - k_spinlock_key_t key; - - ipc->core = core; - key = k_spin_lock(&ipc->lock); - ipc->task_mask |= IPC_TASK_SECONDARY_CORE; - k_spin_unlock(&ipc->lock, key); - } - - /* send IDC message */ - ret = idc_send_msg(&msg, blocking ? IDC_BLOCKING : IDC_NON_BLOCKING); - if (ret < 0) - return ret; - - /* reply written by other core */ - return 1; -} - /* * Components, buffers and pipelines are stored in the same lists, hence * type and ID have to be used for the identification. @@ -202,320 +147,7 @@ struct ipc_comp_dev *ipc_get_ppl_comp(struct ipc *ipc, uint32_t pipeline_id, int return next_ppl_icd; } -void ipc_send_queued_msg(void) -{ - struct ipc *ipc = ipc_get(); - struct ipc_msg *msg; - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - - if (ipc->pm_prepare_D3) - goto out; - - /* any messages to send ? */ - if (list_is_empty(&ipc->msg_list)) - goto out; - - msg = list_first_item(&ipc->msg_list, struct ipc_msg, - list); - - if (ipc_platform_send_msg(msg) == 0) { - /* Remove the message from the list if it has been successfully sent. */ - list_item_del(&msg->list); - /* Invoke a callback to notify that the message has been sent. */ - if (msg->callback) - msg->callback(msg); -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - /* Increment performance counters */ - io_perf_monitor_update_data(ipc->io_perf_out_msg_count, 1); -#endif - } -out: - k_spin_unlock(&ipc->lock, key); -} - -#ifdef __ZEPHYR__ -static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); - -/* - * Kernel-only: these must not live in struct ipc, which user threads can - * write with CONFIG_SOF_USERSPACE_LL. The work queue embeds the k_thread of - * a supervisor thread and the work item holds the handler it calls. - */ -static struct k_work_q ipc_send_wq; -static struct k_work_delayable ipc_send_work; - -struct k_work_q *ipc_send_wq_get(void) -{ - return &ipc_send_wq; -} -#endif - -static void schedule_ipc_worker(void) -{ - /* - * note: in XTOS builds, this is handled in - * task_main_primary_core() - */ -#ifdef __ZEPHYR__ - k_work_schedule_for_queue(&ipc_send_wq, &ipc_send_work, K_USEC(IPC_PERIOD_USEC)); -#endif -} - -__cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - int ret; - - assert_can_be_cold(); - - key = k_spin_lock(&ipc->lock); - - /* copy mailbox data to message if not already copied */ - if (data && msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE && - msg->tx_data != data) { - ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); - assert(!ret); - } - - ipc_platform_send_msg_direct(msg); - - k_spin_unlock(&ipc->lock, key); -} - -void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - int ret; - - key = k_spin_lock(&ipc->lock); - - /* copy mailbox data to message if not already copied */ - if (data && (msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE) && - msg->tx_data != data) { - ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); - assert(!ret); - } - - /* - * note: This function can be executed in LL or EDF context, from any core. - * In Zephyr builds, there is IPC queue that is always handled by the primary core, - * whereas submitting to the queue is allowed from any core. Therefore disable option - * of sending IPC immediately by any context/core to secure IPC registers/mailbox - * access. - */ -#ifndef __ZEPHYR__ - /* try to send critical notifications right away */ - if (high_priority) { - ret = ipc_platform_send_msg(msg); - if (!ret) { - k_spin_unlock(&ipc->lock, key); - return; - } - } -#endif - /* add to queue unless already there */ - if (list_is_empty(&msg->list)) { - if (high_priority) - list_item_prepend(&msg->list, &ipc->msg_list); - else - list_item_append(&msg->list, &ipc->msg_list); - } - - schedule_ipc_worker(); - - k_spin_unlock(&ipc->lock, key); -} -EXPORT_SYMBOL(ipc_msg_send); - -#ifdef CONFIG_USERSPACE -void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) -{ - /* No caller check: DP modules send notifications, see mod_ipc_msg_init() */ - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); - - K_OOPS(K_SYSCALL_VERIFY(msg->tx_size <= SOF_IPC_MSG_MAX_SIZE)); - - K_OOPS(K_SYSCALL_VERIFY(list_is_empty(&msg->list))); - - if (msg->tx_size > 0) { - K_OOPS(K_SYSCALL_VERIFY(msg->tx_data != NULL)); - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg->tx_data, msg->tx_size)); - } - - /* If data != NULL and tx_size > 0, verify the data buffer */ - if (data && msg->tx_size > 0) - K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); - - z_impl_ipc_msg_send(msg, data, high_priority); -} -#include -#endif /* CONFIG_USERSPACE */ - -void z_impl_ipc_msg_list_remove(struct ipc_msg *msg) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - list_item_del(&msg->list); - k_spin_unlock(&ipc->lock, key); -} - -#ifdef CONFIG_USERSPACE -void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) -{ - /* No caller check: called by mod_ipc_msg_free() in DP module threads */ - struct ipc *ipc = ipc_get(); - struct list_item *mlist, *_mlist; - bool found = false; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); - - /* - * special case: empty list was passed. we can't trust where - * list->prev points to, so do not pass to - * z_impl_ipc_msg_list_remove(), but handle here - */ - if (list_is_empty(&msg->list)) - return; - - list_for_item_safe(mlist, _mlist, &ipc->msg_list) { - if (mlist == &msg->list) { - found = true; - break; - } - } - K_OOPS(K_SYSCALL_VERIFY(found)); - z_impl_ipc_msg_list_remove(msg); -} -#include -#endif /* CONFIG_USERSPACE */ - -#ifdef __ZEPHYR__ -static void ipc_work_handler(struct k_work *work) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - - ipc_send_queued_msg(); - - key = k_spin_lock(&ipc->lock); - - if (!list_is_empty(&ipc->msg_list) && !ipc->pm_prepare_D3) - schedule_ipc_worker(); - - k_spin_unlock(&ipc->lock, key); -} -#endif - -void ipc_schedule_process(struct ipc *ipc) -{ -#if CONFIG_TWB_IPC_TASK - schedule_task(ipc->ipc_task, 0, IPC_PERIOD_USEC); -#else - schedule_task(&ipc->ipc_task, 0, IPC_PERIOD_USEC); -#endif -} - #ifdef CONFIG_SOF_USERSPACE_LL -/* Generic user-space IPC handling thread infrastructure. Protocol-specific - * command interpretation lives in ipc_user_thread_dispatch(), implemented by - * the active IPC major (see src/ipc/ipc4/). - */ - -#define IPC_USER_EVENT_CMD BIT(0) -#define IPC_USER_EVENT_STOP BIT(1) -#define IPC_USER_EVENT_XCORE BIT(2) - -BUILD_ASSERT(IPC_USER_XCORE_PAYLOAD_SIZE >= IDC_MAX_PAYLOAD_SIZE, - "cross-core request payload smaller than IDC payload"); -#if CONFIG_IPC_MAJOR_4 -BUILD_ASSERT(sizeof(((struct ipc_user_xcore *)0)->ipc4_bind_data) == - sizeof(struct ipc4_module_bind_unbind), - "cross-core request bind data size mismatch"); -#endif - -SOF_DEFINE_REG_UUID(sec_core_init); - -static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, - CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); - -/* - * Kernel-only copies of the IPC user thread state. struct ipc_user and the - * ipc->ipc_user_pdata pointer are both writable by user threads, so kernel - * code must use these instead of reading kernel object pointers from there. - */ -static struct ipc_user *ipc_user_kptr; -static struct k_sem *ipc_user_ksem; -static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; -static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; -static struct k_thread *ipc_user_kaudio_thread[CONFIG_CORE_COUNT]; -static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; - -/* Cross-core request slot semaphores, static so they're never freed */ -static struct k_sem ipc_user_xcore_lock[CONFIG_CORE_COUNT]; -static struct k_sem ipc_user_xcore_done[CONFIG_CORE_COUNT]; - -/** - * @brief Forward an IPC command to the user-space thread. - * - * Called from kernel context (IPC EDF task) to forward the message words - * to the user-space thread for processing. Sets IPC_TASK_IN_THREAD in - * task_mask so the host is not signaled until the user thread completes. - * Blocks until the user thread finishes processing and returns the result. - * - * @param primary Primary message word - * @param extension Extension message word - * @return Result from user thread processing - */ -int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) -{ - struct ipc *ipc = ipc_get(); - struct ipc_user *pdata = ipc_user_kptr; - k_spinlock_key_t key; - int ret; - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - LOG_DBG("IPC: forward cmd %08x", primary); - - /* Copy message words — original buffer may be reused */ - pdata->ipc_msg_pri = primary; - pdata->ipc_msg_ext = extension; - pdata->ipc = ipc; - - /* Prevent host completion until user thread finishes */ - key = k_spin_lock(&ipc->lock); - ipc->task_mask |= IPC_TASK_IN_THREAD; - k_spin_unlock(&ipc->lock, key); - - /* - * Wake the user thread. Post, not set: a cross-core request - * (IPC_USER_EVENT_XCORE) may be pending on the same event. - */ - k_event_post(ipc_user_kevent[core], IPC_USER_EVENT_CMD); - - /* Wait for user thread to complete */ - ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); - if (ret) { - LOG_ERR("IPC user: sem error %d\n", ret); - /* fall through to complete the cmd */ - } - - /* Clear the task mask bit and check for completion */ - key = k_spin_lock(&ipc->lock); - ipc->task_mask &= ~IPC_TASK_IN_THREAD; - ipc_complete_cmd(ipc); - k_spin_unlock(&ipc->lock, key); - - return !ret ? pdata->result : ret; -} - void ipc_user_xcore_reserve(uint32_t cores) { struct ipc_user *ipc_user = ipc_get()->ipc_user_pdata; @@ -651,7 +283,7 @@ __weak int ipc_user_thread_dispatch(struct ipc_user *ipc_user) * User-space IPC thread entry point. p1 points to the ipc_user context * shared with the kernel launcher. */ -static void ipc_user_thread_fn(void *p1, void *p2, void *p3) +void ipc_user_thread_fn(void *p1, void *p2, void *p3) { struct ipc_user *ipc_user = p1; unsigned int core = POINTER_TO_UINT(p2); @@ -696,473 +328,4 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) break; } } - -__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) -{ - char thread_name[] = "ll_user0"; - struct k_thread *thread; - struct k_event *event; - int ret; - - assert_can_be_cold(); - - /* - * Build the objects in locals and publish them to struct ipc_user only - * at the end. When a secondary core is brought up, user threads on - * other cores are already running and can overwrite struct ipc_user, - * so its fields must not be read back here. - */ - event = k_object_alloc(K_OBJ_EVENT); - if (!event) { - LOG_ERR("user IPC event alloc failed"); - return -ENOMEM; - } - k_event_init(event); - - thread = k_object_alloc(K_OBJ_THREAD); - if (!thread) { - LOG_ERR("user IPC thread alloc failed"); - ret = -ENOMEM; - goto e_event; - } - - k_thread_create(thread, ipc_user_stack[core], - CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, - ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, - -1, K_USER, K_FOREVER); - - k_thread_cpu_pin(thread, core); - thread_name[sizeof(thread_name) - 2] = '0' + core; - k_thread_name_set(thread, thread_name); - - /* - * Each userspace IPC thread must be able to wait on its private event - * and signal completion on the primary core semaphore - */ - k_thread_access_grant(thread, ipc_user_ksem, event); - user_grant_dai_access_all(thread); - user_grant_dma_access_all(thread); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); - user_ll_grant_access(thread, core); - pipeline_posn_grant_access(thread); - - ipc_user_kevent[core] = event; - ipc_user_kthread[core] = thread; - sof_sysuser_thread_set(SOF_SYSUSER_IPC, core, thread); - - /* for ipc_user_thread_fn() and ipc_thread_user() in user context */ - ipc_user->event[core] = event; - ipc_user->thread[core] = thread; - - return 0; - -e_event: - k_object_free(event); - - return ret; -} - -/* - * Grant the threads of the newly initialized core and of the other already - * initialized cores access to each other's cross-core request slots and LL - * scheduler locks. Uses only the kernel-only copies of the thread state. - */ -__cold static void ipc_user_xcore_grant(unsigned int core) -{ - struct k_thread *new_threads[] = { - ipc_user_kthread[core], ipc_user_kaudio_thread[core], - }; - unsigned int c, i; - - assert_can_be_cold(); - - for (c = 0; c < CONFIG_CORE_COUNT; c++) { - struct k_thread *threads[] = { - ipc_user_kthread[c], ipc_user_kaudio_thread[c], - }; - - /* - * Grant the slot semaphores of all cores. Note: braces needed, - * k_thread_access_grant() expands to multiple statements. - */ - for (i = 0; i < ARRAY_SIZE(new_threads); i++) { - if (new_threads[i]) { - k_thread_access_grant(new_threads[i], &ipc_user_xcore_lock[c], - &ipc_user_xcore_done[c]); - } - } - - if (!ipc_user_kthread[c]) - continue; - - for (i = 0; i < ARRAY_SIZE(threads); i++) { - if (!threads[i]) - continue; - k_thread_access_grant(threads[i], ipc_user_kevent[core]); - user_ll_grant_access(threads[i], core); - } - - for (i = 0; i < ARRAY_SIZE(new_threads); i++) { - if (!new_threads[i]) - continue; - k_thread_access_grant(new_threads[i], ipc_user_kevent[c]); - user_ll_grant_access(new_threads[i], c); - } - } -} - -__cold int ipc_user_init_secondary(unsigned int core) -{ - struct ipc_user *ipc_user = ipc_user_kptr; - struct k_thread *audio_thread; - int ret; - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - ret = ipc_user_init_thread(ipc_user, core); - - if (ret < 0) - return ret; - - assert_can_be_cold(); - - k_thread_start(ipc_user_kthread[core]); - - struct task *task = zephyr_ll_task_alloc(); - - if (!task) { - LOG_ERR("user LL task allocation failed"); - k_panic(); - } - - schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, core, 0); - - audio_thread = scheduler_init_context(task); - if (!audio_thread) { - LOG_ERR("user LL thread init failed"); - k_panic(); - } - - k_thread_access_grant(ipc_user_kthread[core], audio_thread); - ipc_user_kaudio_thread[core] = audio_thread; - ipc_user->audio_thread[core] = audio_thread; - ipc_user_xcore_grant(core); - - return 0; -} - -struct k_thread *ipc_thread_user(unsigned int core) -{ - if (core >= CONFIG_CORE_COUNT) - return NULL; - - /* user threads can not read the kernel-only copy */ - if (k_is_user_context()) - return ipc_get()->ipc_user_pdata->thread[core]; - - return ipc_user_kthread[core]; -} - -struct ipc_user *ipc_user_get(void) -{ - return ipc_user_kptr; -} - -void ipc_user_wait_core_init(unsigned int core) -{ - if (core >= CONFIG_CORE_COUNT) - return; - - if (ipc_user_kinit_needed[core]) { - /* wait for IPC thread (ipc_user_thread_fn()) */ - k_sem_take(ipc_user_ksem, K_FOREVER); - ipc_user_kinit_needed[core] = false; - } -} - -/* Primary core only */ -__cold static void ipc_user_init(void) -{ - struct ipc *ipc = ipc_get(); - struct k_thread *audio_thread; - struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ipc_user), 0); - int ret, core; - - if (!ipc_user) { - LOG_ERR("user IPC pdata alloc failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - - /* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */ - memset(ipc_user, 0, sizeof(*ipc_user)); - - assert_can_be_cold(); - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { - if (core != PLATFORM_PRIMARY_CORE_ID) - ipc_user_kinit_needed[core] = true; - } - - ipc_user_ksem = k_object_alloc(K_OBJ_SEM); - if (!ipc_user_ksem) { - LOG_ERR("user IPC sem alloc failed"); - k_panic(); - } - ipc_user_kptr = ipc_user; - ipc_user->sem = ipc_user_ksem; - - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); - if (ret < 0) - LOG_WRN("ipc context partition add failed: %d", ret); - - /* - * Grant user-space read access to the .coldrodata section in IMR. The - * prepare path walks component code that may reference __cold_rodata - * data. - * - * .cold text is deliberately NOT added as a partition: Zephyr already - * maps it executable in the shared ring. - */ -#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM - extern char __coldrodata_start[]; - extern char _imr_end[]; - - if (&_imr_end[0] > &__coldrodata_start[0]) { - struct k_mem_partition cold_part = { - .start = (uintptr_t)__coldrodata_start, - .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, - CONFIG_MMU_PAGE_SIZE), - .attr = K_MEM_PARTITION_P_RO_U_RO, - }; - - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); - if (ret < 0) - LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", - cold_part.size, cold_part.start, ret); - } -#endif /* CONFIG_COLD_STORE_EXECUTE_DRAM */ - - k_sem_init(ipc_user_ksem, 0, 1); - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { - struct ipc_user_xcore *xc = &ipc_user->xcore[core]; - - xc->lock = &ipc_user_xcore_lock[core]; - xc->done = &ipc_user_xcore_done[core]; - k_sem_init(xc->lock, 1, 1); - k_sem_init(xc->done, 0, 1); - } - - ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); - if (ret < 0) { - LOG_ERR("user IPC thread initialization failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - - ret = user_access_to_mailbox(zephyr_ll_mem_domain(), - ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); - if (ret < 0) { - LOG_ERR("ipc user: mailbox access grant failed: %d", ret); - sof_panic(SOF_IPC_PANIC_IPC); - } - - /* Store references in ipc struct so kernel handler can forward commands */ - ipc->ipc_user_pdata = ipc_user; - - struct task *task = zephyr_ll_task_alloc(); - - if (!task) { - LOG_ERR("task allocation failed"); - k_panic(); - } - - schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); - audio_thread = scheduler_init_context(task); - if (!audio_thread) { - LOG_ERR("user LL thread init failed"); - k_panic(); - } - - /* Grant ipc_user thread permission on the audio thread object. - * Needed so user-space dai_common_new() can call - * k_thread_access_grant(audio_thread, dai_mutex) from user context. - */ - k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); - ipc_user_kaudio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; - ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; - ipc_user_xcore_grant(PLATFORM_PRIMARY_CORE_ID); - - k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); - - /* Wait for user thread startup — consumes the initial k_sem_give from thread */ - k_sem_take(ipc_user_ksem, K_FOREVER); -} -#else /* CONFIG_SOF_USERSPACE_LL */ -static void ipc_user_init(void) -{ -} #endif /* CONFIG_SOF_USERSPACE_LL */ - -__cold int ipc_init(struct sof *sof) -{ - struct k_heap *heap; - struct ipc *ipc; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "entry"); - -#ifdef CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); - - ipc = ipc_get(); - memset(ipc, 0, sizeof(*ipc)); - ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ipc->ll_alloc), 0); - if (!ipc->ll_alloc) { - tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); - sof_panic(SOF_IPC_PANIC_IPC); - } - ipc->ll_alloc->heap = heap; - ipc->ll_alloc->vreg = NULL; -#else - heap = NULL; - - /* init ipc data */ - ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc)); - if (!ipc) { - tr_err(&ipc_tr, "Unable to allocate IPC data"); - return -ENOMEM; - } - sof->ipc = ipc; -#endif /* CONFIG_SOF_USERSPACE_LL */ - - ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - SOF_IPC_MSG_MAX_SIZE, 0); - if (!ipc->comp_data) { - tr_err(&ipc_tr, "Unable to allocate IPC component data"); - sof_panic(SOF_IPC_PANIC_IPC); - } - memset(ipc->comp_data, 0, SOF_IPC_MSG_MAX_SIZE); - - k_spinlock_init(&ipc->lock); - list_init(&ipc->msg_list); - list_init(&ipc->comp_list); - -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - struct io_perf_data_item init_data = {IO_PERF_IPC_ID, - cpu_get_id(), - IO_PERF_INPUT_DIRECTION, - IO_PERF_POWERED_UP_ENABLED, - IO_PERF_D0IX_POWER_MODE, - 0, 0, 0 }; - io_perf_monitor_init_data(&ipc->io_perf_in_msg_count, &init_data); - init_data.direction = IO_PERF_OUTPUT_DIRECTION; - io_perf_monitor_init_data(&ipc->io_perf_out_msg_count, &init_data); -#endif - -#if CONFIG_SOF_BOOT_TEST_STANDALONE - LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); - return 0; -#endif - -#ifdef __ZEPHYR__ - k_tid_t thread; - - k_work_queue_init(&ipc_send_wq); - k_work_queue_start(&ipc_send_wq, ipc_send_wq_stack, - K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); - - thread = k_work_queue_thread_get(&ipc_send_wq); - - k_thread_suspend(thread); - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); -#endif - k_thread_name_set(thread, "ipc_send_wq"); - - k_thread_resume(thread); - - k_work_init_delayable(&ipc_send_work, ipc_work_handler); -#endif /* __ZEPHYR__ */ - - ipc_user_init(); - - return platform_ipc_init(ipc); -} - -/* Locking: call with ipc->lock held and interrupts disabled */ -void ipc_complete_cmd(struct ipc *ipc) -{ - /* - * We have up to three contexts, attempting to complete IPC processing: - * the original IPC EDF task, the IDC EDF task on a secondary core, or - * an LL pipeline thread, running either on the primary or one of - * secondary cores. All these three contexts execute asynchronously. It - * is important to only signal the host that the IPC processing has - * completed after *all* tasks have completed. Therefore only the last - * context should do that. We accomplish this by setting IPC_TASK_* bits - * in ipc->task_mask for each used IPC context and by clearing them when - * each of those contexts completes. Only when the mask is 0 we can - * signal the host. - */ - if (ipc->task_mask) - return; - - ipc_platform_complete_cmd(ipc); -} - -bool ipc_enter_gdb; - -__attribute__((weak)) void ipc_platform_wait_ack(struct ipc *ipc) -{ - k_msleep(1); -} - -static void ipc_complete_task(void *data) -{ - struct ipc *ipc = data; - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - ipc->task_mask &= ~IPC_TASK_INLINE; - ipc_complete_cmd(ipc); - k_spin_unlock(&ipc->lock, key); -#if CONFIG_GDBSTUB - if (ipc_enter_gdb) { - ipc_enter_gdb = false; - ipc_platform_wait_ack(ipc); - gdb_init(); - } -#endif -} - -static enum task_state ipc_do_cmd(void *data) -{ - struct ipc *ipc = data; - -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - /* Increment performance counters */ - io_perf_monitor_update_data(ipc->io_perf_in_msg_count, 1); -#endif - - /* - * 32-bit writes are atomic and at the moment no IPC processing is - * taking place, so, no need for a lock. - */ - ipc->task_mask = IPC_TASK_INLINE; - - return ipc_platform_do_cmd(ipc); -} - -struct task_ops ipc_task_ops = { - .run = ipc_do_cmd, - .complete = ipc_complete_task, - .get_deadline = ipc_task_deadline, -}; diff --git a/src/kernel/ipc/CMakeLists.txt b/src/kernel/ipc/CMakeLists.txt index 0f9d63c0aee6..a3e7c61638c6 100644 --- a/src/kernel/ipc/CMakeLists.txt +++ b/src/kernel/ipc/CMakeLists.txt @@ -7,7 +7,7 @@ elseif (CONFIG_IPC_MAJOR_4) endif() # Files common to Zephyr and the host library -add_local_sources(sof notification_pool.c) +add_local_sources(sof ipc-common-kernel.c notification_pool.c) is_zephyr(zephyr) if(zephyr) diff --git a/src/kernel/ipc/ipc-common-kernel.c b/src/kernel/ipc/ipc-common-kernel.c new file mode 100644 index 000000000000..82cfb9d6049c --- /dev/null +++ b/src/kernel/ipc/ipc-common-kernel.c @@ -0,0 +1,895 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2016 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Keyon Jie + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if CONFIG_IPC_MAJOR_4 +#include +#endif +#include +#include +#include +#include + +#ifdef __ZEPHYR__ +#include +#include +#endif + +#ifdef CONFIG_SOF_USERSPACE_LL +#include +#endif + +#include +#include + +/* Kernel part of the common IPC code; the part that also runs in user mode is in + * src/ipc/ipc-common.c + */ + +LOG_MODULE_REGISTER(ipc, CONFIG_SOF_LOG_LEVEL); + +SOF_DEFINE_REG_UUID(ipc); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(ipc_tr, SOF_UUID(ipc_uuid), LOG_LEVEL_INFO); + +#ifdef CONFIG_SOF_USERSPACE_LL +K_APPMEM_PARTITION_DEFINE(ipc_context_part); + +K_APP_BMEM(ipc_context_part) static struct ipc ipc_context; + +struct ipc *ipc_get(void) +{ + return &ipc_context; +} +#endif + +int ipc_process_on_core(uint32_t core, bool blocking) +{ + struct ipc *ipc = ipc_get(); + struct idc_msg msg = { .header = IDC_MSG_IPC, .core = core, }; + int ret; + + /* check if requested core is enabled */ + if (!cpu_is_core_enabled(core)) { + tr_err(&ipc_tr, "core #%d is disabled", core); + return -EACCES; + } + +#if CONFIG_IPC_MAJOR_3 + /* The other core will write there its response */ + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + ((struct sof_ipc_cmd_hdr *)ipc->comp_data)->size); +#endif + + /* + * If the primary core is waiting for secondary cores to complete, it + * will also reply to the host + */ + if (!blocking) { + k_spinlock_key_t key; + + ipc->core = core; + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_SECONDARY_CORE; + k_spin_unlock(&ipc->lock, key); + } + + /* send IDC message */ + ret = idc_send_msg(&msg, blocking ? IDC_BLOCKING : IDC_NON_BLOCKING); + if (ret < 0) + return ret; + + /* reply written by other core */ + return 1; +} + +void ipc_send_queued_msg(void) +{ + struct ipc *ipc = ipc_get(); + struct ipc_msg *msg; + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + + if (ipc->pm_prepare_D3) + goto out; + + /* any messages to send ? */ + if (list_is_empty(&ipc->msg_list)) + goto out; + + msg = list_first_item(&ipc->msg_list, struct ipc_msg, + list); + + if (ipc_platform_send_msg(msg) == 0) { + /* Remove the message from the list if it has been successfully sent. */ + list_item_del(&msg->list); + /* Invoke a callback to notify that the message has been sent. */ + if (msg->callback) + msg->callback(msg); +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + /* Increment performance counters */ + io_perf_monitor_update_data(ipc->io_perf_out_msg_count, 1); +#endif + } +out: + k_spin_unlock(&ipc->lock, key); +} + +#ifdef __ZEPHYR__ +static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); + +/* + * Kernel-only: these must not live in struct ipc, which user threads can + * write with CONFIG_SOF_USERSPACE_LL. The work queue embeds the k_thread of + * a supervisor thread and the work item holds the handler it calls. + */ +static struct k_work_q ipc_send_wq; +static struct k_work_delayable ipc_send_work; + +struct k_work_q *ipc_send_wq_get(void) +{ + return &ipc_send_wq; +} +#endif + +static void schedule_ipc_worker(void) +{ + /* + * note: in XTOS builds, this is handled in + * task_main_primary_core() + */ +#ifdef __ZEPHYR__ + k_work_schedule_for_queue(&ipc_send_wq, &ipc_send_work, K_USEC(IPC_PERIOD_USEC)); +#endif +} + +__cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + int ret; + + assert_can_be_cold(); + + key = k_spin_lock(&ipc->lock); + + /* copy mailbox data to message if not already copied */ + if (data && msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE && + msg->tx_data != data) { + ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); + assert(!ret); + } + + ipc_platform_send_msg_direct(msg); + + k_spin_unlock(&ipc->lock, key); +} + +void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + int ret; + + key = k_spin_lock(&ipc->lock); + + /* copy mailbox data to message if not already copied */ + if (data && (msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE) && + msg->tx_data != data) { + ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); + assert(!ret); + } + + /* + * note: This function can be executed in LL or EDF context, from any core. + * In Zephyr builds, there is IPC queue that is always handled by the primary core, + * whereas submitting to the queue is allowed from any core. Therefore disable option + * of sending IPC immediately by any context/core to secure IPC registers/mailbox + * access. + */ +#ifndef __ZEPHYR__ + /* try to send critical notifications right away */ + if (high_priority) { + ret = ipc_platform_send_msg(msg); + if (!ret) { + k_spin_unlock(&ipc->lock, key); + return; + } + } +#endif + /* add to queue unless already there */ + if (list_is_empty(&msg->list)) { + if (high_priority) + list_item_prepend(&msg->list, &ipc->msg_list); + else + list_item_append(&msg->list, &ipc->msg_list); + } + + schedule_ipc_worker(); + + k_spin_unlock(&ipc->lock, key); +} +EXPORT_SYMBOL(ipc_msg_send); + +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + /* No caller check: DP modules send notifications, see mod_ipc_msg_init() */ + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + K_OOPS(K_SYSCALL_VERIFY(msg->tx_size <= SOF_IPC_MSG_MAX_SIZE)); + + K_OOPS(K_SYSCALL_VERIFY(list_is_empty(&msg->list))); + + if (msg->tx_size > 0) { + K_OOPS(K_SYSCALL_VERIFY(msg->tx_data != NULL)); + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg->tx_data, msg->tx_size)); + } + + /* If data != NULL and tx_size > 0, verify the data buffer */ + if (data && msg->tx_size > 0) + K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); + + z_impl_ipc_msg_send(msg, data, high_priority); +} +#include +#endif /* CONFIG_USERSPACE */ + +void z_impl_ipc_msg_list_remove(struct ipc_msg *msg) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + list_item_del(&msg->list); + k_spin_unlock(&ipc->lock, key); +} + +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) +{ + /* No caller check: called by mod_ipc_msg_free() in DP module threads */ + struct ipc *ipc = ipc_get(); + struct list_item *mlist, *_mlist; + bool found = false; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + /* + * special case: empty list was passed. we can't trust where + * list->prev points to, so do not pass to + * z_impl_ipc_msg_list_remove(), but handle here + */ + if (list_is_empty(&msg->list)) + return; + + list_for_item_safe(mlist, _mlist, &ipc->msg_list) { + if (mlist == &msg->list) { + found = true; + break; + } + } + K_OOPS(K_SYSCALL_VERIFY(found)); + z_impl_ipc_msg_list_remove(msg); +} +#include +#endif /* CONFIG_USERSPACE */ + +#ifdef __ZEPHYR__ +static void ipc_work_handler(struct k_work *work) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + ipc_send_queued_msg(); + + key = k_spin_lock(&ipc->lock); + + if (!list_is_empty(&ipc->msg_list) && !ipc->pm_prepare_D3) + schedule_ipc_worker(); + + k_spin_unlock(&ipc->lock, key); +} +#endif + +void ipc_schedule_process(struct ipc *ipc) +{ +#if CONFIG_TWB_IPC_TASK + schedule_task(ipc->ipc_task, 0, IPC_PERIOD_USEC); +#else + schedule_task(&ipc->ipc_task, 0, IPC_PERIOD_USEC); +#endif +} + +#ifdef CONFIG_SOF_USERSPACE_LL +/* Generic user-space IPC handling thread infrastructure. Protocol-specific + * command interpretation lives in ipc_user_thread_dispatch(), implemented by + * the active IPC major (see src/ipc/ipc4/). + */ + +BUILD_ASSERT(IPC_USER_XCORE_PAYLOAD_SIZE >= IDC_MAX_PAYLOAD_SIZE, + "cross-core request payload smaller than IDC payload"); +#if CONFIG_IPC_MAJOR_4 +BUILD_ASSERT(sizeof(((struct ipc_user_xcore *)0)->ipc4_bind_data) == + sizeof(struct ipc4_module_bind_unbind), + "cross-core request bind data size mismatch"); +#endif + +SOF_DEFINE_REG_UUID(sec_core_init); + +static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); + +/* + * Kernel-only copies of the IPC user thread state. struct ipc_user and the + * ipc->ipc_user_pdata pointer are both writable by user threads, so kernel + * code must use these instead of reading kernel object pointers from there. + */ +static struct ipc_user *ipc_user_kptr; +static struct k_sem *ipc_user_ksem; +static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; +static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; +static struct k_thread *ipc_user_kaudio_thread[CONFIG_CORE_COUNT]; +static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; + +/* Cross-core request slot semaphores, static so they're never freed */ +static struct k_sem ipc_user_xcore_lock[CONFIG_CORE_COUNT]; +static struct k_sem ipc_user_xcore_done[CONFIG_CORE_COUNT]; + +/** + * @brief Forward an IPC command to the user-space thread. + * + * Called from kernel context (IPC EDF task) to forward the message words + * to the user-space thread for processing. Sets IPC_TASK_IN_THREAD in + * task_mask so the host is not signaled until the user thread completes. + * Blocks until the user thread finishes processing and returns the result. + * + * @param primary Primary message word + * @param extension Extension message word + * @return Result from user thread processing + */ +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc_user_kptr; + k_spinlock_key_t key; + int ret; + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + LOG_DBG("IPC: forward cmd %08x", primary); + + /* Copy message words — original buffer may be reused */ + pdata->ipc_msg_pri = primary; + pdata->ipc_msg_ext = extension; + pdata->ipc = ipc; + + /* Prevent host completion until user thread finishes */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_IN_THREAD; + k_spin_unlock(&ipc->lock, key); + + /* + * Wake the user thread. Post, not set: a cross-core request + * (IPC_USER_EVENT_XCORE) may be pending on the same event. + */ + k_event_post(ipc_user_kevent[core], IPC_USER_EVENT_CMD); + + /* Wait for user thread to complete */ + ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); + if (ret) { + LOG_ERR("IPC user: sem error %d\n", ret); + /* fall through to complete the cmd */ + } + + /* Clear the task mask bit and check for completion */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_IN_THREAD; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); + + return !ret ? pdata->result : ret; +} + +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) +{ + char thread_name[] = "ll_user0"; + struct k_thread *thread; + struct k_event *event; + int ret; + + assert_can_be_cold(); + + /* + * Build the objects in locals and publish them to struct ipc_user only + * at the end. When a secondary core is brought up, user threads on + * other cores are already running and can overwrite struct ipc_user, + * so its fields must not be read back here. + */ + event = k_object_alloc(K_OBJ_EVENT); + if (!event) { + LOG_ERR("user IPC event alloc failed"); + return -ENOMEM; + } + k_event_init(event); + + thread = k_object_alloc(K_OBJ_THREAD); + if (!thread) { + LOG_ERR("user IPC thread alloc failed"); + ret = -ENOMEM; + goto e_event; + } + + k_thread_create(thread, ipc_user_stack[core], + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, + ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, + -1, K_USER, K_FOREVER); + + k_thread_cpu_pin(thread, core); + thread_name[sizeof(thread_name) - 2] = '0' + core; + k_thread_name_set(thread, thread_name); + + /* + * Each userspace IPC thread must be able to wait on its private event + * and signal completion on the primary core semaphore + */ + k_thread_access_grant(thread, ipc_user_ksem, event); + user_grant_dai_access_all(thread); + user_grant_dma_access_all(thread); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); + user_ll_grant_access(thread, core); + pipeline_posn_grant_access(thread); + + ipc_user_kevent[core] = event; + ipc_user_kthread[core] = thread; + sof_sysuser_thread_set(SOF_SYSUSER_IPC, core, thread); + + /* for ipc_user_thread_fn() and ipc_thread_user() in user context */ + ipc_user->event[core] = event; + ipc_user->thread[core] = thread; + + return 0; + +e_event: + k_object_free(event); + + return ret; +} + +/* + * Grant the threads of the newly initialized core and of the other already + * initialized cores access to each other's cross-core request slots and LL + * scheduler locks. Uses only the kernel-only copies of the thread state. + */ +__cold static void ipc_user_xcore_grant(unsigned int core) +{ + struct k_thread *new_threads[] = { + ipc_user_kthread[core], ipc_user_kaudio_thread[core], + }; + unsigned int c, i; + + assert_can_be_cold(); + + for (c = 0; c < CONFIG_CORE_COUNT; c++) { + struct k_thread *threads[] = { + ipc_user_kthread[c], ipc_user_kaudio_thread[c], + }; + + /* + * Grant the slot semaphores of all cores. Note: braces needed, + * k_thread_access_grant() expands to multiple statements. + */ + for (i = 0; i < ARRAY_SIZE(new_threads); i++) { + if (new_threads[i]) { + k_thread_access_grant(new_threads[i], &ipc_user_xcore_lock[c], + &ipc_user_xcore_done[c]); + } + } + + if (!ipc_user_kthread[c]) + continue; + + for (i = 0; i < ARRAY_SIZE(threads); i++) { + if (!threads[i]) + continue; + k_thread_access_grant(threads[i], ipc_user_kevent[core]); + user_ll_grant_access(threads[i], core); + } + + for (i = 0; i < ARRAY_SIZE(new_threads); i++) { + if (!new_threads[i]) + continue; + k_thread_access_grant(new_threads[i], ipc_user_kevent[c]); + user_ll_grant_access(new_threads[i], c); + } + } +} + +__cold int ipc_user_init_secondary(unsigned int core) +{ + struct ipc_user *ipc_user = ipc_user_kptr; + struct k_thread *audio_thread; + int ret; + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + ret = ipc_user_init_thread(ipc_user, core); + + if (ret < 0) + return ret; + + assert_can_be_cold(); + + k_thread_start(ipc_user_kthread[core]); + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("user LL task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, core, 0); + + audio_thread = scheduler_init_context(task); + if (!audio_thread) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + k_thread_access_grant(ipc_user_kthread[core], audio_thread); + ipc_user_kaudio_thread[core] = audio_thread; + ipc_user->audio_thread[core] = audio_thread; + ipc_user_xcore_grant(core); + + return 0; +} + +struct k_thread *ipc_thread_user(unsigned int core) +{ + if (core >= CONFIG_CORE_COUNT) + return NULL; + + /* user threads can not read the kernel-only copy */ + if (k_is_user_context()) + return ipc_get()->ipc_user_pdata->thread[core]; + + return ipc_user_kthread[core]; +} + +struct ipc_user *ipc_user_get(void) +{ + return ipc_user_kptr; +} + +void ipc_user_wait_core_init(unsigned int core) +{ + if (core >= CONFIG_CORE_COUNT) + return; + + if (ipc_user_kinit_needed[core]) { + /* wait for IPC thread (ipc_user_thread_fn()) */ + k_sem_take(ipc_user_ksem, K_FOREVER); + ipc_user_kinit_needed[core] = false; + } +} + +/* Primary core only */ +__cold static void ipc_user_init(void) +{ + struct ipc *ipc = ipc_get(); + struct k_thread *audio_thread; + struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc_user), 0); + int ret, core; + + if (!ipc_user) { + LOG_ERR("user IPC pdata alloc failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + + /* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */ + memset(ipc_user, 0, sizeof(*ipc_user)); + + assert_can_be_cold(); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + if (core != PLATFORM_PRIMARY_CORE_ID) + ipc_user_kinit_needed[core] = true; + } + + ipc_user_ksem = k_object_alloc(K_OBJ_SEM); + if (!ipc_user_ksem) { + LOG_ERR("user IPC sem alloc failed"); + k_panic(); + } + ipc_user_kptr = ipc_user; + ipc_user->sem = ipc_user_ksem; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); + if (ret < 0) + LOG_WRN("ipc context partition add failed: %d", ret); + + /* + * Grant user-space read access to the .coldrodata section in IMR. The + * prepare path walks component code that may reference __cold_rodata + * data. + * + * .cold text is deliberately NOT added as a partition: Zephyr already + * maps it executable in the shared ring. + */ +#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM + extern char __coldrodata_start[]; + extern char _imr_end[]; + + if (&_imr_end[0] > &__coldrodata_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__coldrodata_start, + .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", + cold_part.size, cold_part.start, ret); + } +#endif /* CONFIG_COLD_STORE_EXECUTE_DRAM */ + + k_sem_init(ipc_user_ksem, 0, 1); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + struct ipc_user_xcore *xc = &ipc_user->xcore[core]; + + xc->lock = &ipc_user_xcore_lock[core]; + xc->done = &ipc_user_xcore_done[core]; + k_sem_init(xc->lock, 1, 1); + k_sem_init(xc->done, 0, 1); + } + + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); + if (ret < 0) { + LOG_ERR("user IPC thread initialization failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + + ret = user_access_to_mailbox(zephyr_ll_mem_domain(), + ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); + if (ret < 0) { + LOG_ERR("ipc user: mailbox access grant failed: %d", ret); + sof_panic(SOF_IPC_PANIC_IPC); + } + + /* Store references in ipc struct so kernel handler can forward commands */ + ipc->ipc_user_pdata = ipc_user; + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); + audio_thread = scheduler_init_context(task); + if (!audio_thread) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + /* Grant ipc_user thread permission on the audio thread object. + * Needed so user-space dai_common_new() can call + * k_thread_access_grant(audio_thread, dai_mutex) from user context. + */ + k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); + ipc_user_kaudio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; + ipc_user_xcore_grant(PLATFORM_PRIMARY_CORE_ID); + + k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); + + /* Wait for user thread startup — consumes the initial k_sem_give from thread */ + k_sem_take(ipc_user_ksem, K_FOREVER); +} +#else /* CONFIG_SOF_USERSPACE_LL */ +static void ipc_user_init(void) +{ +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc_init(struct sof *sof) +{ + struct k_heap *heap; + struct ipc *ipc; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "entry"); + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); + + ipc = ipc_get(); + memset(ipc, 0, sizeof(*ipc)); + ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc->ll_alloc), 0); + if (!ipc->ll_alloc) { + tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); + sof_panic(SOF_IPC_PANIC_IPC); + } + ipc->ll_alloc->heap = heap; + ipc->ll_alloc->vreg = NULL; +#else + heap = NULL; + + /* init ipc data */ + ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc)); + if (!ipc) { + tr_err(&ipc_tr, "Unable to allocate IPC data"); + return -ENOMEM; + } + sof->ipc = ipc; +#endif /* CONFIG_SOF_USERSPACE_LL */ + + ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + SOF_IPC_MSG_MAX_SIZE, 0); + if (!ipc->comp_data) { + tr_err(&ipc_tr, "Unable to allocate IPC component data"); + sof_panic(SOF_IPC_PANIC_IPC); + } + memset(ipc->comp_data, 0, SOF_IPC_MSG_MAX_SIZE); + + k_spinlock_init(&ipc->lock); + list_init(&ipc->msg_list); + list_init(&ipc->comp_list); + +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + struct io_perf_data_item init_data = {IO_PERF_IPC_ID, + cpu_get_id(), + IO_PERF_INPUT_DIRECTION, + IO_PERF_POWERED_UP_ENABLED, + IO_PERF_D0IX_POWER_MODE, + 0, 0, 0 }; + io_perf_monitor_init_data(&ipc->io_perf_in_msg_count, &init_data); + init_data.direction = IO_PERF_OUTPUT_DIRECTION; + io_perf_monitor_init_data(&ipc->io_perf_out_msg_count, &init_data); +#endif + +#if CONFIG_SOF_BOOT_TEST_STANDALONE + LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); + return 0; +#endif + +#ifdef __ZEPHYR__ + k_tid_t thread; + + k_work_queue_init(&ipc_send_wq); + k_work_queue_start(&ipc_send_wq, ipc_send_wq_stack, + K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); + + thread = k_work_queue_thread_get(&ipc_send_wq); + + k_thread_suspend(thread); + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); +#endif + k_thread_name_set(thread, "ipc_send_wq"); + + k_thread_resume(thread); + + k_work_init_delayable(&ipc_send_work, ipc_work_handler); +#endif /* __ZEPHYR__ */ + + ipc_user_init(); + + return platform_ipc_init(ipc); +} + +/* Locking: call with ipc->lock held and interrupts disabled */ +void ipc_complete_cmd(struct ipc *ipc) +{ + /* + * We have up to three contexts, attempting to complete IPC processing: + * the original IPC EDF task, the IDC EDF task on a secondary core, or + * an LL pipeline thread, running either on the primary or one of + * secondary cores. All these three contexts execute asynchronously. It + * is important to only signal the host that the IPC processing has + * completed after *all* tasks have completed. Therefore only the last + * context should do that. We accomplish this by setting IPC_TASK_* bits + * in ipc->task_mask for each used IPC context and by clearing them when + * each of those contexts completes. Only when the mask is 0 we can + * signal the host. + */ + if (ipc->task_mask) + return; + + ipc_platform_complete_cmd(ipc); +} + +bool ipc_enter_gdb; + +__attribute__((weak)) void ipc_platform_wait_ack(struct ipc *ipc) +{ + k_msleep(1); +} + +static void ipc_complete_task(void *data) +{ + struct ipc *ipc = data; + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_INLINE; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); +#if CONFIG_GDBSTUB + if (ipc_enter_gdb) { + ipc_enter_gdb = false; + ipc_platform_wait_ack(ipc); + gdb_init(); + } +#endif +} + +static enum task_state ipc_do_cmd(void *data) +{ + struct ipc *ipc = data; + +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + /* Increment performance counters */ + io_perf_monitor_update_data(ipc->io_perf_in_msg_count, 1); +#endif + + /* + * 32-bit writes are atomic and at the moment no IPC processing is + * taking place, so, no need for a lock. + */ + ipc->task_mask = IPC_TASK_INLINE; + + return ipc_platform_do_cmd(ipc); +} + +struct task_ops ipc_task_ops = { + .run = ipc_do_cmd, + .complete = ipc_complete_task, + .get_deadline = ipc_task_deadline, +}; diff --git a/test/cmocka/src/audio/buffer/CMakeLists.txt b/test/cmocka/src/audio/buffer/CMakeLists.txt index cb60f07aa22e..805aaa2da784 100644 --- a/test/cmocka/src/audio/buffer/CMakeLists.txt +++ b/test/cmocka/src/audio/buffer/CMakeLists.txt @@ -14,6 +14,7 @@ cmocka_test(buffer_copy ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -38,6 +39,7 @@ cmocka_test(buffer_new ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -62,6 +64,7 @@ cmocka_test(buffer_wrap ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -86,6 +89,7 @@ cmocka_test(buffer_write ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/cmocka/src/audio/component/CMakeLists.txt b/test/cmocka/src/audio/component/CMakeLists.txt index b82756073b25..50389cfe4535 100644 --- a/test/cmocka/src/audio/component/CMakeLists.txt +++ b/test/cmocka/src/audio/component/CMakeLists.txt @@ -8,6 +8,7 @@ cmocka_test(comp_set_state ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c diff --git a/test/cmocka/src/audio/dcblock/CMakeLists.txt b/test/cmocka/src/audio/dcblock/CMakeLists.txt index 9c7f58e8feed..0e5754b022d5 100644 --- a/test/cmocka/src/audio/dcblock/CMakeLists.txt +++ b/test/cmocka/src/audio/dcblock/CMakeLists.txt @@ -41,6 +41,7 @@ add_library(audio_for_dcblock STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/eq_fir/CMakeLists.txt b/test/cmocka/src/audio/eq_fir/CMakeLists.txt index 0bb81cc57b0a..485d1996dd8d 100644 --- a/test/cmocka/src/audio/eq_fir/CMakeLists.txt +++ b/test/cmocka/src/audio/eq_fir/CMakeLists.txt @@ -37,6 +37,7 @@ add_library(audio_for_eq_fir STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/eq_iir/CMakeLists.txt b/test/cmocka/src/audio/eq_iir/CMakeLists.txt index 5087e77b4b83..53a550b6b6cf 100644 --- a/test/cmocka/src/audio/eq_iir/CMakeLists.txt +++ b/test/cmocka/src/audio/eq_iir/CMakeLists.txt @@ -40,6 +40,7 @@ add_library(audio_for_eq_iir STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/mixer/CMakeLists.txt b/test/cmocka/src/audio/mixer/CMakeLists.txt index a6c1cdb1a78f..b0b086816e3b 100644 --- a/test/cmocka/src/audio/mixer/CMakeLists.txt +++ b/test/cmocka/src/audio/mixer/CMakeLists.txt @@ -9,6 +9,7 @@ cmocka_test(mixer ${PROJECT_SOURCE_DIR}/src/audio/mixer/mixer_hifi3.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/src/audio/module_adapter/module_adapter.c diff --git a/test/cmocka/src/audio/pcm_converter/CMakeLists.txt b/test/cmocka/src/audio/pcm_converter/CMakeLists.txt index 4dca80136e68..ee72311e004b 100644 --- a/test/cmocka/src/audio/pcm_converter/CMakeLists.txt +++ b/test/cmocka/src/audio/pcm_converter/CMakeLists.txt @@ -19,6 +19,7 @@ if(CONFIG_FORMAT_FLOAT) ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/cmocka/src/audio/pipeline/CMakeLists.txt b/test/cmocka/src/audio/pipeline/CMakeLists.txt index 628398576a43..27d27757d6ae 100644 --- a/test/cmocka/src/audio/pipeline/CMakeLists.txt +++ b/test/cmocka/src/audio/pipeline/CMakeLists.txt @@ -6,6 +6,7 @@ cmocka_test(pipeline_new ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c @@ -31,6 +32,7 @@ cmocka_test(pipeline_connect_upstream ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c @@ -56,6 +58,7 @@ cmocka_test(pipeline_free ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c diff --git a/test/cmocka/src/audio/selector/CMakeLists.txt b/test/cmocka/src/audio/selector/CMakeLists.txt index 8ea8d969ae7c..eb8f742cc5f5 100644 --- a/test/cmocka/src/audio/selector/CMakeLists.txt +++ b/test/cmocka/src/audio/selector/CMakeLists.txt @@ -26,6 +26,7 @@ add_library(audio_for_selector STATIC ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c diff --git a/test/cmocka/src/audio/volume/CMakeLists.txt b/test/cmocka/src/audio/volume/CMakeLists.txt index d5e1a3ecb757..7a6dc7b2c854 100644 --- a/test/cmocka/src/audio/volume/CMakeLists.txt +++ b/test/cmocka/src/audio/volume/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(audio_for_volume STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/math/fft/CMakeLists.txt b/test/cmocka/src/math/fft/CMakeLists.txt index f8a0510f7027..79234b7b12f9 100644 --- a/test/cmocka/src/math/fft/CMakeLists.txt +++ b/test/cmocka/src/math/fft/CMakeLists.txt @@ -18,6 +18,7 @@ cmocka_test(fft ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/ztest/unit/audio/drc/CMakeLists.txt b/test/ztest/unit/audio/drc/CMakeLists.txt index 2faef72b825c..39691eefa371 100644 --- a/test/ztest/unit/audio/drc/CMakeLists.txt +++ b/test/ztest/unit/audio/drc/CMakeLists.txt @@ -61,6 +61,7 @@ target_sources(app PRIVATE ${SOF_ROOT}/src/module/audio/sink_api.c ${SOF_ROOT}/src/kernel/ipc/ipc3/helper.c ${SOF_ROOT}/src/ipc/ipc-common.c + ${SOF_ROOT}/src/kernel/ipc/ipc-common-kernel.c ${SOF_ROOT}/src/ipc/ipc-helper.c ${SOF_ROOT}/src/lib/objpool.c ${SOF_ROOT}/src/audio/pipeline/pipeline-graph.c diff --git a/test/ztest/unit/math/fft/CMakeLists.txt b/test/ztest/unit/math/fft/CMakeLists.txt index 920f41bdb642..0c7a8f13783f 100644 --- a/test/ztest/unit/math/fft/CMakeLists.txt +++ b/test/ztest/unit/math/fft/CMakeLists.txt @@ -65,6 +65,7 @@ target_sources(app PRIVATE ${SOF_ROOT}/src/module/audio/sink_api.c ${SOF_ROOT}/src/kernel/ipc/ipc3/helper.c ${SOF_ROOT}/src/ipc/ipc-common.c + ${SOF_ROOT}/src/kernel/ipc/ipc-common-kernel.c ${SOF_ROOT}/src/ipc/ipc-helper.c ${SOF_ROOT}/src/audio/pipeline/pipeline-graph.c ${SOF_ROOT}/src/audio/pipeline/pipeline-params.c From 3e430012e1be681a4baa0158ed6fff801fa415b8 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 22:22:35 +0000 Subject: [PATCH 2/4] ipc4: split the kernel part of handler-user.c into src/kernel/ipc/ipc4 handler-user.c mixes the IPC4 global and module message routers, which run in the kernel IPC thread, with the handlers that also run in the user-space IPC thread. Move the kernel part to src/kernel/ipc/ipc4, in three files: - handler-user-kernel.c, common to all builds: the global and module message routers and the reply message they fill in, the CHAIN_DMA, IPC gateway and GDB commands, the module init that runs in the kernel (DP modules and builds without user-space LL), and the base firmware LARGE_CONFIG get and set, which use IDC and the IMR manifest; - handler-ll-user.c, built with the LL scheduler in user space (CONFIG_SOF_USERSPACE_LL): the routing of the pipeline and module messages to the user-space IPC thread, with the target core lookup, the multicore SET_PIPELINE_STATE sequencing and the MOD_INIT_INSTANCE driver lookup and module loading; - handler-ll-kernel.c, built with the LL scheduler in kernel space: the routing of the same messages directly to their handlers. Instead of switching on CONFIG_SOF_USERSPACE_LL per message, the routers call the ipc4_route_*() functions of the two routing files, which are declared in the private handler-route.h. A comment at the top of handler-user-kernel.c describes the split. handler-user.c keeps the pipeline state handling, the module config and large config processing and the user-space IPC thread dispatch. The pipeline ID lookup and the vendor config and kcontrol helpers, which both parts use, are no longer static and are declared in ipc4/handler.h. handler-user.c no longer needs the IPC gateway copier header. The handlers are unchanged. The routers now call the ipc4_route_*() functions, and through them ipc4_init_module_instance() and the base firmware LARGE_CONFIG handlers, instead of inlining that code. As before, the compiler can no longer inline or specialise the shared helpers in the callers on the other side of the split: their callers pass the full argument list instead of calling GCC's constant-propagated clones, and ipc4_set_pipeline_state() only partially inlines ipc4_pipeline_id_get() in builds without user-space LL. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/ipc4/handler.h | 70 ++ src/ipc/ipc4/README.md | 7 +- src/ipc/ipc4/handler-user.c | 844 +--------------------- src/kernel/ipc/ipc4/CMakeLists.txt | 9 + src/kernel/ipc/ipc4/handler-ll-kernel.c | 177 +++++ src/kernel/ipc/ipc4/handler-ll-user.c | 331 +++++++++ src/kernel/ipc/ipc4/handler-route.h | 53 ++ src/kernel/ipc/ipc4/handler-user-kernel.c | 509 +++++++++++++ 8 files changed, 1170 insertions(+), 830 deletions(-) create mode 100644 src/kernel/ipc/ipc4/handler-ll-kernel.c create mode 100644 src/kernel/ipc/ipc4/handler-ll-user.c create mode 100644 src/kernel/ipc/ipc4/handler-route.h create mode 100644 src/kernel/ipc/ipc4/handler-user-kernel.c diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index 3335c108fa29..5fc9f749e7fb 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -12,7 +12,10 @@ #ifndef __SOF_IPC4_HANDLER_H__ #define __SOF_IPC4_HANDLER_H__ +struct comp_dev; +struct comp_driver; struct ipc4_message_request; +struct ipc4_pipeline_set_state; /** * \brief Processes an IPC4 userspace module message. @@ -69,6 +72,73 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_ */ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4); +/** + * \brief Get the first pipeline ID of a SET_PIPELINE_STATE message. + * \ingroup ipc4_envelope_api + * @param[in] ipc4 IPC4 message request structure. + * @param[in] state Copy of the SET_PIPELINE_STATE message header. + * @param[out] ppl_id Receives the pipeline ID array of a multi-pipeline + * message, NULL otherwise (may be NULL). + * @param[out] ppl_count Receives the number of pipelines (may be NULL). + * @return The first pipeline ID, or a negative error code. + */ +int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, + struct ipc4_pipeline_set_state *state, + const uint32_t **ppl_id, unsigned int *ppl_count); + +/** + * \brief Copy the kcontrol payload header of a LARGE_CONFIG_GET from the hostbox. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] param_id Large config parameter ID. + * @param[out] data_out Reply data buffer. + * @param[in] data_size Size of the payload in the hostbox. + */ +void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, + char *data_out, uint32_t data_size); + +/** + * \brief Process a vendor config LARGE_CONFIG_GET. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] drv Component driver. + * @param[in] init_block True for the first block. + * @param[in] final_block True for the last block. + * @param[in,out] data_off_size Payload size in, reply data size out. + * @param[out] data_out Reply data buffer. + * @param[in] data_in Request payload. + * @return IPC4 status code (IPC4_SUCCESS on success). + */ +int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + bool init_block, + bool final_block, + uint32_t *data_off_size, + char *data_out, + const char *data_in); + +/** + * \brief Process a vendor config LARGE_CONFIG_SET. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] drv Component driver. + * @param[in] module_id Module ID, for error messages. + * @param[in] instance_id Instance ID, for error messages. + * @param[in] init_block True for the first block. + * @param[in] final_block True for the last block. + * @param[in] data_off_size Payload size. + * @param[in] data Request payload. + * @return IPC4 status code (IPC4_SUCCESS on success). + */ +int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + uint32_t module_id, + uint32_t instance_id, + bool init_block, + bool final_block, + uint32_t data_off_size, + const char *data); + /** * \brief Complete an IPC compound message transaction and signal waiting threads. * \ingroup ipc4_envelope_api diff --git a/src/ipc/ipc4/README.md b/src/ipc/ipc4/README.md index 237a7c1ecb36..70ee4d606af2 100644 --- a/src/ipc/ipc4/README.md +++ b/src/ipc/ipc4/README.md @@ -2,9 +2,10 @@ > **Runs in:** Boundary / bridge — `handler-user.c` and `notification-user.c` run in the user > thread that services modules. Their kernel counterparts, `handler-kernel.c` (the kernel -> dispatcher) and `notification.c`, are in `src/kernel/ipc/ipc4`, together with the -> `z_vrfy_`/`z_impl_` pairs for the compound-message, reply and notification syscalls. See -> [User/kernel split](../../include/sof/userspace/README.md). +> dispatcher), `handler-user-kernel.c` (the global and module message routers, which forward +> commands to the user thread) and `notification.c`, are in `src/kernel/ipc/ipc4`, together +> with the `z_vrfy_`/`z_impl_` pairs for the compound-message, reply and notification +> syscalls. See [User/kernel split](../../include/sof/userspace/README.md). This directory holds the handlers and topology parsing logic for Inter-Processor Communication Version 4. IPC4 introduces a significantly denser, compound-command structure heavily based around the concept of "pipelines" and dynamic "modules" rather than static DSP stream roles. diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 167639253152..58df68018242 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -44,8 +44,6 @@ #include #include -#include "../audio/copier/ipcgtw_copier.h" - /* Command format errors during fuzzing are reported for virtually all * commands, and the resulting flood of logging becomes a severe * performance penalty (i.e. we get a lot less fuzzing done per CPU @@ -57,11 +55,11 @@ #define ipc_cmd_err(...) tr_err(__VA_ARGS__) #endif -LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); +/* The IPC4 routers, which run in kernel mode, are in + * src/kernel/ipc/ipc4/handler-user-kernel.c + */ -/* Userspace message context, copied in/out by kernel IPC thread. */ -/* fw sends a fw ipc message to send the status of the last host ipc message */ -static struct ipc_msg *msg_reply; +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); #if CONFIG_LIBRARY @@ -90,54 +88,6 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( /* * Global IPC Operations. */ -#ifdef CONFIG_SOF_USERSPACE_LL -/* - * Determine the target core for an IPC4 module message. - * Falls back to current core when no component is bound yet. - */ -static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) -{ - /* - * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, - * struct ipc4_module_delete_instance - */ - struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; - uint32_t module_id = config->primary.r.module_id; - - if (module_id) { - uint32_t instance_id = config->primary.r.instance_id; - struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); - - if (dev) - return dev->ipc_config.core; - } - - return cpu_get_id(); -} -#else /* CONFIG_SOF_USERSPACE_LL */ -__cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) -{ - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); -} - -__cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) -{ - struct ipc4_pipeline_delete *pipe; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - pipe = (struct ipc4_pipeline_delete *)ipc4; - tr_dbg(&ipc_tr, "ipc4 delete pipeline %x:", (uint32_t)pipe->primary.r.instance_id); - - return ipc_pipeline_free(ipc, pipe->primary.r.instance_id); -} -#endif /* CONFIG_SOF_USERSPACE_LL */ - static int ipc4_pcm_params(struct ipc_comp_dev *pcm_dev) { int err, reset_err; @@ -450,9 +400,9 @@ __cold const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data_wrapper return ipc4_get_pipeline_data(); } -static int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, - struct ipc4_pipeline_set_state *state, - const uint32_t **ppl_id, unsigned int *ppl_count) +int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, + struct ipc4_pipeline_set_state *state, + const uint32_t **ppl_id, unsigned int *ppl_count) { if (!state->extension.r.multi_ppl) { if (ppl_count) @@ -648,364 +598,8 @@ static int ipc4_set_pipeline_state_phase(struct ipc4_message_request *ipc4, uint return IPC4_SUCCESS; } - -/* - * Handle a multi-pipeline SET_PIPELINE_STATE, where the pipelines are on - * different cores. The user-space IPC thread can't use IDC to reach the - * other cores, so the kernel runs the prepare phase on all pipelines and - * then the trigger phase on all pipelines, forwarding each step to the - * user-space IPC thread of the core, owning the pipeline. - */ -__cold static int ipc4_user_set_pipeline_state_multicore(struct ipc4_message_request *ipc4, - const uint32_t *ppl_id, - unsigned int ppl_count) -{ - static const uint32_t phases[] = { - IDC_PPL_STATE_PHASE_PREPARE, - IDC_PPL_STATE_PHASE_TRIGGER, - }; - struct ipc *ipc = ipc_get(); - struct ipc_user *pdata = ipc->ipc_user_pdata; - struct ipc4_pipeline_set_state state; - struct ipc_comp_dev *ppl_icd; - unsigned int i, j; - int ret; - - assert_can_be_cold(); - - for (j = 0; j < ARRAY_SIZE(phases); j++) { - for (i = 0; i < ppl_count; i++) { - ppl_icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, ppl_id[i], - IPC_COMP_ALL); - if (!ppl_icd) { - ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); - return IPC4_INVALID_RESOURCE_ID; - } - - /* Forward as a single-pipeline request */ - state.primary.dat = ipc4->primary.dat; - state.extension.dat = ipc4->extension.dat; - state.primary.r.ppl_id = ppl_id[i]; - state.extension.r.multi_ppl = 0; - - pdata->ppl_phase = phases[j]; - ret = ipc_user_forward_cmd(state.primary.dat, state.extension.dat, - ppl_icd->core); - pdata->ppl_phase = 0; - if (ret != 0) - return ret; - } - } - - return IPC4_SUCCESS; -} #endif /* CONFIG_SOF_USERSPACE_LL */ -__cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) -{ - assert_can_be_cold(); - -#if CONFIG_COMP_CHAIN_DMA - struct ipc_comp_dev *cdma_comp; - struct ipc *ipc = ipc_get(); - const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4; - int comp_id; - int ret; - - comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); - cdma_comp = ipc_get_comp_by_id(ipc, comp_id); - - if (!cdma_comp) { - /* - * Nothing to do when the chainDMA is not allocated and asked to - * be freed - */ - if (!cdma->primary.r.allocate && !cdma->primary.r.enable) - return IPC4_SUCCESS; - - ret = ipc4_chain_manager_create(cdma); - if (ret < 0) - return IPC4_FAILURE; - - cdma_comp = ipc_get_comp_by_id(ipc, comp_id); - if (!cdma_comp) { - return IPC4_FAILURE; - } - - ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); - if (ret < 0) { - comp_free(cdma_comp->cd); - list_item_del(&cdma_comp->list); - sof_heap_free(sof_sys_user_heap_get(), cdma_comp); - return IPC4_FAILURE; - } - - return IPC4_SUCCESS; - } - - ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); - if (ret < 0) - return IPC4_INVALID_CHAIN_STATE_TRANSITION; - - return IPC4_SUCCESS; -#else /* CONFIG_COMP_CHAIN_DMA */ - return IPC4_UNAVAILABLE; -#endif /* CONFIG_COMP_CHAIN_DMA */ -} - -__cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4) -{ - assert_can_be_cold(); - -#if CONFIG_IPC4_GATEWAY - struct ipc *ipc = ipc_get(); - uint32_t reply_size = 0; - int err; - - err = copier_ipcgtw_process((const struct ipc4_ipcgtw_cmd *)ipc4, ipc->comp_data, - &reply_size); - /* reply size is returned in header extension dword */ - msg_reply->extension = reply_size; - - if (reply_size > 0) { - msg_reply->tx_data = ipc->comp_data; - msg_reply->tx_size = reply_size; - } - - return err < 0 ? IPC4_FAILURE : IPC4_SUCCESS; -#else /* CONFIG_IPC4_GATEWAY */ - ipc_cmd_err(&ipc_tr, "CONFIG_IPC4_GATEWAY is disabled"); - return IPC4_UNAVAILABLE; -#endif /* CONFIG_IPC4_GATEWAY */ -} - -static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) -{ -#if CONFIG_GDBSTUB - ipc_enter_gdb = true; - return IPC4_SUCCESS; -#else - return IPC4_UNAVAILABLE; -#endif -} - -int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, - struct ipc_msg *reply) -{ -#ifdef CONFIG_SOF_USERSPACE_LL - struct ipc *ipc = ipc_get(); -#endif - uint32_t type; - int ret; - - type = ipc4->primary.r.type; - msg_reply = reply; - - switch (type) { - case SOF_IPC4_GLB_BOOT_CONFIG: - case SOF_IPC4_GLB_ROM_CONTROL: - case SOF_IPC4_GLB_PERF_MEASUREMENTS_CMD: - case SOF_IPC4_GLB_LOAD_MULTIPLE_MODULES: - case SOF_IPC4_GLB_UNLOAD_MULTIPLE_MODULES: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_CHAIN_DMA: - ret = ipc4_process_chain_dma(ipc4); - break; - - /* pipeline settings */ - case SOF_IPC4_GLB_CREATE_PIPELINE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_pipeline_create *create = - (const struct ipc4_pipeline_create *)ipc4; - - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - create->extension.r.core_id); - } -#else - ret = ipc4_new_pipeline(ipc4); -#endif - break; - case SOF_IPC4_GLB_DELETE_PIPELINE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; - struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, - del->primary.r.instance_id, - IPC_COMP_ALL); - - if (!ppl) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); - } -#else - ret = ipc4_delete_pipeline(ipc4); -#endif - break; - case SOF_IPC4_GLB_SET_PIPELINE_STATE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - struct ipc4_pipeline_set_state state = { - .primary.dat = ipc4->primary.dat, - .extension.dat = ipc4->extension.dat, - }; - const uint32_t *ppl_id; - unsigned int ppl_count; - int id = ipc4_pipeline_id_get(ipc4, &state, &ppl_id, &ppl_count); - if (id < 0) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, - IPC_COMP_ALL); - - if (!ppl) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - /* - * Validate all pipelines before any state change and check, - * if they span multiple cores, needing per-core processing - */ - bool multicore = false; - - ret = IPC4_SUCCESS; - for (unsigned int i = 1; i < ppl_count; i++) { - struct ipc_comp_dev *icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, - ppl_id[i], IPC_COMP_ALL); - - if (!icd) { - ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - if (icd->core != ppl->core) - multicore = true; - } - - if (ret != IPC4_SUCCESS) - break; - - if (multicore) - ret = ipc4_user_set_pipeline_state_multicore(ipc4, ppl_id, ppl_count); - else - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ppl->core); - } -#else /* CONFIG_SOF_USERSPACE_LL */ - ret = ipc4_set_pipeline_state(ipc4); -#endif /* CONFIG_SOF_USERSPACE_LL */ - break; - - case SOF_IPC4_GLB_GET_PIPELINE_STATE: - case SOF_IPC4_GLB_GET_PIPELINE_CONTEXT_SIZE: - case SOF_IPC4_GLB_SAVE_PIPELINE: - case SOF_IPC4_GLB_RESTORE_PIPELINE: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_INTERNAL_MESSAGE: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - /* Notification (FW to SW driver) */ - case SOF_IPC4_GLB_NOTIFICATION: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_IPCGATEWAY_CMD: - ret = ipc4_process_ipcgtw_cmd(ipc4); - break; - - case SOF_IPC4_GLB_ENTER_GDB: - ret = ipc_glb_gdb_debug(ipc4); - break; - - default: - ipc_cmd_err(&ipc_tr, "unsupported ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - } - - return ret; -} - -/* - * Ipc4 Module message <------> ipc3 module message - * init module <-------> create component - * bind modules <-------> connect components - * module set_large_config <-------> component cmd - * delete module <-------> free component - */ - -__cold static int ipc4_init_module_instance(struct ipc4_message_request *ipc4) -{ - const struct ipc4_module_init_instance *module_init = - (const struct ipc4_module_init_instance *)ipc4; - struct comp_dev *dev; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)module_init->primary.r.module_id, - (uint32_t)module_init->primary.r.instance_id); - - /* Pass IPC to target core */ - if (!cpu_is_me(module_init->extension.r.core_id)) - return ipc4_process_on_core(module_init->extension.r.core_id, false); - - dev = comp_new_ipc4(module_init); - if (!dev) { - ipc_cmd_err(&ipc_tr, "error: failed to init module %x : %x", - (uint32_t)module_init->primary.r.module_id, - (uint32_t)module_init->primary.r.instance_id); - return IPC4_MOD_NOT_INITIALIZED; - } - - return 0; -} - -#ifndef CONFIG_SOF_USERSPACE_LL -__cold static int ipc4_bind_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x with %x : %x", - (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, - (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); - - return ipc4_comp_connect(ipc, bu); -} - -__cold static int ipc4_unbind_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x with %x : %x", - (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, - (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); - - return ipc4_comp_disconnect(ipc, bu); -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - /** * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context. * @@ -1077,23 +671,8 @@ __cold int ipc4_process_module_config(struct ipc4_message_request *ipc4, return ret; } -#ifndef CONFIG_SOF_USERSPACE_LL -static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4, bool set) -{ - uint32_t reply_ext; - int ret; - - ret = ipc4_process_module_config(ipc4, set, &reply_ext); - - if (!set) - msg_reply->extension = reply_ext; - - return ret; -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - -__cold static void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, - char *data_out, uint32_t data_size) +__cold void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, + char *data_out, uint32_t data_size) { const char *hostbox; @@ -1122,13 +701,13 @@ __cold static void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t p } } -__cold static int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, - const struct comp_driver *drv, - bool init_block, - bool final_block, - uint32_t *data_off_size, - char *data_out, - const char *data_in) +__cold int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + bool init_block, + bool final_block, + uint32_t *data_off_size, + char *data_out, + const char *data_in) { const struct sof_tl * const input_tl = (struct sof_tl *)data_in; int ret; @@ -1324,119 +903,7 @@ __cold static int ipc4_process_large_config_get(struct ipc4_message_request *ipc } #endif -__cold static int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_large_config_reply reply; - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - char *data = ipc_get()->comp_data; - const struct comp_driver *drv; - struct comp_dev *dev = NULL; - uint32_t data_offset; - int ret; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); - - /* get component dev for non-basefw since there is no - * component dev for basefw - */ - if (config->primary.r.module_id) { - uint32_t comp_id; - - comp_id = IPC4_COMP_ID(config->primary.r.module_id, - config->primary.r.instance_id); - dev = ipc4_get_comp_dev(comp_id); - if (!dev) - return IPC4_INVALID_RESOURCE_ID; - - drv = dev->drv; - - /* Pass IPC to target core */ - if (!cpu_is_me(dev->ipc_config.core)) - return ipc4_process_on_core(dev->ipc_config.core, false); - } else { - /* BaseFW module has only 0th instance */ - if (config->primary.r.instance_id) - return IPC4_INVALID_RESOURCE_ID; - - drv = ipc4_get_comp_drv(config->primary.r.module_id); - } - - if (!drv) - return IPC4_INVALID_RESOURCE_ID; - - if (!drv->ops.get_large_config) - return IPC4_INVALID_REQUEST; - - data_offset = config->extension.r.data_off_size; - - /* check for vendor param first */ - if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { - /* data_off_size is a 20-bit host-controlled field, so it can - * claim far more than the hostbox can physically hold. - */ - if (data_offset > MAILBOX_HOSTBOX_SIZE) { - ipc_cmd_err(&ipc_tr, "data_off_size %u exceeds mailbox bound", - data_offset); - return IPC4_INVALID_CONFIG_DATA_STRUCT; - } - /* For now only vendor_config case uses payload from hostbox */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config->extension.r.data_off_size); - ret = ipc4_get_vendor_config_module_instance(dev, drv, - config->extension.r.init_block, - config->extension.r.final_block, - &data_offset, data, - (const char *)MAILBOX_HOSTBOX_BASE); - } else { -#if CONFIG_LIBRARY - data += sizeof(reply); -#endif - ipc4_prepare_for_kcontrol_get(dev, config->extension.r.large_param_id, - data, data_offset); - - ret = drv->ops.get_large_config(dev, config->extension.r.large_param_id, - config->extension.r.init_block, - config->extension.r.final_block, - &data_offset, data); - } - - /* set up ipc4 error code for reply data */ - if (ret < 0) - ret = IPC4_INVALID_RESOURCE_ID; - - /* Copy host config and overwrite */ - reply.extension.dat = config->extension.dat; - reply.extension.r.data_off_size = data_offset; - - /* The last block, no more data */ - if (!config->extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) - reply.extension.r.final_block = 1; - - /* Indicate last block if error occurs */ - if (ret) - reply.extension.r.final_block = 1; - - /* no need to allocate memory for reply msg */ - if (ret) - return ret; - - /* A module must not report more reply data than the DSP mailbox can hold. */ - if (data_offset > MAILBOX_DSPBOX_SIZE) { - ipc_cmd_err(&ipc_tr, "reply size %u exceeds mailbox bound", data_offset); - return IPC4_INVALID_CONFIG_DATA_STRUCT; - } - - msg_reply->extension = reply.extension.dat; - msg_reply->tx_size = data_offset; - msg_reply->tx_data = data; - return ret; -} - -__cold static int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, +__cold int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, const struct comp_driver *drv, uint32_t module_id, uint32_t instance_id, @@ -1586,283 +1053,6 @@ __cold int ipc4_process_large_config_set(struct ipc4_message_request *ipc4) return ret; } -__cold static int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4) -{ - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - struct comp_dev *dev = NULL; - const struct comp_driver *drv; - int ret; - - assert_can_be_cold(); - - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config->extension.r.data_off_size); - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); - - if (config->primary.r.module_id) { - uint32_t comp_id; - - comp_id = IPC4_COMP_ID(config->primary.r.module_id, config->primary.r.instance_id); - dev = ipc4_get_comp_dev(comp_id); - if (!dev) - return IPC4_INVALID_RESOURCE_ID; - - drv = dev->drv; - - /* Pass IPC to target core */ - if (!cpu_is_me(dev->ipc_config.core)) - return ipc4_process_on_core(dev->ipc_config.core, false); - } else { - /* BaseFW module has only 0th instance */ - if (config->primary.r.instance_id) - return IPC4_INVALID_RESOURCE_ID; - - drv = ipc4_get_comp_drv(config->primary.r.module_id); - } - - if (!drv) - return IPC4_INVALID_RESOURCE_ID; - - if (!drv->ops.set_large_config) - return IPC4_INVALID_REQUEST; - - /* check for vendor param first */ - if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { - ret = ipc4_set_vendor_config_module_instance(dev, drv, - (uint32_t)config->primary.r.module_id, - (uint32_t)config->primary.r.instance_id, - config->extension.r.init_block, - config->extension.r.final_block, - config->extension.r.data_off_size, - (const char *)MAILBOX_HOSTBOX_BASE); - } else { -#if CONFIG_LIBRARY - struct ipc *ipc = ipc_get(); - const char *data = (const char *)ipc->comp_data + sizeof(*config); -#else - const char *data = (const char *)MAILBOX_HOSTBOX_BASE; -#endif - ret = drv->ops.set_large_config(dev, config->extension.r.large_param_id, - config->extension.r.init_block, config->extension.r.final_block, - config->extension.r.data_off_size, data); - if (ret < 0) { - ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", - (uint32_t)config->primary.r.module_id, - (uint32_t)config->primary.r.instance_id); - ret = IPC4_INVALID_RESOURCE_ID; - } - } - - return ret; -} - -#ifndef CONFIG_SOF_USERSPACE_LL -__cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_delete_instance module; - struct ipc *ipc = ipc_get(); - uint32_t comp_id; - - assert_can_be_cold(); - - int ret = memcpy_s(&module, sizeof(module), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - - tr_dbg(&ipc_tr, "%x : %x", (uint32_t)module.primary.r.module_id, - (uint32_t)module.primary.r.instance_id); - - comp_id = IPC4_COMP_ID(module.primary.r.module_id, module.primary.r.instance_id); - ret = ipc_comp_free(ipc, comp_id); - if (ret < 0) { - ipc_cmd_err(&ipc_tr, "failed to delete module instance %x : %x", - (uint32_t)module.primary.r.module_id, - (uint32_t)module.primary.r.instance_id); - ret = IPC4_INVALID_RESOURCE_ID; - } - - return ret; -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - -__cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, - struct ipc_msg *reply) -{ - const struct ipc4_module_init_instance *mi; - uint32_t type; - int ret; - - assert_can_be_cold(); - - type = ipc4->primary.r.type; - msg_reply = reply; - - switch (type) { - case SOF_IPC4_MOD_INIT_INSTANCE: - mi = (const struct ipc4_module_init_instance *)ipc4; - - if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain && - IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { -#ifdef CONFIG_SOF_USERSPACE_LL - BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= - sizeof(((struct ipc_user *)0)->init_drv_data), - "ipc_user.init_drv_data too small for driver copy"); - - /* User-space init: kernel does driver lookup only (requires - * access to IMR manifest and driver list in kernel memory). - * Component creation (drv->ops.create) runs in user thread - * so untrusted module code does not execute in kernel context. - */ - uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, - mi->primary.r.instance_id); - const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); - - if (!drv || !drv->ops.create) { - ret = IPC4_MOD_NOT_INITIALIZED; - break; - } - - struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); - - if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { - int err = ipc4_user_module_load(drv, mi); - - if (err < 0) { - ret = IPC4_MOD_NOT_INITIALIZED; - break; - } - } - - ipc_user_get()->init_drv = drv; - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - mi->extension.r.core_id); -#endif /* CONFIG_SOF_USERSPACE_LL */ - } else { - /* - * DP module creation starts running in kernel mode and - * switches to userspace later via a scheduler_dp_thread_ipc() - * call in module_init(). - */ - ret = ipc4_init_module_instance(ipc4); - } - break; - case SOF_IPC4_MOD_CONFIG_GET: -#ifdef CONFIG_SOF_USERSPACE_LL - /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - if (!ret) { - struct ipc_user *pdata = ipc_user_get(); - - msg_reply->extension = pdata->reply_ext; - } -#else - ret = ipc4_set_get_config_module_instance(ipc4, false); -#endif - break; - case SOF_IPC4_MOD_CONFIG_SET: -#ifdef CONFIG_SOF_USERSPACE_LL - /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_set_get_config_module_instance(ipc4, true); -#endif - break; - case SOF_IPC4_MOD_LARGE_CONFIG_GET: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - - if (config->primary.r.module_id) { - /* Module case: forward to user thread */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - if (!ret) { - struct ipc_user *pdata = ipc_user_get(); - - msg_reply->extension = pdata->reply_ext; - msg_reply->tx_size = pdata->reply_tx_size; - msg_reply->tx_data = pdata->reply_tx_data; - } - } else { - /* Base firmware (module_id==0): keep in kernel — - * ipc4_get_comp_drv() accesses IMR manifest which - * has no user-space partition. - */ - ret = ipc4_get_large_config_module_instance(ipc4); - } - } -#else /* CONFIG_SOF_USERSPACE_LL */ - ret = ipc4_get_large_config_module_instance(ipc4); -#endif /* CONFIG_SOF_USERSPACE_LL */ - break; - case SOF_IPC4_MOD_LARGE_CONFIG_SET: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - - if (config->primary.r.module_id) { - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - } else { - /* Base firmware: keep in kernel (IMR access) */ - ret = ipc4_set_large_config_module_instance(ipc4); - } - } -#else - ret = ipc4_set_large_config_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_BIND: -#ifdef CONFIG_SOF_USERSPACE_LL - /* - * bind and unbind can connect LL with DP. In that case it isn't - * immediately clear whether the handler should run in the DP - * thread context or in the LL IPC thread context. The LL IPC - * thread has access to DP modules, so we have to perform - * binding in the LL IPC thread context - */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_bind_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_UNBIND: -#ifdef CONFIG_SOF_USERSPACE_LL - /* DP / LL: see comment above */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_unbind_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_DELETE_INSTANCE: -#ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_delete_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: - case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: - ret = IPC4_UNAVAILABLE; - break; - default: - ret = IPC4_UNKNOWN_MESSAGE_TYPE; - break; - } - - return ret; -} - #ifdef CONFIG_SOF_USERSPACE_LL /* * IPC4 implementation of the user-thread dispatch hook. Runs in the diff --git a/src/kernel/ipc/ipc4/CMakeLists.txt b/src/kernel/ipc/ipc4/CMakeLists.txt index 53a37d553675..2cc1dae13f2d 100644 --- a/src/kernel/ipc/ipc4/CMakeLists.txt +++ b/src/kernel/ipc/ipc4/CMakeLists.txt @@ -3,6 +3,15 @@ # Files common to Zephyr and the host library add_local_sources(sof handler-kernel.c + handler-user-kernel.c logging.c notification.c ) + +# The IPC4 message routing for the LL scheduler in user space or in kernel +# space, see handler-user-kernel.c +if(CONFIG_SOF_USERSPACE_LL) + add_local_sources(sof handler-ll-user.c) +else() + add_local_sources(sof handler-ll-kernel.c) +endif() diff --git a/src/kernel/ipc/ipc4/handler-ll-kernel.c b/src/kernel/ipc/ipc4/handler-ll-kernel.c new file mode 100644 index 000000000000..6571cf54288a --- /dev/null +++ b/src/kernel/ipc/ipc4/handler-ll-kernel.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang +/* + * IPC (InterProcessor Communication) provides a method of two way + * communication between the host processor and the DSP. The IPC used here + * utilises a shared mailbox and door bell between the host and DSP. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + + +#include "handler-route.h" + +/* + * IPC4 message routing with the LL scheduler in kernel space (without + * CONFIG_SOF_USERSPACE_LL). The routers in handler-user-kernel.c run in the + * kernel IPC thread and call these functions, which run the handlers in + * src/ipc/ipc4/handler-user.c and src/ipc/ipc4/helper.c directly in that + * thread. With the LL scheduler in user space, handler-ll-user.c forwards + * the messages to the user-space IPC thread instead. + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +__cold int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4) +{ + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); +} + + +__cold int ipc4_route_delete_pipeline(struct ipc4_message_request *ipc4) +{ + struct ipc4_pipeline_delete *pipe; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + pipe = (struct ipc4_pipeline_delete *)ipc4; + tr_dbg(&ipc_tr, "ipc4 delete pipeline %x:", (uint32_t)pipe->primary.r.instance_id); + + return ipc_pipeline_free(ipc, pipe->primary.r.instance_id); +} + + +int ipc4_route_set_pipeline_state(struct ipc4_message_request *ipc4) +{ + return ipc4_set_pipeline_state(ipc4); +} + +__cold int ipc4_route_init_module(struct ipc4_message_request *ipc4) +{ + return ipc4_init_module_instance(ipc4); +} + +int ipc4_route_module_config(struct ipc4_message_request *ipc4, bool set, + struct ipc_msg *reply) +{ + uint32_t reply_ext; + int ret; + + ret = ipc4_process_module_config(ipc4, set, &reply_ext); + + if (!set) + reply->extension = reply_ext; + + return ret; +} + + +__cold int ipc4_route_large_config_get(struct ipc4_message_request *ipc4, struct ipc_msg *reply) +{ + return ipc4_get_large_config_module_instance(ipc4); +} + +__cold int ipc4_route_large_config_set(struct ipc4_message_request *ipc4) +{ + return ipc4_set_large_config_module_instance(ipc4); +} + +__cold int ipc4_route_bind(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x with %x : %x", + (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, + (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); + + return ipc4_comp_connect(ipc, bu); +} + + +__cold int ipc4_route_unbind(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x with %x : %x", + (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, + (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); + + return ipc4_comp_disconnect(ipc, bu); +} + + +__cold int ipc4_route_delete_module(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_delete_instance module; + struct ipc *ipc = ipc_get(); + uint32_t comp_id; + + assert_can_be_cold(); + + int ret = memcpy_s(&module, sizeof(module), ipc4, sizeof(*ipc4)); + + if (ret < 0) + return IPC4_FAILURE; + + tr_dbg(&ipc_tr, "%x : %x", (uint32_t)module.primary.r.module_id, + (uint32_t)module.primary.r.instance_id); + + comp_id = IPC4_COMP_ID(module.primary.r.module_id, module.primary.r.instance_id); + ret = ipc_comp_free(ipc, comp_id); + if (ret < 0) { + ipc_cmd_err(&ipc_tr, "failed to delete module instance %x : %x", + (uint32_t)module.primary.r.module_id, + (uint32_t)module.primary.r.instance_id); + ret = IPC4_INVALID_RESOURCE_ID; + } + + return ret; +} + diff --git a/src/kernel/ipc/ipc4/handler-ll-user.c b/src/kernel/ipc/ipc4/handler-ll-user.c new file mode 100644 index 000000000000..e69f88485587 --- /dev/null +++ b/src/kernel/ipc/ipc4/handler-ll-user.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang +/* + * IPC (InterProcessor Communication) provides a method of two way + * communication between the host processor and the DSP. The IPC used here + * utilises a shared mailbox and door bell between the host and DSP. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + + +#include "handler-route.h" + +/* + * IPC4 message routing with the LL scheduler in user space + * (CONFIG_SOF_USERSPACE_LL). The routers in handler-user-kernel.c run in the + * kernel IPC thread and call these functions for the messages that create, + * configure, connect and delete pipelines and modules. They do what needs + * kernel mode (target core lookup, multicore sequencing, driver lookup and + * module loading) and forward the message to the user-space IPC thread of the + * target core, which runs the handlers in src/ipc/ipc4/handler-user.c. With + * the LL scheduler in kernel space, handler-ll-kernel.c calls those handlers + * directly instead. + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +/* + * Determine the target core for an IPC4 module message. + * Falls back to current core when no component is bound yet. + */ +static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) +{ + /* + * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, + * struct ipc4_module_delete_instance + */ + struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; + uint32_t module_id = config->primary.r.module_id; + + if (module_id) { + uint32_t instance_id = config->primary.r.instance_id; + struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); + + if (dev) + return dev->ipc_config.core; + } + + return cpu_get_id(); +} + +/* + * Handle a multi-pipeline SET_PIPELINE_STATE, where the pipelines are on + * different cores. The user-space IPC thread can't use IDC to reach the + * other cores, so the kernel runs the prepare phase on all pipelines and + * then the trigger phase on all pipelines, forwarding each step to the + * user-space IPC thread of the core, owning the pipeline. + */ +__cold static int ipc4_user_set_pipeline_state_multicore(struct ipc4_message_request *ipc4, + const uint32_t *ppl_id, + unsigned int ppl_count) +{ + static const uint32_t phases[] = { + IDC_PPL_STATE_PHASE_PREPARE, + IDC_PPL_STATE_PHASE_TRIGGER, + }; + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc->ipc_user_pdata; + struct ipc4_pipeline_set_state state; + struct ipc_comp_dev *ppl_icd; + unsigned int i, j; + int ret; + + assert_can_be_cold(); + + for (j = 0; j < ARRAY_SIZE(phases); j++) { + for (i = 0; i < ppl_count; i++) { + ppl_icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, ppl_id[i], + IPC_COMP_ALL); + if (!ppl_icd) { + ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); + return IPC4_INVALID_RESOURCE_ID; + } + + /* Forward as a single-pipeline request */ + state.primary.dat = ipc4->primary.dat; + state.extension.dat = ipc4->extension.dat; + state.primary.r.ppl_id = ppl_id[i]; + state.extension.r.multi_ppl = 0; + + pdata->ppl_phase = phases[j]; + ret = ipc_user_forward_cmd(state.primary.dat, state.extension.dat, + ppl_icd->core); + pdata->ppl_phase = 0; + if (ret != 0) + return ret; + } + } + + return IPC4_SUCCESS; +} + +int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4) +{ + const struct ipc4_pipeline_create *create = + (const struct ipc4_pipeline_create *)ipc4; + + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + create->extension.r.core_id); +} + +int ipc4_route_delete_pipeline(struct ipc4_message_request *ipc4) +{ + struct ipc *ipc = ipc_get(); + const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + del->primary.r.instance_id, + IPC_COMP_ALL); + + if (!ppl) + return IPC4_INVALID_RESOURCE_ID; + + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); +} + +int ipc4_route_set_pipeline_state(struct ipc4_message_request *ipc4) +{ + struct ipc *ipc = ipc_get(); + struct ipc4_pipeline_set_state state = { + .primary.dat = ipc4->primary.dat, + .extension.dat = ipc4->extension.dat, + }; + const uint32_t *ppl_id; + unsigned int ppl_count; + int id = ipc4_pipeline_id_get(ipc4, &state, &ppl_id, &ppl_count); + + if (id < 0) + return IPC4_INVALID_RESOURCE_ID; + + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, + IPC_COMP_ALL); + + if (!ppl) + return IPC4_INVALID_RESOURCE_ID; + + /* + * Validate all pipelines before any state change and check, + * if they span multiple cores, needing per-core processing + */ + bool multicore = false; + + for (unsigned int i = 1; i < ppl_count; i++) { + struct ipc_comp_dev *icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + ppl_id[i], IPC_COMP_ALL); + + if (!icd) { + ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); + return IPC4_INVALID_RESOURCE_ID; + } + + if (icd->core != ppl->core) + multicore = true; + } + + if (multicore) + return ipc4_user_set_pipeline_state_multicore(ipc4, ppl_id, ppl_count); + + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); +} + +__cold int ipc4_route_init_module(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_init_instance *mi = + (const struct ipc4_module_init_instance *)ipc4; + + if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain) { + BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= + sizeof(((struct ipc_user *)0)->init_drv_data), + "ipc_user.init_drv_data too small for driver copy"); + + /* User-space init: kernel does driver lookup only (requires + * access to IMR manifest and driver list in kernel memory). + * Component creation (drv->ops.create) runs in user thread + * so untrusted module code does not execute in kernel context. + */ + uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, + mi->primary.r.instance_id); + const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); + + if (!drv || !drv->ops.create) + return IPC4_MOD_NOT_INITIALIZED; + + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); + + if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { + int err = ipc4_user_module_load(drv, mi); + + if (err < 0) + return IPC4_MOD_NOT_INITIALIZED; + } + + ipc_user_get()->init_drv = drv; + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + mi->extension.r.core_id); + } + + /* + * DP module creation starts running in kernel mode and + * switches to userspace later via a scheduler_dp_thread_ipc() + * call in module_init(). + */ + return ipc4_init_module_instance(ipc4); +} + +__cold int ipc4_route_module_config(struct ipc4_message_request *ipc4, bool set, + struct ipc_msg *reply) +{ + /* Forward to user thread for privilege-separated execution */ + int ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); + + if (!set && !ret) { + struct ipc_user *pdata = ipc_user_get(); + + reply->extension = pdata->reply_ext; + } + + return ret; +} + +__cold int ipc4_route_large_config_get(struct ipc4_message_request *ipc4, struct ipc_msg *reply) +{ + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + int ret; + + /* Base firmware (module_id==0): keep in kernel — + * ipc4_get_comp_drv() accesses IMR manifest which + * has no user-space partition. + */ + if (!config->primary.r.module_id) + return ipc4_get_large_config_module_instance(ipc4); + + /* Module case: forward to user thread */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); + if (!ret) { + struct ipc_user *pdata = ipc_user_get(); + + reply->extension = pdata->reply_ext; + reply->tx_size = pdata->reply_tx_size; + reply->tx_data = pdata->reply_tx_data; + } + + return ret; +} + +__cold int ipc4_route_large_config_set(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + + /* Base firmware: keep in kernel (IMR access) */ + if (!config->primary.r.module_id) + return ipc4_set_large_config_module_instance(ipc4); + + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +} + +/* + * bind and unbind can connect LL with DP. In that case it isn't + * immediately clear whether the handler should run in the DP + * thread context or in the LL IPC thread context. The LL IPC + * thread has access to DP modules, so we have to perform + * binding in the LL IPC thread context + */ +__cold int ipc4_route_bind(struct ipc4_message_request *ipc4) +{ + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +} + +/* DP / LL: see ipc4_route_bind() */ +__cold int ipc4_route_unbind(struct ipc4_message_request *ipc4) +{ + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +} + +__cold int ipc4_route_delete_module(struct ipc4_message_request *ipc4) +{ + return ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +} diff --git a/src/kernel/ipc/ipc4/handler-route.h b/src/kernel/ipc/ipc4/handler-route.h new file mode 100644 index 000000000000..75d80d0cea4c --- /dev/null +++ b/src/kernel/ipc/ipc4/handler-route.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + */ + +/* + * Private to the kernel IPC4 message routing in src/kernel/ipc/ipc4: + * handler-user-kernel.c, the routers common to all builds, calls the + * ipc4_route_*() functions for the messages that are handled differently + * with the LL scheduler in user space (handler-ll-user.c, built with + * CONFIG_SOF_USERSPACE_LL) and in kernel space (handler-ll-kernel.c). + */ + +#ifndef __KERNEL_IPC_IPC4_HANDLER_ROUTE_H__ +#define __KERNEL_IPC_IPC4_HANDLER_ROUTE_H__ + +#include + +struct ipc4_message_request; +struct ipc_msg; + +/* Command format errors during fuzzing are reported for virtually all + * commands, and the resulting flood of logging becomes a severe + * performance penalty (i.e. we get a lot less fuzzing done per CPU + * cycle). + */ +#ifdef CONFIG_ARCH_POSIX_LIBFUZZER +#define ipc_cmd_err(...) +#else +#define ipc_cmd_err(...) tr_err(__VA_ARGS__) +#endif + +/* Handlers in handler-user-kernel.c, used by both routing variants */ +int ipc4_init_module_instance(struct ipc4_message_request *ipc4); +int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4); +int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4); + +/* Global messages */ +int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4); +int ipc4_route_delete_pipeline(struct ipc4_message_request *ipc4); +int ipc4_route_set_pipeline_state(struct ipc4_message_request *ipc4); + +/* Module messages */ +int ipc4_route_init_module(struct ipc4_message_request *ipc4); +int ipc4_route_module_config(struct ipc4_message_request *ipc4, bool set, + struct ipc_msg *reply); +int ipc4_route_large_config_get(struct ipc4_message_request *ipc4, struct ipc_msg *reply); +int ipc4_route_large_config_set(struct ipc4_message_request *ipc4); +int ipc4_route_bind(struct ipc4_message_request *ipc4); +int ipc4_route_unbind(struct ipc4_message_request *ipc4); +int ipc4_route_delete_module(struct ipc4_message_request *ipc4); + +#endif /* __KERNEL_IPC_IPC4_HANDLER_ROUTE_H__ */ diff --git a/src/kernel/ipc/ipc4/handler-user-kernel.c b/src/kernel/ipc/ipc4/handler-user-kernel.c new file mode 100644 index 000000000000..e99808955540 --- /dev/null +++ b/src/kernel/ipc/ipc4/handler-user-kernel.c @@ -0,0 +1,509 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang +/* + * IPC (InterProcessor Communication) provides a method of two way + * communication between the host processor and the DSP. The IPC used here + * utilises a shared mailbox and door bell between the host and DSP. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include "../audio/copier/ipcgtw_copier.h" + +#include "handler-route.h" + +/* + * Kernel part of the IPC4 global and module message handling. + * + * handler-kernel.c receives the host IPC in the kernel IPC thread and passes + * the global messages to ipc4_user_process_glb_message() and the module + * messages to ipc4_user_process_module_message() here. These routers, also + * running in the kernel IPC thread, are the boundary between kernel and user + * space for IPC4: + * - some messages are handled here, in kernel mode, in all builds: CHAIN_DMA, + * the IPC gateway, GDB, and LARGE_CONFIG get and set of the base firmware + * (module_id 0), which reads the IMR manifest; + * - the messages that create, configure, connect and delete pipelines and + * modules are passed to the ipc4_route_*() functions. With the LL scheduler + * in user space (CONFIG_SOF_USERSPACE_LL) these are in handler-ll-user.c and + * forward the message to the user-space IPC thread, which runs the handlers + * in src/ipc/ipc4/handler-user.c. Otherwise they are in handler-ll-kernel.c + * and run those handlers directly in the kernel IPC thread. + * The "user" in the name of this file and of the routers refers to the + * messages, which are handled by the user-space part where possible. + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +/* Userspace message context, copied in/out by kernel IPC thread. */ +/* fw sends a fw ipc message to send the status of the last host ipc message */ +static struct ipc_msg *msg_reply; + +__cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) +{ + assert_can_be_cold(); + +#if CONFIG_COMP_CHAIN_DMA + struct ipc_comp_dev *cdma_comp; + struct ipc *ipc = ipc_get(); + const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4; + int comp_id; + int ret; + + comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); + cdma_comp = ipc_get_comp_by_id(ipc, comp_id); + + if (!cdma_comp) { + /* + * Nothing to do when the chainDMA is not allocated and asked to + * be freed + */ + if (!cdma->primary.r.allocate && !cdma->primary.r.enable) + return IPC4_SUCCESS; + + ret = ipc4_chain_manager_create(cdma); + if (ret < 0) + return IPC4_FAILURE; + + cdma_comp = ipc_get_comp_by_id(ipc, comp_id); + if (!cdma_comp) { + return IPC4_FAILURE; + } + + ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); + if (ret < 0) { + comp_free(cdma_comp->cd); + list_item_del(&cdma_comp->list); + sof_heap_free(sof_sys_user_heap_get(), cdma_comp); + return IPC4_FAILURE; + } + + return IPC4_SUCCESS; + } + + ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); + if (ret < 0) + return IPC4_INVALID_CHAIN_STATE_TRANSITION; + + return IPC4_SUCCESS; +#else /* CONFIG_COMP_CHAIN_DMA */ + return IPC4_UNAVAILABLE; +#endif /* CONFIG_COMP_CHAIN_DMA */ +} + + +__cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4) +{ + assert_can_be_cold(); + +#if CONFIG_IPC4_GATEWAY + struct ipc *ipc = ipc_get(); + uint32_t reply_size = 0; + int err; + + err = copier_ipcgtw_process((const struct ipc4_ipcgtw_cmd *)ipc4, ipc->comp_data, + &reply_size); + /* reply size is returned in header extension dword */ + msg_reply->extension = reply_size; + + if (reply_size > 0) { + msg_reply->tx_data = ipc->comp_data; + msg_reply->tx_size = reply_size; + } + + return err < 0 ? IPC4_FAILURE : IPC4_SUCCESS; +#else /* CONFIG_IPC4_GATEWAY */ + ipc_cmd_err(&ipc_tr, "CONFIG_IPC4_GATEWAY is disabled"); + return IPC4_UNAVAILABLE; +#endif /* CONFIG_IPC4_GATEWAY */ +} + + +static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) +{ +#if CONFIG_GDBSTUB + ipc_enter_gdb = true; + return IPC4_SUCCESS; +#else + return IPC4_UNAVAILABLE; +#endif +} + + +int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + uint32_t type; + int ret; + + type = ipc4->primary.r.type; + msg_reply = reply; + + switch (type) { + case SOF_IPC4_GLB_BOOT_CONFIG: + case SOF_IPC4_GLB_ROM_CONTROL: + case SOF_IPC4_GLB_PERF_MEASUREMENTS_CMD: + case SOF_IPC4_GLB_LOAD_MULTIPLE_MODULES: + case SOF_IPC4_GLB_UNLOAD_MULTIPLE_MODULES: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_CHAIN_DMA: + ret = ipc4_process_chain_dma(ipc4); + break; + + /* pipeline settings */ + case SOF_IPC4_GLB_CREATE_PIPELINE: + ret = ipc4_route_create_pipeline(ipc4); + break; + case SOF_IPC4_GLB_DELETE_PIPELINE: + ret = ipc4_route_delete_pipeline(ipc4); + break; + case SOF_IPC4_GLB_SET_PIPELINE_STATE: + ret = ipc4_route_set_pipeline_state(ipc4); + break; + + case SOF_IPC4_GLB_GET_PIPELINE_STATE: + case SOF_IPC4_GLB_GET_PIPELINE_CONTEXT_SIZE: + case SOF_IPC4_GLB_SAVE_PIPELINE: + case SOF_IPC4_GLB_RESTORE_PIPELINE: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_INTERNAL_MESSAGE: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + /* Notification (FW to SW driver) */ + case SOF_IPC4_GLB_NOTIFICATION: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_IPCGATEWAY_CMD: + ret = ipc4_process_ipcgtw_cmd(ipc4); + break; + + case SOF_IPC4_GLB_ENTER_GDB: + ret = ipc_glb_gdb_debug(ipc4); + break; + + default: + ipc_cmd_err(&ipc_tr, "unsupported ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + } + + return ret; +} + + +/* + * Ipc4 Module message <------> ipc3 module message + * init module <-------> create component + * bind modules <-------> connect components + * module set_large_config <-------> component cmd + * delete module <-------> free component + */ + +__cold int ipc4_init_module_instance(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_init_instance *module_init = + (const struct ipc4_module_init_instance *)ipc4; + struct comp_dev *dev; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id); + + /* Pass IPC to target core */ + if (!cpu_is_me(module_init->extension.r.core_id)) + return ipc4_process_on_core(module_init->extension.r.core_id, false); + + dev = comp_new_ipc4(module_init); + if (!dev) { + ipc_cmd_err(&ipc_tr, "error: failed to init module %x : %x", + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id); + return IPC4_MOD_NOT_INITIALIZED; + } + + return 0; +} + + +__cold int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_large_config_reply reply; + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + char *data = ipc_get()->comp_data; + const struct comp_driver *drv; + struct comp_dev *dev = NULL; + uint32_t data_offset; + int ret; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); + + /* get component dev for non-basefw since there is no + * component dev for basefw + */ + if (config->primary.r.module_id) { + uint32_t comp_id; + + comp_id = IPC4_COMP_ID(config->primary.r.module_id, + config->primary.r.instance_id); + dev = ipc4_get_comp_dev(comp_id); + if (!dev) + return IPC4_INVALID_RESOURCE_ID; + + drv = dev->drv; + + /* Pass IPC to target core */ + if (!cpu_is_me(dev->ipc_config.core)) + return ipc4_process_on_core(dev->ipc_config.core, false); + } else { + /* BaseFW module has only 0th instance */ + if (config->primary.r.instance_id) + return IPC4_INVALID_RESOURCE_ID; + + drv = ipc4_get_comp_drv(config->primary.r.module_id); + } + + if (!drv) + return IPC4_INVALID_RESOURCE_ID; + + if (!drv->ops.get_large_config) + return IPC4_INVALID_REQUEST; + + data_offset = config->extension.r.data_off_size; + + /* check for vendor param first */ + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + /* data_off_size is a 20-bit host-controlled field, so it can + * claim far more than the hostbox can physically hold. + */ + if (data_offset > MAILBOX_HOSTBOX_SIZE) { + ipc_cmd_err(&ipc_tr, "data_off_size %u exceeds mailbox bound", + data_offset); + return IPC4_INVALID_CONFIG_DATA_STRUCT; + } + /* For now only vendor_config case uses payload from hostbox */ + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + config->extension.r.data_off_size); + ret = ipc4_get_vendor_config_module_instance(dev, drv, + config->extension.r.init_block, + config->extension.r.final_block, + &data_offset, data, + (const char *)MAILBOX_HOSTBOX_BASE); + } else { +#if CONFIG_LIBRARY + data += sizeof(reply); +#endif + ipc4_prepare_for_kcontrol_get(dev, config->extension.r.large_param_id, + data, data_offset); + + ret = drv->ops.get_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, + config->extension.r.final_block, + &data_offset, data); + } + + /* set up ipc4 error code for reply data */ + if (ret < 0) + ret = IPC4_INVALID_RESOURCE_ID; + + /* Copy host config and overwrite */ + reply.extension.dat = config->extension.dat; + reply.extension.r.data_off_size = data_offset; + + /* The last block, no more data */ + if (!config->extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) + reply.extension.r.final_block = 1; + + /* Indicate last block if error occurs */ + if (ret) + reply.extension.r.final_block = 1; + + /* no need to allocate memory for reply msg */ + if (ret) + return ret; + + /* A module must not report more reply data than the DSP mailbox can hold. */ + if (data_offset > MAILBOX_DSPBOX_SIZE) { + ipc_cmd_err(&ipc_tr, "reply size %u exceeds mailbox bound", data_offset); + return IPC4_INVALID_CONFIG_DATA_STRUCT; + } + + msg_reply->extension = reply.extension.dat; + msg_reply->tx_size = data_offset; + msg_reply->tx_data = data; + return ret; +} + + +__cold int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + struct comp_dev *dev = NULL; + const struct comp_driver *drv; + int ret; + + assert_can_be_cold(); + + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + config->extension.r.data_off_size); + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); + + if (config->primary.r.module_id) { + uint32_t comp_id; + + comp_id = IPC4_COMP_ID(config->primary.r.module_id, config->primary.r.instance_id); + dev = ipc4_get_comp_dev(comp_id); + if (!dev) + return IPC4_INVALID_RESOURCE_ID; + + drv = dev->drv; + + /* Pass IPC to target core */ + if (!cpu_is_me(dev->ipc_config.core)) + return ipc4_process_on_core(dev->ipc_config.core, false); + } else { + /* BaseFW module has only 0th instance */ + if (config->primary.r.instance_id) + return IPC4_INVALID_RESOURCE_ID; + + drv = ipc4_get_comp_drv(config->primary.r.module_id); + } + + if (!drv) + return IPC4_INVALID_RESOURCE_ID; + + if (!drv->ops.set_large_config) + return IPC4_INVALID_REQUEST; + + /* check for vendor param first */ + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + ret = ipc4_set_vendor_config_module_instance(dev, drv, + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id, + config->extension.r.init_block, + config->extension.r.final_block, + config->extension.r.data_off_size, + (const char *)MAILBOX_HOSTBOX_BASE); + } else { +#if CONFIG_LIBRARY + struct ipc *ipc = ipc_get(); + const char *data = (const char *)ipc->comp_data + sizeof(*config); +#else + const char *data = (const char *)MAILBOX_HOSTBOX_BASE; +#endif + ret = drv->ops.set_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, config->extension.r.final_block, + config->extension.r.data_off_size, data); + if (ret < 0) { + ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id); + ret = IPC4_INVALID_RESOURCE_ID; + } + } + + return ret; +} + + +__cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + uint32_t type; + int ret; + + assert_can_be_cold(); + + type = ipc4->primary.r.type; + msg_reply = reply; + + switch (type) { + case SOF_IPC4_MOD_INIT_INSTANCE: + ret = ipc4_route_init_module(ipc4); + break; + case SOF_IPC4_MOD_CONFIG_GET: + ret = ipc4_route_module_config(ipc4, false, msg_reply); + break; + case SOF_IPC4_MOD_CONFIG_SET: + ret = ipc4_route_module_config(ipc4, true, msg_reply); + break; + case SOF_IPC4_MOD_LARGE_CONFIG_GET: + ret = ipc4_route_large_config_get(ipc4, msg_reply); + break; + case SOF_IPC4_MOD_LARGE_CONFIG_SET: + ret = ipc4_route_large_config_set(ipc4); + break; + case SOF_IPC4_MOD_BIND: + ret = ipc4_route_bind(ipc4); + break; + case SOF_IPC4_MOD_UNBIND: + ret = ipc4_route_unbind(ipc4); + break; + case SOF_IPC4_MOD_DELETE_INSTANCE: + ret = ipc4_route_delete_module(ipc4); + break; + case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: + case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: + ret = IPC4_UNAVAILABLE; + break; + default: + ret = IPC4_UNKNOWN_MESSAGE_TYPE; + break; + } + + return ret; +} + From 3e2230575213a0d1b74e7f8045620e88105499a8 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 22:25:07 +0000 Subject: [PATCH 3/4] ipc4: split the kernel part of helper.c into src/kernel/ipc/ipc4 helper.c mixes the IPC4 topology helpers, which also run in the user-space IPC thread, with code that needs kernel mode. Move the kernel part to src/kernel/ipc/ipc4/helper-kernel.c: - the component driver lookup, which reads the IMR manifest, registers loadable modules with the library manager and walks the driver list; - comp_new_ipc4(), the module creation of the kernel IPC thread (DP modules and builds without user-space LL), and the privileged part of the user-space module creation, ipc4_user_module_load(); - the IDC helpers, ipc4_process_on_core() and ipc_comp_free_remote(); - the chain DMA manager, which only the kernel CHAIN_DMA command uses. helper.c keeps the pipeline and component management, binding and the DMA config parsing. The module config parsing and the module init payload lookup, which both parts use, are no longer static and are declared in sof/ipc/topology.h, as is ipc_comp_free_remote(). helper.c no longer includes the rimage manifest, platform memory map and library manager headers. The code is unchanged, but the functions that now call across the split can no longer inline their callees: ipc_pipeline_new(), ipc_pipeline_free(), ipc4_comp_connect() and ipc4_comp_disconnect() call ipc4_process_on_core() and ipc_comp_free_remote(), and comp_new_ipc4(), ipc4_user_module_load() and ipc4_chain_manager_create() call the module config and component list helpers, instead of inlining them. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/ipc/topology.h | 5 + src/ipc/ipc4/helper.c | 410 +------------------------ src/kernel/ipc/ipc4/CMakeLists.txt | 1 + src/kernel/ipc/ipc4/helper-kernel.c | 459 ++++++++++++++++++++++++++++ 4 files changed, 471 insertions(+), 404 deletions(-) create mode 100644 src/kernel/ipc/ipc4/helper-kernel.c diff --git a/src/include/sof/ipc/topology.h b/src/include/sof/ipc/topology.h index d072abc16a24..1711a0fe3a1c 100644 --- a/src/include/sof/ipc/topology.h +++ b/src/include/sof/ipc/topology.h @@ -48,6 +48,11 @@ struct ipc_comp_dev; const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id); struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id); int ipc4_add_comp_dev(struct comp_dev *dev); +struct comp_ipc_config; +int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init); +unsigned char *ipc4_get_comp_new_data(void); +int ipc_comp_free_remote(struct comp_dev *dev); #ifdef CONFIG_SOF_USERSPACE_LL struct ipc4_message_request; struct comp_driver; diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 5aba6378c3ff..8f69c0d635e7 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -30,26 +30,14 @@ #include #include -/* TODO: Remove platform-specific code, see https://github.com/thesofproject/sof/issues/7549 */ -#if defined(CONFIG_SOC_SERIES_INTEL_ADSP_ACE) || defined(CONFIG_SOC_SERIES_INTEL_ADSP_CAVS) -#define RIMAGE_MANIFEST 1 -#endif - -#ifdef RIMAGE_MANIFEST -#include -#endif - #include #include -#include -#include #include #include #include #include #include #include -#include #include #include @@ -60,11 +48,9 @@ #include #include -LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); - -extern struct tr_ctx comp_tr; +/* The IPC4 helpers that run in kernel mode are in src/kernel/ipc/ipc4/helper-kernel.c */ -static const struct comp_driver *ipc4_get_drv(const void *uuid); +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); void ipc_build_stream_posn(struct sof_ipc_stream_posn *posn, uint32_t type, uint32_t id) @@ -78,7 +64,7 @@ void ipc_build_comp_event(struct sof_ipc_comp_event *event, uint32_t type, } #if CONFIG_LIBRARY -static inline unsigned char *ipc4_get_comp_new_data(void) +unsigned char *ipc4_get_comp_new_data(void) { struct ipc *ipc = ipc_get(); unsigned char *data = (unsigned char *)ipc->comp_data + @@ -86,13 +72,8 @@ static inline unsigned char *ipc4_get_comp_new_data(void) return data; } - -static const struct comp_driver *ipc4_library_get_comp_drv(unsigned char *data) -{ - return ipc4_get_drv(data); -} #else -__cold static inline unsigned char *ipc4_get_comp_new_data(void) +__cold unsigned char *ipc4_get_comp_new_data(void) { assert_can_be_cold(); @@ -100,8 +81,8 @@ __cold static inline unsigned char *ipc4_get_comp_new_data(void) } #endif /* CONFIG_LIBRARY */ -__cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, - const struct ipc4_module_init_instance *module_init) +__cold int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init) { uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, module_init->primary.r.instance_id); @@ -180,112 +161,7 @@ __cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, return 0; } -/* Only called from ipc4_init_module_instance(), which is __cold */ -__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) -{ - struct comp_ipc_config ipc_config; - const struct comp_driver *drv; - struct comp_dev *dev; - unsigned char *data; - uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, - module_init->primary.r.instance_id); - - assert_can_be_cold(); - - int ret = ipc4_comp_new_config(&ipc_config, module_init); - - if (ret < 0) - return NULL; - -#ifdef CONFIG_DCACHE_LINE_SIZE - if (!IS_ENABLED(CONFIG_LIBRARY)) - sys_cache_data_invd_range((__sparse_force void __sparse_cache *) - MAILBOX_HOSTBOX_BASE, - ALIGN_UP(ipc_config.ipc_config_size, - CONFIG_DCACHE_LINE_SIZE)); -#endif - data = ipc4_get_comp_new_data(); -#if CONFIG_LIBRARY - ipc_config.ipc_config_size -= sizeof(struct sof_uuid); - drv = ipc4_library_get_comp_drv(data + ipc_config.ipc_config_size); -#else - drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); -#endif - if (!drv) - return NULL; - - if (!drv->ops.create) { - tr_err(&ipc_tr, "comp 0x%x: module %u is not instantiable", comp_id, - (uint32_t)IPC4_MOD_ID(comp_id)); - return NULL; - } - - if (drv->type == SOF_COMP_MODULE_ADAPTER) { - const struct ipc_config_process spec = { - .data = data, - .size = ipc_config.ipc_config_size, - }; - - dev = drv->ops.create(drv, &ipc_config, &spec); - } else { - dev = drv->ops.create(drv, &ipc_config, data); - } - if (!dev) - return NULL; - - list_init(&dev->bsource_list); - list_init(&dev->bsink_list); - -#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS - /* init global performance measurement */ - dev->perf_data.perf_data_item = perf_data_getnext(); - /* this can be null, just no performance measurements in this case */ - if (dev->perf_data.perf_data_item) { - dev->perf_data.perf_data_item->item.resource_id = comp_id; - if (perf_meas_get_state() != IPC4_PERF_MEASUREMENTS_DISABLED) - comp_init_performance_data(dev); - } -#endif - - ipc4_add_comp_dev(dev); - - comp_update_ibs_obs_cpc(dev); - - return dev; -} - #ifdef CONFIG_SOF_USERSPACE_LL - -int ipc4_user_module_load(const struct comp_driver *drv, - const struct ipc4_module_init_instance *mi) -{ - /* - * A large part of module instantiation is executed in the userspace - * mode, including calling the module's .init() method. But another part - * of it has to be executed in the kernel mode, including calling - * lib_manager_mod_create_priv(). This is the privileged part of module - * instantiation. - */ - struct comp_ipc_config ipc_config; - int ret = ipc4_comp_new_config(&ipc_config, mi); - - if (ret < 0) - return ret; - - const struct ipc_config_process spec = { - .data = ipc4_get_comp_new_data(), - .size = ipc_config.ipc_config_size, - }; - -#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY - sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); -#endif - - struct userspace_context *userspace = NULL; - - return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace); -} - /** * comp_new_ipc4_user - Create component in user-space IPC thread context. * @@ -578,16 +454,6 @@ __cold int ipc_pipeline_new(struct ipc *ipc, ipc_pipe_new *_pipe_desc) return ipc4_create_pipeline(pipe_desc, valid_pparams ? &pparams : NULL); } -__cold static inline int ipc_comp_free_remote(struct comp_dev *dev) -{ - struct idc_msg msg = { IDC_MSG_FREE, IDC_MSG_FREE_EXT(dev->ipc_config.id), - dev->ipc_config.core,}; - - assert_can_be_cold(); - - return idc_send_msg(&msg, IDC_BLOCKING); -} - __cold static int ipc_pipeline_module_free(uint32_t pipeline_id, uint32_t ppl_core) { struct ipc *ipc = ipc_get(); @@ -1263,76 +1129,6 @@ __cold int ipc4_comp_disconnect(struct ipc *ipc, const struct ipc4_module_bind_u return IPC4_SUCCESS; } -#if CONFIG_COMP_CHAIN_DMA -/* Only called from ipc4_process_chain_dma(), which is __cold */ -__cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma) -{ - const struct sof_uuid uuid = SOF_REG_UUID(chain_dma); - const struct comp_driver *drv; - struct comp_dev *dev; - - assert_can_be_cold(); - - drv = ipc4_get_drv(&uuid); - if (!drv) - return -EINVAL; - - dev = drv->ops.create(drv, NULL, cdma); - if (!dev) - return -EINVAL; - - /* differentiate instance by unique ids assignment */ - const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id - + IPC4_MAX_MODULE_COUNT, 0); - dev->ipc_config.id = comp_id; - dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT; - - return ipc4_add_comp_dev(dev); -} - -/* Only called from ipc4_process_chain_dma(), which is __cold */ -__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma) -{ - const bool allocate = cdma->primary.r.allocate; - const bool enable = cdma->primary.r.enable; - struct ipc *ipc = ipc_get(); - struct ipc_comp_dev *icd; - struct list_item *clist, *_tmp; - int ret; - - assert_can_be_cold(); - - if (!dev) - return -EINVAL; - - if (allocate) { - if (enable) - ret = comp_trigger(dev, COMP_TRIGGER_START); - else - ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); - } else { - if (enable) - return -EINVAL; - - /* remove chain part */ - ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); - if (ret < 0) - return ret; - - list_for_item_safe(clist, _tmp, &ipc->comp_list) { - icd = container_of(clist, struct ipc_comp_dev, list); - if (icd->cd != dev) - continue; - list_item_del(&icd->list); - sof_heap_free(sof_sys_user_heap_get(), icd); - break; - } - comp_free(dev); - } - return ret; -} -#endif /* CONFIG_COMP_CHAIN_DMA */ - __cold static int ipc4_update_comps_direction(struct ipc *ipc, uint32_t ppl_id) { struct ipc_comp_dev *icd; @@ -1395,200 +1191,6 @@ int ipc4_pipeline_complete(struct ipc *ipc, uint32_t comp_id, uint32_t cmd) return ipc_pipeline_complete(ipc, comp_id); } -int ipc4_process_on_core(uint32_t core, bool blocking) -{ - int ret; - - ret = ipc_process_on_core(core, blocking); - switch (ret) { - case 0: - case 1: - return IPC4_SUCCESS; - case -EACCES: - return IPC4_INVALID_CORE_ID; - case -ETIME: - case -EBUSY: - return IPC4_BUSY; - default: - return IPC4_FAILURE; - } - - return IPC4_SUCCESS; -} - -__cold static const struct comp_driver *ipc4_search_for_drv(const void *uuid) -{ - struct comp_driver_list *drivers = comp_drivers_get(); - struct list_item *clist; - const struct comp_driver *drv = NULL; - struct comp_driver_info *info; - - assert_can_be_cold(); - - /* search driver list with UUID */ - list_for_item(clist, &drivers->list) { - info = container_of(clist, struct comp_driver_info, list); - if (!memcmp(info->drv->uid, uuid, UUID_SIZE)) { - tr_dbg(&comp_tr, "found type %d, uuid %pU", - info->drv->type, SOF_DRV_UID_NAME(info->drv)); - drv = info->drv; - break; - } - } - - return drv; -} - -__cold static const struct comp_driver *ipc4_get_drv(const void *uuid) -{ - const struct sof_uuid *const __maybe_unused sof_uuid = (const struct sof_uuid *)uuid; - const struct comp_driver *drv; - - assert_can_be_cold(); - - drv = ipc4_search_for_drv(uuid); - if (!drv) - tr_err(&comp_tr, - "the provided UUID (%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x) can't be found!", - sof_uuid->a, sof_uuid->b, sof_uuid->c, sof_uuid->d[0], sof_uuid->d[1], - sof_uuid->d[2], sof_uuid->d[3], sof_uuid->d[4], sof_uuid->d[5], - sof_uuid->d[6], sof_uuid->d[7]); - - return drv; -} - -#if defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) -/* - * ipc4_get_fuzzer_drv - resolve a module driver for native_sim fuzz builds. - * - * native_sim fuzz builds have no rimage manifest, so the manifest lookup in - * ipc4_get_comp_drv() can never resolve a driver and every module instance - * verb (INIT_INSTANCE, CONFIG_GET/SET, LARGE_CONFIG, BIND, UNBIND, - * DELETE_INSTANCE) would be unreachable from the fuzzer. - * - * Mirror the IPC3 whitebox hack in posix/ipc.c and treat module_id as a - * 1-based index into the runtime component driver list, counting only - * instantiable module-adapter drivers so the fuzzer's module_id space matches - * what a real signed manifest would expose. - */ -static const struct comp_driver *ipc4_get_fuzzer_drv(uint32_t module_id) -{ - static const struct sof_uuid basefw_reg_uuid = SOF_REG_UUID(basefw); - struct comp_driver_list *dlist = comp_drivers_get(); - struct list_item *iter; - uint32_t idx = 0; - - if (!module_id) - return ipc4_search_for_drv(&basefw_reg_uuid); - - list_for_item(iter, &dlist->list) { - struct comp_driver_info *inf = - container_of(iter, struct comp_driver_info, list); - - /* - * Skip query-only entries (e.g. BaseFW) that cannot be - * instantiated because they have no create op. - */ - if (!inf->drv->ops.create) - continue; - - /* - * A real signed manifest only lists module-adapter modules. - * The internal gateway drivers (SOF_COMP_HOST, SOF_COMP_DAI) - * are registered for IPC3 but are never IPC4 modules: on the - * IPC4 path they receive no params() pass and cannot be - * configured (e.g. their DMA buffer is never allocated). Skip - * non-module-adapter drivers so the fuzzer's module_id space - * matches a real manifest and cannot map to an unconfigurable - * component. - */ - if (inf->drv->type != SOF_COMP_MODULE_ADAPTER) - continue; - - if (++idx == module_id) - return inf->drv; - } - - tr_err(&comp_tr, "no instantiable driver at module_id %u (%u available)", - module_id, idx); - return NULL; -} -#endif /* defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) */ - -/* - * Called from - * - ipc4_get_large_config_module_instance() - * - ipc4_set_large_config_module_instance() - * - comp_new_ipc4() - * all of which are __cold - */ -__cold const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id) -{ - const struct sof_man_fw_desc *desc = NULL; - const struct sof_man_module *mod; - uint32_t entry_index; - - assert_can_be_cold(); - -#ifdef RIMAGE_MANIFEST - desc = (const struct sof_man_fw_desc *)IMR_BOOT_LDR_MANIFEST_BASE; -#elif defined(CONFIG_ARCH_POSIX_LIBFUZZER) - return ipc4_get_fuzzer_drv(module_id); -#else - /* Non-rimage platforms have no component facility yet. - * This needs to move to the platform layer. - */ - return NULL; -#endif - - uint32_t lib_idx = LIB_MANAGER_GET_LIB_ID(module_id); - - if (lib_idx == 0) { - /* module_id 0 is used for base fw which is in entry 1 or 2 */ - if (!module_id) - entry_index = 1 + IS_ENABLED(CONFIG_COLD_STORE_EXECUTE_DRAM); - else - entry_index = module_id; - - if (entry_index >= desc->header.num_module_entries) { - tr_err(&comp_tr, "Error: entry index %d out of bounds.", entry_index); - return NULL; - } - - mod = (const struct sof_man_module *)((const char *)desc + - SOF_MAN_MODULE_OFFSET(entry_index)); - } else { - /* Library index greater than 0 possible only when LIBRARY_MANAGER - * support enabled. - */ -#if CONFIG_LIBRARY_MANAGER - mod = lib_manager_get_module_manifest(module_id); - - if (!mod) { - tr_err(&comp_tr, "Error: Couldn't find loadable module with id %#x.", - module_id); - return NULL; - } -#else - tr_err(&comp_tr, "Error: lib index:%d, while loadable libraries are not supported!!!", - lib_idx); - return NULL; -#endif - } - -#if CONFIG_LIBRARY_MANAGER - /* Check already registered components */ - const struct comp_driver *drv = ipc4_search_for_drv(&mod->uuid); - - if (drv) - return drv; - - /* New module not registered yet. */ - lib_manager_register_module(module_id); -#endif - return ipc4_get_drv(&mod->uuid); -} - struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id) { struct ipc_comp_dev *icd = ipc_get_comp_by_id(ipc_get(), comp_id); diff --git a/src/kernel/ipc/ipc4/CMakeLists.txt b/src/kernel/ipc/ipc4/CMakeLists.txt index 2cc1dae13f2d..71d856d78322 100644 --- a/src/kernel/ipc/ipc4/CMakeLists.txt +++ b/src/kernel/ipc/ipc4/CMakeLists.txt @@ -4,6 +4,7 @@ add_local_sources(sof handler-kernel.c handler-user-kernel.c + helper-kernel.c logging.c notification.c ) diff --git a/src/kernel/ipc/ipc4/helper-kernel.c b/src/kernel/ipc/ipc4/helper-kernel.c new file mode 100644 index 000000000000..2e8f5ee40474 --- /dev/null +++ b/src/kernel/ipc/ipc4/helper-kernel.c @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* TODO: Remove platform-specific code, see https://github.com/thesofproject/sof/issues/7549 */ +#if defined(CONFIG_SOC_SERIES_INTEL_ADSP_ACE) || defined(CONFIG_SOC_SERIES_INTEL_ADSP_CAVS) +#define RIMAGE_MANIFEST 1 +#endif + +#ifdef RIMAGE_MANIFEST +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +/* Kernel part of the IPC4 helpers: the component driver lookup, which reads the + * IMR manifest and the driver list, module creation in kernel mode, IDC to other + * cores and the chain DMA manager. The part that also runs in user mode is in + * src/ipc/ipc4/helper.c + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +extern struct tr_ctx comp_tr; + +static const struct comp_driver *ipc4_get_drv(const void *uuid); + +#if CONFIG_LIBRARY +static const struct comp_driver *ipc4_library_get_comp_drv(unsigned char *data) +{ + return ipc4_get_drv(data); +} +#endif /* CONFIG_LIBRARY */ + +/* Only called from ipc4_init_module_instance(), which is __cold */ +__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +{ + struct comp_ipc_config ipc_config; + const struct comp_driver *drv; + struct comp_dev *dev; + unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); + + assert_can_be_cold(); + + int ret = ipc4_comp_new_config(&ipc_config, module_init); + + if (ret < 0) + return NULL; + +#ifdef CONFIG_DCACHE_LINE_SIZE + if (!IS_ENABLED(CONFIG_LIBRARY)) + sys_cache_data_invd_range((__sparse_force void __sparse_cache *) + MAILBOX_HOSTBOX_BASE, + ALIGN_UP(ipc_config.ipc_config_size, + CONFIG_DCACHE_LINE_SIZE)); +#endif + data = ipc4_get_comp_new_data(); +#if CONFIG_LIBRARY + ipc_config.ipc_config_size -= sizeof(struct sof_uuid); + drv = ipc4_library_get_comp_drv(data + ipc_config.ipc_config_size); +#else + drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); +#endif + if (!drv) + return NULL; + + if (!drv->ops.create) { + tr_err(&ipc_tr, "comp 0x%x: module %u is not instantiable", comp_id, + (uint32_t)IPC4_MOD_ID(comp_id)); + return NULL; + } + + if (drv->type == SOF_COMP_MODULE_ADAPTER) { + const struct ipc_config_process spec = { + .data = data, + .size = ipc_config.ipc_config_size, + }; + + dev = drv->ops.create(drv, &ipc_config, &spec); + } else { + dev = drv->ops.create(drv, &ipc_config, data); + } + if (!dev) + return NULL; + + list_init(&dev->bsource_list); + list_init(&dev->bsink_list); + +#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + /* init global performance measurement */ + dev->perf_data.perf_data_item = perf_data_getnext(); + /* this can be null, just no performance measurements in this case */ + if (dev->perf_data.perf_data_item) { + dev->perf_data.perf_data_item->item.resource_id = comp_id; + if (perf_meas_get_state() != IPC4_PERF_MEASUREMENTS_DISABLED) + comp_init_performance_data(dev); + } +#endif + + ipc4_add_comp_dev(dev); + + comp_update_ibs_obs_cpc(dev); + + return dev; +} + +#ifdef CONFIG_SOF_USERSPACE_LL +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi) +{ + /* + * A large part of module instantiation is executed in the userspace + * mode, including calling the module's .init() method. But another part + * of it has to be executed in the kernel mode, including calling + * lib_manager_mod_create_priv(). This is the privileged part of module + * instantiation. + */ + struct comp_ipc_config ipc_config; + int ret = ipc4_comp_new_config(&ipc_config, mi); + + if (ret < 0) + return ret; + + const struct ipc_config_process spec = { + .data = ipc4_get_comp_new_data(), + .size = ipc_config.ipc_config_size, + }; + +#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY + sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); +#endif + + struct userspace_context *userspace = NULL; + + return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace); +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc_comp_free_remote(struct comp_dev *dev) +{ + struct idc_msg msg = { IDC_MSG_FREE, IDC_MSG_FREE_EXT(dev->ipc_config.id), + dev->ipc_config.core,}; + + assert_can_be_cold(); + + return idc_send_msg(&msg, IDC_BLOCKING); +} + +#if CONFIG_COMP_CHAIN_DMA +/* Only called from ipc4_process_chain_dma(), which is __cold */ +__cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma) +{ + const struct sof_uuid uuid = SOF_REG_UUID(chain_dma); + const struct comp_driver *drv; + struct comp_dev *dev; + + assert_can_be_cold(); + + drv = ipc4_get_drv(&uuid); + if (!drv) + return -EINVAL; + + dev = drv->ops.create(drv, NULL, cdma); + if (!dev) + return -EINVAL; + + /* differentiate instance by unique ids assignment */ + const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + + IPC4_MAX_MODULE_COUNT, 0); + dev->ipc_config.id = comp_id; + dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT; + + return ipc4_add_comp_dev(dev); +} + +/* Only called from ipc4_process_chain_dma(), which is __cold */ +__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma) +{ + const bool allocate = cdma->primary.r.allocate; + const bool enable = cdma->primary.r.enable; + struct ipc *ipc = ipc_get(); + struct ipc_comp_dev *icd; + struct list_item *clist, *_tmp; + int ret; + + assert_can_be_cold(); + + if (!dev) + return -EINVAL; + + if (allocate) { + if (enable) + ret = comp_trigger(dev, COMP_TRIGGER_START); + else + ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); + } else { + if (enable) + return -EINVAL; + + /* remove chain part */ + ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); + if (ret < 0) + return ret; + + list_for_item_safe(clist, _tmp, &ipc->comp_list) { + icd = container_of(clist, struct ipc_comp_dev, list); + if (icd->cd != dev) + continue; + list_item_del(&icd->list); + sof_heap_free(sof_sys_user_heap_get(), icd); + break; + } + comp_free(dev); + } + return ret; +} +#endif /* CONFIG_COMP_CHAIN_DMA */ + +int ipc4_process_on_core(uint32_t core, bool blocking) +{ + int ret; + + ret = ipc_process_on_core(core, blocking); + switch (ret) { + case 0: + case 1: + return IPC4_SUCCESS; + case -EACCES: + return IPC4_INVALID_CORE_ID; + case -ETIME: + case -EBUSY: + return IPC4_BUSY; + default: + return IPC4_FAILURE; + } + + return IPC4_SUCCESS; +} + +__cold static const struct comp_driver *ipc4_search_for_drv(const void *uuid) +{ + struct comp_driver_list *drivers = comp_drivers_get(); + struct list_item *clist; + const struct comp_driver *drv = NULL; + struct comp_driver_info *info; + + assert_can_be_cold(); + + /* search driver list with UUID */ + list_for_item(clist, &drivers->list) { + info = container_of(clist, struct comp_driver_info, list); + if (!memcmp(info->drv->uid, uuid, UUID_SIZE)) { + tr_dbg(&comp_tr, "found type %d, uuid %pU", + info->drv->type, SOF_DRV_UID_NAME(info->drv)); + drv = info->drv; + break; + } + } + + return drv; +} + +__cold static const struct comp_driver *ipc4_get_drv(const void *uuid) +{ + const struct sof_uuid *const __maybe_unused sof_uuid = (const struct sof_uuid *)uuid; + const struct comp_driver *drv; + + assert_can_be_cold(); + + drv = ipc4_search_for_drv(uuid); + if (!drv) + tr_err(&comp_tr, + "the provided UUID (%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x) can't be found!", + sof_uuid->a, sof_uuid->b, sof_uuid->c, sof_uuid->d[0], sof_uuid->d[1], + sof_uuid->d[2], sof_uuid->d[3], sof_uuid->d[4], sof_uuid->d[5], + sof_uuid->d[6], sof_uuid->d[7]); + + return drv; +} + +#if defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) +/* + * ipc4_get_fuzzer_drv - resolve a module driver for native_sim fuzz builds. + * + * native_sim fuzz builds have no rimage manifest, so the manifest lookup in + * ipc4_get_comp_drv() can never resolve a driver and every module instance + * verb (INIT_INSTANCE, CONFIG_GET/SET, LARGE_CONFIG, BIND, UNBIND, + * DELETE_INSTANCE) would be unreachable from the fuzzer. + * + * Mirror the IPC3 whitebox hack in posix/ipc.c and treat module_id as a + * 1-based index into the runtime component driver list, counting only + * instantiable module-adapter drivers so the fuzzer's module_id space matches + * what a real signed manifest would expose. + */ +static const struct comp_driver *ipc4_get_fuzzer_drv(uint32_t module_id) +{ + static const struct sof_uuid basefw_reg_uuid = SOF_REG_UUID(basefw); + struct comp_driver_list *dlist = comp_drivers_get(); + struct list_item *iter; + uint32_t idx = 0; + + if (!module_id) + return ipc4_search_for_drv(&basefw_reg_uuid); + + list_for_item(iter, &dlist->list) { + struct comp_driver_info *inf = + container_of(iter, struct comp_driver_info, list); + + /* + * Skip query-only entries (e.g. BaseFW) that cannot be + * instantiated because they have no create op. + */ + if (!inf->drv->ops.create) + continue; + + /* + * A real signed manifest only lists module-adapter modules. + * The internal gateway drivers (SOF_COMP_HOST, SOF_COMP_DAI) + * are registered for IPC3 but are never IPC4 modules: on the + * IPC4 path they receive no params() pass and cannot be + * configured (e.g. their DMA buffer is never allocated). Skip + * non-module-adapter drivers so the fuzzer's module_id space + * matches a real manifest and cannot map to an unconfigurable + * component. + */ + if (inf->drv->type != SOF_COMP_MODULE_ADAPTER) + continue; + + if (++idx == module_id) + return inf->drv; + } + + tr_err(&comp_tr, "no instantiable driver at module_id %u (%u available)", + module_id, idx); + return NULL; +} +#endif /* defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) */ + +/* + * Called from + * - ipc4_get_large_config_module_instance() + * - ipc4_set_large_config_module_instance() + * - comp_new_ipc4() + * all of which are __cold + */ +__cold const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id) +{ + const struct sof_man_fw_desc *desc = NULL; + const struct sof_man_module *mod; + uint32_t entry_index; + + assert_can_be_cold(); + +#ifdef RIMAGE_MANIFEST + desc = (const struct sof_man_fw_desc *)IMR_BOOT_LDR_MANIFEST_BASE; +#elif defined(CONFIG_ARCH_POSIX_LIBFUZZER) + return ipc4_get_fuzzer_drv(module_id); +#else + /* Non-rimage platforms have no component facility yet. + * This needs to move to the platform layer. + */ + return NULL; +#endif + + uint32_t lib_idx = LIB_MANAGER_GET_LIB_ID(module_id); + + if (lib_idx == 0) { + /* module_id 0 is used for base fw which is in entry 1 or 2 */ + if (!module_id) + entry_index = 1 + IS_ENABLED(CONFIG_COLD_STORE_EXECUTE_DRAM); + else + entry_index = module_id; + + if (entry_index >= desc->header.num_module_entries) { + tr_err(&comp_tr, "Error: entry index %d out of bounds.", entry_index); + return NULL; + } + + mod = (const struct sof_man_module *)((const char *)desc + + SOF_MAN_MODULE_OFFSET(entry_index)); + } else { + /* Library index greater than 0 possible only when LIBRARY_MANAGER + * support enabled. + */ +#if CONFIG_LIBRARY_MANAGER + mod = lib_manager_get_module_manifest(module_id); + + if (!mod) { + tr_err(&comp_tr, "Error: Couldn't find loadable module with id %#x.", + module_id); + return NULL; + } +#else + tr_err(&comp_tr, "Error: lib index:%d, while loadable libraries are not supported!!!", + lib_idx); + return NULL; +#endif + } + +#if CONFIG_LIBRARY_MANAGER + /* Check already registered components */ + const struct comp_driver *drv = ipc4_search_for_drv(&mod->uuid); + + if (drv) + return drv; + + /* New module not registered yet. */ + lib_manager_register_module(module_id); +#endif + return ipc4_get_drv(&mod->uuid); +} From 709a32b8fdc6573956db9d3bcde0f54d8ab6f9bf Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 14 Nov 2026 19:50:39 +0000 Subject: [PATCH 4/4] docs: ipc: describe the user/kernel partitioning of IPC4 Add a User/Kernel Partitioning section to the IPC4 README. It shows how a message passes from the kernel dispatcher through the routers and the routing of the LL scheduler in user or kernel space to the user-space IPC thread, and lists which IPC4 messages are handled only in the kernel, in the kernel and in user space, and only in user space. Point the top-level IPC README at it and list the kernel and user halves of the split IPC files. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/ipc/README.md | 12 ++++--- src/ipc/ipc4/README.md | 73 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 76 insertions(+), 9 deletions(-) diff --git a/src/ipc/README.md b/src/ipc/README.md index c240cc54a50e..4e7ea8679663 100644 --- a/src/ipc/README.md +++ b/src/ipc/README.md @@ -1,10 +1,14 @@ # Inter-Processor Communication (IPC) Core Architecture > **Runs in:** Boundary / bridge — the kernel receives and dispatches IPC, then forwards -> module-facing commands to a user thread. The split is visible in the file names -> (`src/kernel/ipc/ipc4/handler-kernel.c` vs `ipc4/handler-user.c`, -> `ipc4/notification-user.c`), and this subsystem hosts the `ipc_msg_*`, `ipc_compound_*` and -> `send_resource_notif` syscalls. See +> module-facing commands to a user thread. The kernel halves of the mixed files are in +> `src/kernel/ipc` (`ipc-common-kernel.c`, `ipc4/handler-kernel.c`, +> `ipc4/handler-user-kernel.c`, `ipc4/handler-ll-user.c`, `ipc4/handler-ll-kernel.c`, +> `ipc4/helper-kernel.c`), the user halves stay here (`ipc-common.c`, `ipc4/handler-user.c`, +> `ipc4/helper.c`, `ipc4/notification-user.c`), and this subsystem hosts the `ipc_msg_*`, +> `ipc_compound_*` and `send_resource_notif` syscalls. Which IPC4 messages are handled in the +> kernel, in user space or in both is listed in +> [IPC4 User/Kernel Partitioning](ipc4/README.md#userkernel-partitioning). See also > [User/kernel split](../include/sof/userspace/README.md). This directory contains the common foundation for all Inter-Processor Communication (IPC) within the Sound Open Firmware (SOF) project. It bridges the gap between hardware mailbox interrupts and the version-specific (IPC3/IPC4) message handlers. diff --git a/src/ipc/ipc4/README.md b/src/ipc/ipc4/README.md index 70ee4d606af2..9dfd96825d90 100644 --- a/src/ipc/ipc4/README.md +++ b/src/ipc/ipc4/README.md @@ -1,11 +1,13 @@ # IPC4 Architecture > **Runs in:** Boundary / bridge — `handler-user.c` and `notification-user.c` run in the user -> thread that services modules. Their kernel counterparts, `handler-kernel.c` (the kernel -> dispatcher), `handler-user-kernel.c` (the global and module message routers, which forward -> commands to the user thread) and `notification.c`, are in `src/kernel/ipc/ipc4`, together -> with the `z_vrfy_`/`z_impl_` pairs for the compound-message, reply and notification -> syscalls. See [User/kernel split](../../include/sof/userspace/README.md). +> thread that services modules. Their kernel counterparts are in `src/kernel/ipc/ipc4`: +> `handler-kernel.c` (the kernel dispatcher), `handler-user-kernel.c` (the global and module +> message routers), `handler-ll-user.c` and `handler-ll-kernel.c` (the routing for the LL +> scheduler in user and in kernel space) and `notification.c`, together with the +> `z_vrfy_`/`z_impl_` pairs for the compound-message, reply and notification syscalls. See +> [User/Kernel Partitioning](#userkernel-partitioning) below and +> [User/kernel split](../../include/sof/userspace/README.md). This directory holds the handlers and topology parsing logic for Inter-Processor Communication Version 4. IPC4 introduces a significantly denser, compound-command structure heavily based around the concept of "pipelines" and dynamic "modules" rather than static DSP stream roles. @@ -41,6 +43,67 @@ graph TD end ``` +## User/Kernel Partitioning + +With the LL scheduler in user space (`CONFIG_SOF_USERSPACE_LL`), each IPC4 message is handled +in up to two places: + +- **Kernel IPC thread**: `handler-kernel.c` receives the message from the host mailbox and + replies. It handles the kernel-only messages itself and passes the others to the routers in + `handler-user-kernel.c`. The routers handle the messages that need kernel mode in all builds. + For the rest they call the `ipc4_route_*()` functions of `handler-ll-user.c`, which do any + privileged part of the message and forward it to the user-space IPC thread of the target + core with `ipc_user_forward_cmd()`. +- **User-space IPC thread**: `ipc_user_thread_dispatch()` in `handler-user.c` runs the handlers + in `handler-user.c` and `helper.c` for the forwarded message. + +Without `CONFIG_SOF_USERSPACE_LL`, `handler-ll-kernel.c` provides the `ipc4_route_*()` +functions instead and calls the same handlers directly, so all messages are handled in the +kernel IPC thread. The split still marks which code must be user-space safe. + +```mermaid +graph TD + Host[Host IPC] --> Disp["handler-kernel.c
ipc4_process_glb/module_message()"] + Disp -->|kernel only| KOnly["library load, D0ix/Dx"] + Disp --> Routers["handler-user-kernel.c
ipc4_user_process_glb/module_message()"] + Routers -->|kernel only| KRouted["CHAIN_DMA, IPC gateway, GDB,
base firmware LARGE_CONFIG"] + Routers --> Route{"ipc4_route_*()"} + Route -->|CONFIG_SOF_USERSPACE_LL| LLUser["handler-ll-user.c
target core, driver lookup,
module load, multicore sequencing"] + Route -->|otherwise| LLKernel["handler-ll-kernel.c
direct call"] + LLUser -->|ipc_user_forward_cmd| UThread["user-space IPC thread
ipc_user_thread_dispatch()"] + UThread --> Handlers["handler-user.c / helper.c handlers"] + LLKernel --> Handlers +``` + +With `CONFIG_SOF_USERSPACE_LL`, the messages are handled as follows: + +| Message | Kernel only | Kernel and user | User | +|---|---|---|---| +| `GLB_LOAD_LIBRARY`, `GLB_LOAD_LIBRARY_PREPARE` | ✓ library manager | | | +| `GLB_CHAIN_DMA` | ✓ | | | +| `GLB_IPCGATEWAY_CMD` | ✓ | | | +| `GLB_ENTER_GDB` | ✓ | | | +| `MOD_SET_D0IX`, `MOD_SET_DX` | ✓ power management | | | +| `MOD_LARGE_CONFIG_GET`/`SET`, base firmware (module_id 0) | ✓ reads the IMR manifest | | | +| `GLB_CREATE_PIPELINE` | | kernel: target core; user: pipeline creation | | +| `GLB_DELETE_PIPELINE` | | kernel: owning core; user: pipeline free | | +| `GLB_SET_PIPELINE_STATE` | | kernel: pipeline list check, multicore prepare/trigger sequencing; user: prepare and trigger | | +| `MOD_INIT_INSTANCE` | | kernel: driver lookup, loadable module setup; user: component creation | | +| `MOD_CONFIG_GET`/`SET` | | | ✓ | +| `MOD_LARGE_CONFIG_GET`/`SET`, modules | | | ✓ | +| `MOD_BIND`, `MOD_UNBIND` | | | ✓ | +| `MOD_DELETE_INSTANCE` | | | ✓ | + +For the messages in the user column, the kernel only selects the target core from the module +and forwards the message unchanged. DP modules and modules on another core are created by the +kernel in `ipc4_init_module_instance()`. With `CONFIG_SOF_USERSPACE_APPLICATION`, the IPCs of a +DP module are then passed on to its DP thread (`scheduler_dp_thread_ipc()`). + +The other global messages (boot config, ROM control, performance measurements, multiple module +load and unload, pipeline state get, context size, save and restore, internal messages and +notifications) and the module restore messages are not implemented and are rejected by the +routers. + ## Processing Flows ### Pipeline State Management (`ipc4_set_pipeline_state`)