diff --git a/app/ll_userspace_overlay.conf b/app/ll_userspace_overlay.conf
index df7835e24b54..c6f1a753df25 100644
--- a/app/ll_userspace_overlay.conf
+++ b/app/ll_userspace_overlay.conf
@@ -44,6 +44,11 @@ CONFIG_SOF_BOOT_TEST_ALLOWED=n
CONFIG_INTEL_ADSP_MIC_PRIVACY=n
CONFIG_XRUN_NOTIFICATIONS_ENABLE=n
+# IPC gateway: the gateway list is kernel-only but is updated by the IPC
+# user thread, and the gateway commands are handled in kernel mode on
+# buffers that user threads can write
+CONFIG_IPC4_GATEWAY=n
+
# increase L2 allocation for LL userspace and
# max two isolated DP modules
CONFIG_XTENSA_MMU_NUM_L2_TABLES=70
diff --git a/src/audio/chain_dma.c b/src/audio/chain_dma.c
index b4ceb879f594..7ca9b3fb9702 100644
--- a/src/audio/chain_dma.c
+++ b/src/audio/chain_dma.c
@@ -348,7 +348,10 @@ static int chain_task_start(struct comp_dev *dev)
goto error_task;
}
+#ifndef CONFIG_SOF_USERSPACE_LL
+ /* with LL in user space, the kernel takes it, see ipc4_route_chain_dma() */
pm_policy_state_lock_get(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES);
+#endif
return 0;
@@ -379,7 +382,9 @@ static int chain_task_pause(struct comp_dev *dev)
ret = ret2;
schedule_task_free(&cd->chain_task);
+#ifndef CONFIG_SOF_USERSPACE_LL
pm_policy_state_lock_put(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES);
+#endif
return ret;
}
diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h
index 5fc9f749e7fb..66ee9be5728c 100644
--- a/src/include/ipc4/handler.h
+++ b/src/include/ipc4/handler.h
@@ -36,6 +36,16 @@ int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct i
int ipc4_user_module_load(const struct comp_driver *drv,
const struct ipc4_module_init_instance *mi);
+/**
+ * \brief Process GLB_CHAIN_DMA in any execution context.
+ * \ingroup ipc4_envelope_api
+ * @param[in] ipc4 IPC4 message request structure.
+ * @param[in] drv Chain DMA component driver, looked up by the kernel.
+ * @return IPC4 status code (IPC4_SUCCESS on success).
+ */
+int ipc4_process_chain_dma(struct ipc4_message_request *ipc4,
+ const struct comp_driver *drv);
+
/**
* \brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context.
* \ingroup ipc4_envelope_api
diff --git a/src/include/sof/ipc/topology.h b/src/include/sof/ipc/topology.h
index 1711a0fe3a1c..94093cee4e18 100644
--- a/src/include/sof/ipc/topology.h
+++ b/src/include/sof/ipc/topology.h
@@ -59,7 +59,9 @@ struct comp_driver;
struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4,
const struct comp_driver *drv);
#endif
-int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma);
+const struct comp_driver *ipc4_chain_dma_drv(void);
+int ipc4_chain_manager_create(const struct comp_driver *drv,
+ const struct ipc4_chain_dma *cdma);
int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma);
int ipc4_create_chain_dma(struct ipc *ipc, struct ipc4_chain_dma *cdma);
int ipc4_trigger_chain_dma(struct ipc *ipc, struct ipc4_chain_dma *cdma, bool *delay);
diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md
index cee658fc1da3..cf7e378602e8 100644
--- a/src/include/sof/userspace/README.md
+++ b/src/include/sof/userspace/README.md
@@ -190,11 +190,15 @@ calls from LL threads: only kernel threads can write objects there.
Objects in the kernel areas:
- RO: the user scheduler list heads and descriptors (`struct schedule_data`, including the
- scheduler ops), and the LL allocation context `ipc->ll_alloc`. Kernel code finds the user
- scheduler lists through a kernel-only copy of the per-core list heads.
+ scheduler ops), the LL allocation context `ipc->ll_alloc`, and the drivers of the loadable
+ modules (`struct comp_driver`, including the component ops), which are never unregistered.
+ Kernel code finds the user scheduler lists through a kernel-only copy of the per-core list
+ heads.
- RW: `struct ipc_user`, the IPC data buffer `ipc->comp_data`, the LL scheduler state
(`struct zephyr_ll`), the LL scheduling domain and its Zephyr domain data, the DP scheduler
- data, and the LL tasks that kernel threads create at boot together with their task data.
+ data, the LL tasks that kernel threads create at boot together with their task data, and
+ the DMA channel arrays. A channel array is allocated on the first `sof_dma_get()` of its
+ controller and kept in the kernel-only `struct sof_dma_kinfo` for the next one.
DP modules are created by kernel code, but their allocation context, pin format arrays and
configuration buffers are in the module's own vregion, released with it, and the IPC user
@@ -205,9 +209,20 @@ The LL timer interrupt and the LL thread setup and teardown use kernel-only copi
domain data pointer, the per-core semaphores and the LL threads, not the user-writable domain
data.
-The LL user heap keeps its `sys_heap` metadata in user-writable memory. Kernel code is being
-moved off it; `CONFIG_SOF_ZEPHYR_LL_HEAP_KERNEL_TRACE` logs the remaining uses by kernel
-threads. Kernel code still follows some pointers stored in user-writable memory, for example
+Chain DMA messages are forwarded to the IPC user thread like the pipeline messages, so the
+chain DMA component is created, run and freed in user space. The kernel only looks up the
+driver, and takes the runtime idle PM lock of a started chain itself, tracked in a kernel-only
+mask of host DMA channels. The IPC gateway is not supported and is disabled in the LL overlay.
+
+User threads can write the `dma[]` array, so kernel code drives DMA through the `sof_dma_*()`
+wrappers, which take the Zephyr device from `struct sof_dma_kinfo` (`sof_dma_kdev()`), and
+never calls the Zephyr DMA API on `dma->z_dev`.
+
+The LL user heap keeps its `sys_heap` metadata in user-writable memory. Kernel threads no
+longer allocate from it or free to it; `CONFIG_SOF_ZEPHYR_LL_HEAP_KERNEL_TRACE` logs any use
+by a kernel thread. User threads still allocate from it through the `sof_heap_alloc()` and
+`sof_heap_free()` system calls, so the allocator still runs in supervisor mode on this
+metadata. Kernel code still follows some pointers stored in user-writable memory, for example
`task->sch` and the domain `ops`.
## Memory-partition markers
diff --git a/src/ipc/ipc4/README.md b/src/ipc/ipc4/README.md
index 9dfd96825d90..84301d740862 100644
--- a/src/ipc/ipc4/README.md
+++ b/src/ipc/ipc4/README.md
@@ -66,7 +66,7 @@ graph TD
Host[Host IPC] --> Disp["handler-kernel.c
ipc4_process_glb/module_message()"]
Disp -->|kernel only| KOnly["library load, D0ix/Dx"]
Disp --> Routers["handler-user-kernel.c
ipc4_user_process_glb/module_message()"]
- Routers -->|kernel only| KRouted["CHAIN_DMA, IPC gateway, GDB,
base firmware LARGE_CONFIG"]
+ Routers -->|kernel only| KRouted["IPC gateway, GDB,
base firmware LARGE_CONFIG"]
Routers --> Route{"ipc4_route_*()"}
Route -->|CONFIG_SOF_USERSPACE_LL| LLUser["handler-ll-user.c
target core, driver lookup,
module load, multicore sequencing"]
Route -->|otherwise| LLKernel["handler-ll-kernel.c
direct call"]
@@ -80,14 +80,14 @@ With `CONFIG_SOF_USERSPACE_LL`, the messages are handled as follows:
| Message | Kernel only | Kernel and user | User |
|---|---|---|---|
| `GLB_LOAD_LIBRARY`, `GLB_LOAD_LIBRARY_PREPARE` | ✓ library manager | | |
-| `GLB_CHAIN_DMA` | ✓ | | |
-| `GLB_IPCGATEWAY_CMD` | ✓ | | |
+| `GLB_IPCGATEWAY_CMD` | ✓ (the IPC gateway is disabled in the LL overlay) | | |
| `GLB_ENTER_GDB` | ✓ | | |
| `MOD_SET_D0IX`, `MOD_SET_DX` | ✓ power management | | |
| `MOD_LARGE_CONFIG_GET`/`SET`, base firmware (module_id 0) | ✓ reads the IMR manifest | | |
| `GLB_CREATE_PIPELINE` | | kernel: target core; user: pipeline creation | |
| `GLB_DELETE_PIPELINE` | | kernel: owning core; user: pipeline free | |
| `GLB_SET_PIPELINE_STATE` | | kernel: pipeline list check, multicore prepare/trigger sequencing; user: prepare and trigger | |
+| `GLB_CHAIN_DMA` | | kernel: driver lookup, runtime idle PM lock of started chains; user: chain DMA creation, state changes and free | |
| `MOD_INIT_INSTANCE` | | kernel: driver lookup, loadable module setup; user: component creation | |
| `MOD_CONFIG_GET`/`SET` | | | ✓ |
| `MOD_LARGE_CONFIG_GET`/`SET`, modules | | | ✓ |
diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c
index b66bbe57613f..cf5d4bb84df2 100644
--- a/src/ipc/ipc4/handler-user.c
+++ b/src/ipc/ipc4/handler-user.c
@@ -596,6 +596,66 @@ static int ipc4_set_pipeline_state_phase(struct ipc4_message_request *ipc4, uint
}
#endif /* CONFIG_SOF_USERSPACE_LL */
+#if CONFIG_COMP_CHAIN_DMA
+/**
+ * @brief Process GLB_CHAIN_DMA in any execution context.
+ *
+ * Creates the chain DMA component on the first message for a host DMA
+ * channel and changes its state.
+ *
+ * @param ipc4 Pointer to the IPC4 message request (primary + extension)
+ * @param drv Chain DMA component driver, looked up by the kernel
+ * @return IPC4 status code (0 on success)
+ */
+__cold int ipc4_process_chain_dma(struct ipc4_message_request *ipc4,
+ const struct comp_driver *drv)
+{
+ struct ipc_comp_dev *cdma_comp;
+ struct ipc *ipc = ipc_get();
+ const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4;
+ int comp_id;
+ int ret;
+
+ assert_can_be_cold();
+
+ comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0);
+ cdma_comp = ipc_get_comp_by_id(ipc, comp_id);
+
+ if (!cdma_comp) {
+ /*
+ * Nothing to do when the chainDMA is not allocated and asked to
+ * be freed
+ */
+ if (!cdma->primary.r.allocate && !cdma->primary.r.enable)
+ return IPC4_SUCCESS;
+
+ ret = ipc4_chain_manager_create(drv, cdma);
+ if (ret < 0)
+ return IPC4_FAILURE;
+
+ cdma_comp = ipc_get_comp_by_id(ipc, comp_id);
+ if (!cdma_comp)
+ return IPC4_FAILURE;
+
+ ret = ipc4_chain_dma_state(cdma_comp->cd, cdma);
+ if (ret < 0) {
+ comp_free(cdma_comp->cd);
+ list_item_del(&cdma_comp->list);
+ sof_heap_free(sof_sys_user_heap_get(), cdma_comp);
+ return IPC4_FAILURE;
+ }
+
+ return IPC4_SUCCESS;
+ }
+
+ ret = ipc4_chain_dma_state(cdma_comp->cd, cdma);
+ if (ret < 0)
+ return IPC4_INVALID_CHAIN_STATE_TRANSITION;
+
+ return IPC4_SUCCESS;
+}
+#endif /* CONFIG_COMP_CHAIN_DMA */
+
/**
* @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context.
*
@@ -1178,6 +1238,16 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user)
pipe->primary.r.instance_id);
break;
}
+#if CONFIG_COMP_CHAIN_DMA
+ case SOF_IPC4_GLB_CHAIN_DMA: {
+ /* driver looked up by the kernel, see ipc4_route_chain_dma() */
+ const struct comp_driver *drv = ipc_user->init_drv;
+
+ ipc_user->init_drv = NULL;
+ result = ipc4_process_chain_dma(&msg, drv);
+ break;
+ }
+#endif
case SOF_IPC4_GLB_SET_PIPELINE_STATE:
if (ipc_user->ppl_phase)
result = ipc4_set_pipeline_state_phase(&msg, ipc_user->ppl_phase);
diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c
index 7932050a7707..6bf259695403 100644
--- a/src/ipc/ipc4/helper.c
+++ b/src/ipc/ipc4/helper.c
@@ -1247,6 +1247,74 @@ int ipc4_find_dma_config(struct ipc_config_dai *dai, uint8_t *data_buffer, uint3
return IPC4_SUCCESS;
}
+#if CONFIG_COMP_CHAIN_DMA
+/* Only called from ipc4_process_chain_dma(), which is __cold */
+__cold int ipc4_chain_manager_create(const struct comp_driver *drv,
+ const struct ipc4_chain_dma *cdma)
+{
+ struct comp_dev *dev;
+
+ assert_can_be_cold();
+
+ if (!drv)
+ return -EINVAL;
+
+ dev = drv->ops.create(drv, NULL, cdma);
+ if (!dev)
+ return -EINVAL;
+
+ /* differentiate instance by unique ids assignment */
+ const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id
+ + IPC4_MAX_MODULE_COUNT, 0);
+ dev->ipc_config.id = comp_id;
+ dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT;
+
+ return ipc4_add_comp_dev(dev);
+}
+
+/* Only called from ipc4_process_chain_dma(), which is __cold */
+__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma)
+{
+ const bool allocate = cdma->primary.r.allocate;
+ const bool enable = cdma->primary.r.enable;
+ struct ipc *ipc = ipc_get();
+ struct ipc_comp_dev *icd;
+ struct list_item *clist, *_tmp;
+ int ret;
+
+ assert_can_be_cold();
+
+ if (!dev)
+ return -EINVAL;
+
+ if (allocate) {
+ if (enable)
+ ret = comp_trigger(dev, COMP_TRIGGER_START);
+ else
+ ret = comp_trigger(dev, COMP_TRIGGER_PAUSE);
+ } else {
+ if (enable)
+ return -EINVAL;
+
+ /* remove chain part */
+ ret = comp_trigger(dev, COMP_TRIGGER_PAUSE);
+ if (ret < 0)
+ return ret;
+
+ list_for_item_safe(clist, _tmp, &ipc->comp_list) {
+ icd = container_of(clist, struct ipc_comp_dev, list);
+ if (icd->cd != dev)
+ continue;
+ list_item_del(&icd->list);
+ sof_heap_free(sof_sys_user_heap_get(), icd);
+ break;
+ }
+ comp_free(dev);
+ }
+ return ret;
+}
+#endif /* CONFIG_COMP_CHAIN_DMA */
+
/* Unlike the above ipc4_find_dma_config(), this can find multiple DMA configs.
* For example, a UAOL copier may use two DMA channels: one for audio and one
* for clock feedback. This function can only work when all data in data_buffer
diff --git a/src/kernel/ipc/ipc4/handler-ll-kernel.c b/src/kernel/ipc/ipc4/handler-ll-kernel.c
index 7ca33bc24dd2..da0f1b667393 100644
--- a/src/kernel/ipc/ipc4/handler-ll-kernel.c
+++ b/src/kernel/ipc/ipc4/handler-ll-kernel.c
@@ -58,6 +58,17 @@
LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL);
+__cold int ipc4_route_chain_dma(struct ipc4_message_request *ipc4)
+{
+ assert_can_be_cold();
+
+#if CONFIG_COMP_CHAIN_DMA
+ return ipc4_process_chain_dma(ipc4, ipc4_chain_dma_drv());
+#else
+ return IPC4_UNAVAILABLE;
+#endif
+}
+
__cold int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4)
{
struct ipc *ipc = ipc_get();
diff --git a/src/kernel/ipc/ipc4/handler-ll-user.c b/src/kernel/ipc/ipc4/handler-ll-user.c
index 31e249fce0c0..e2f0c32ab744 100644
--- a/src/kernel/ipc/ipc4/handler-ll-user.c
+++ b/src/kernel/ipc/ipc4/handler-ll-user.c
@@ -39,6 +39,8 @@
#include
#include
+#include
+
#include
#include
#include
@@ -136,6 +138,64 @@ __cold static int ipc4_user_set_pipeline_state_multicore(struct ipc4_message_req
return IPC4_SUCCESS;
}
+#if CONFIG_COMP_CHAIN_DMA
+/*
+ * Host DMA channels of the started chain DMAs, kernel-only. A running chain
+ * DMA blocks the runtime idle state, and the PM policy can only be changed
+ * in kernel mode, so the lock is taken here and not by the component.
+ */
+static uint32_t chain_dma_pm_locked;
+#endif
+
+__cold int ipc4_route_chain_dma(struct ipc4_message_request *ipc4)
+{
+ assert_can_be_cold();
+
+#if CONFIG_COMP_CHAIN_DMA
+ const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4;
+ const bool start = cdma->primary.r.allocate && cdma->primary.r.enable;
+ const uint32_t chan = BIT(cdma->primary.r.host_dma_id);
+ bool locked_here = false;
+ int ret;
+
+ /* hold the lock from before the user thread starts the DMA */
+ if (start && !(chain_dma_pm_locked & chan)) {
+ chain_dma_pm_locked |= chan;
+ pm_policy_state_lock_get(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES);
+ locked_here = true;
+ }
+
+ /*
+ * The kernel only looks up the driver: the IPC user thread creates
+ * and controls the component, like for the other pipeline messages.
+ */
+ ipc_user_get()->init_drv = ipc4_chain_dma_drv();
+ ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, cpu_get_id());
+
+ /*
+ * A timed-out command can still run in the user thread later, so keep
+ * the lock: it is only released once the DMA is known to be stopped.
+ * A later pause or free of the chain releases it.
+ */
+ if (ret == -EAGAIN)
+ return ret;
+
+ /*
+ * A failed start leaves the DMA stopped, unless the chain was started
+ * before. Pause and free stop a started chain also when they fail.
+ */
+ if ((start && ret && locked_here) ||
+ (!cdma->primary.r.enable && (chain_dma_pm_locked & chan))) {
+ chain_dma_pm_locked &= ~chan;
+ pm_policy_state_lock_put(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES);
+ }
+
+ return ret;
+#else
+ return IPC4_UNAVAILABLE;
+#endif
+}
+
int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4)
{
const struct ipc4_pipeline_create *create =
diff --git a/src/kernel/ipc/ipc4/handler-route.h b/src/kernel/ipc/ipc4/handler-route.h
index 75d80d0cea4c..609856d627ed 100644
--- a/src/kernel/ipc/ipc4/handler-route.h
+++ b/src/kernel/ipc/ipc4/handler-route.h
@@ -36,6 +36,7 @@ int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4);
int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4);
/* Global messages */
+int ipc4_route_chain_dma(struct ipc4_message_request *ipc4);
int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4);
int ipc4_route_delete_pipeline(struct ipc4_message_request *ipc4);
int ipc4_route_set_pipeline_state(struct ipc4_message_request *ipc4);
diff --git a/src/kernel/ipc/ipc4/handler-user-kernel.c b/src/kernel/ipc/ipc4/handler-user-kernel.c
index dc3762ebe080..076976ac3991 100644
--- a/src/kernel/ipc/ipc4/handler-user-kernel.c
+++ b/src/kernel/ipc/ipc4/handler-user-kernel.c
@@ -56,11 +56,11 @@
* messages to ipc4_user_process_module_message() here. These routers, also
* running in the kernel IPC thread, are the boundary between kernel and user
* space for IPC4:
- * - some messages are handled here, in kernel mode, in all builds: CHAIN_DMA,
- * the IPC gateway, GDB, and LARGE_CONFIG get and set of the base firmware
+ * - some messages are handled here, in kernel mode, in all builds: the IPC
+ * gateway, GDB, and LARGE_CONFIG get and set of the base firmware
* (module_id 0), which reads the IMR manifest;
- * - the messages that create, configure, connect and delete pipelines and
- * modules are passed to the ipc4_route_*() functions. With the LL scheduler
+ * - CHAIN_DMA and the messages that create, configure, connect and delete
+ * pipelines and modules are passed to the ipc4_route_*() functions. With the LL scheduler
* in user space (CONFIG_SOF_USERSPACE_LL) these are in handler-ll-user.c and
* forward the message to the user-space IPC thread, which runs the handlers
* in src/ipc/ipc4/handler-user.c. Otherwise they are in handler-ll-kernel.c
@@ -75,59 +75,6 @@ LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL);
/* fw sends a fw ipc message to send the status of the last host ipc message */
static struct ipc_msg *msg_reply;
-__cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4)
-{
- assert_can_be_cold();
-
-#if CONFIG_COMP_CHAIN_DMA
- struct ipc_comp_dev *cdma_comp;
- struct ipc *ipc = ipc_get();
- const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4;
- int comp_id;
- int ret;
-
- comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0);
- cdma_comp = ipc_get_comp_by_id(ipc, comp_id);
-
- if (!cdma_comp) {
- /*
- * Nothing to do when the chainDMA is not allocated and asked to
- * be freed
- */
- if (!cdma->primary.r.allocate && !cdma->primary.r.enable)
- return IPC4_SUCCESS;
-
- ret = ipc4_chain_manager_create(cdma);
- if (ret < 0)
- return IPC4_FAILURE;
-
- cdma_comp = ipc_get_comp_by_id(ipc, comp_id);
- if (!cdma_comp) {
- return IPC4_FAILURE;
- }
-
- ret = ipc4_chain_dma_state(cdma_comp->cd, cdma);
- if (ret < 0) {
- comp_free(cdma_comp->cd);
- list_item_del(&cdma_comp->list);
- sof_heap_free(sof_sys_user_heap_get(), cdma_comp);
- return IPC4_FAILURE;
- }
-
- return IPC4_SUCCESS;
- }
-
- ret = ipc4_chain_dma_state(cdma_comp->cd, cdma);
- if (ret < 0)
- return IPC4_INVALID_CHAIN_STATE_TRANSITION;
-
- return IPC4_SUCCESS;
-#else /* CONFIG_COMP_CHAIN_DMA */
- return IPC4_UNAVAILABLE;
-#endif /* CONFIG_COMP_CHAIN_DMA */
-}
-
-
__cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4)
{
assert_can_be_cold();
@@ -186,7 +133,7 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4,
break;
case SOF_IPC4_GLB_CHAIN_DMA:
- ret = ipc4_process_chain_dma(ipc4);
+ ret = ipc4_route_chain_dma(ipc4);
break;
/* pipeline settings */
diff --git a/src/kernel/ipc/ipc4/helper-kernel.c b/src/kernel/ipc/ipc4/helper-kernel.c
index 6f0402bd573a..65be0878824b 100644
--- a/src/kernel/ipc/ipc4/helper-kernel.c
+++ b/src/kernel/ipc/ipc4/helper-kernel.c
@@ -202,72 +202,14 @@ __cold int ipc_comp_free_remote(struct comp_dev *dev)
}
#if CONFIG_COMP_CHAIN_DMA
-/* Only called from ipc4_process_chain_dma(), which is __cold */
-__cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma)
+/* Only called from the chain DMA message routers, which are __cold */
+__cold const struct comp_driver *ipc4_chain_dma_drv(void)
{
const struct sof_uuid uuid = SOF_REG_UUID(chain_dma);
- const struct comp_driver *drv;
- struct comp_dev *dev;
assert_can_be_cold();
- drv = ipc4_get_drv(&uuid);
- if (!drv)
- return -EINVAL;
-
- dev = drv->ops.create(drv, NULL, cdma);
- if (!dev)
- return -EINVAL;
-
- /* differentiate instance by unique ids assignment */
- const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id
- + IPC4_MAX_MODULE_COUNT, 0);
- dev->ipc_config.id = comp_id;
- dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT;
-
- return ipc4_add_comp_dev(dev);
-}
-
-/* Only called from ipc4_process_chain_dma(), which is __cold */
-__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma)
-{
- const bool allocate = cdma->primary.r.allocate;
- const bool enable = cdma->primary.r.enable;
- struct ipc *ipc = ipc_get();
- struct ipc_comp_dev *icd;
- struct list_item *clist, *_tmp;
- int ret;
-
- assert_can_be_cold();
-
- if (!dev)
- return -EINVAL;
-
- if (allocate) {
- if (enable)
- ret = comp_trigger(dev, COMP_TRIGGER_START);
- else
- ret = comp_trigger(dev, COMP_TRIGGER_PAUSE);
- } else {
- if (enable)
- return -EINVAL;
-
- /* remove chain part */
- ret = comp_trigger(dev, COMP_TRIGGER_PAUSE);
- if (ret < 0)
- return ret;
-
- list_for_item_safe(clist, _tmp, &ipc->comp_list) {
- icd = container_of(clist, struct ipc_comp_dev, list);
- if (icd->cd != dev)
- continue;
- list_item_del(&icd->list);
- sof_heap_free(sof_sys_user_heap_get(), icd);
- break;
- }
- comp_free(dev);
- }
- return ret;
+ return ipc4_get_drv(&uuid);
}
#endif /* CONFIG_COMP_CHAIN_DMA */
diff --git a/src/kernel/lib/dma-kernel.c b/src/kernel/lib/dma-kernel.c
index 422c89f20d2c..32e76a433395 100644
--- a/src/kernel/lib/dma-kernel.c
+++ b/src/kernel/lib/dma-kernel.c
@@ -17,7 +17,7 @@
#include
#include
#include
-#include /* zephyr_ll_user_heap_verify() */
+#include /* zephyr_ll_karea_alloc() */
#include
#include
#include
@@ -188,7 +188,9 @@ __cold void z_impl_sof_dma_put(struct sof_dma *dma)
--*dma_sref(dma);
dma->sref = *dma_sref(dma);
if (!*dma_sref(dma)) {
-#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA
+#if CONFIG_SOF_USERSPACE_LL
+ /* kinfo->chan is in a kernel area: kept for the next dma_init() */
+#elif defined(CONFIG_SOF_USERSPACE_INTERFACE_DMA)
struct sof_dma_kinfo *kinfo = sof_dma_kinfo_get(dma);
sof_heap_free(kinfo->heap, kinfo->chan);
@@ -218,24 +220,27 @@ __cold static int dma_init(struct sof_dma *dma)
assert_can_be_cold();
-#if CONFIG_SOF_USERSPACE_INTERFACE_DMA && CONFIG_SOF_USERSPACE_LL
+#if CONFIG_SOF_USERSPACE_LL
/*
- * Called in supervisor mode from the sof_dma_get() system call. The
- * heap pointer comes from user-writable memory, so check that it is
- * the LL user heap before running the allocator on it.
+ * Called in supervisor mode, also from the sof_dma_get() system call.
+ * The kernel does not allocate from the LL user heap, whose metadata
+ * user threads can write. Take the array from the kernel RW area of
+ * the LL region instead, which user threads can access too. Kernel
+ * area memory is never freed, so the array of a controller is
+ * allocated on first use and reused after that.
*/
- if (!zephyr_ll_user_heap_verify(heap)) {
- tr_err(&dma_tr, "dma %d: invalid channel heap %p", dma->plat_data.id, heap);
- return -EINVAL;
- }
-#endif
-
+ chans = sof_dma_kinfo_get(dma)->chan;
+ if (!chans)
+ chans = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RW, SOF_MEM_FLAG_COHERENT,
+ sizeof(struct dma_chan_data) * channels, 0);
+#else
/*
* allocate dma channels, use a local pointer until the array is
* initialized: dma->chan is user-writable and must not be read back
*/
chans = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT,
sizeof(struct dma_chan_data) * channels, 0);
+#endif
if (!chans) {
tr_err(&dma_tr, "dma %d allocation of channels failed",
dma->plat_data.id);
diff --git a/src/kernel/library_manager/lib_manager.c b/src/kernel/library_manager/lib_manager.c
index b34dbab56541..49a004c5ee23 100644
--- a/src/kernel/library_manager/lib_manager.c
+++ b/src/kernel/library_manager/lib_manager.c
@@ -29,6 +29,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -63,7 +64,7 @@ DECLARE_TR_CTX(lib_manager_tr, SOF_UUID(lib_manager_uuid), LOG_LEVEL_INFO);
struct lib_manager_dma_ext {
struct sof_dma *dma;
- struct dma_chan_data *chan;
+ uint32_t chan; /**< DMA channel index */
uintptr_t dma_addr; /**< buffer start pointer */
uint32_t addr_align;
};
@@ -939,8 +940,20 @@ __cold int lib_manager_register_module(const uint32_t component_id)
drv_heap = sof_sys_user_heap_get();
#endif /* CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP */
+#if CONFIG_SOF_USERSPACE_LL && !CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP
+ /*
+ * LL user threads read the driver, but the kernel calls its ops, for
+ * example to create DP modules, so user threads must not be able to
+ * write it. The kernel does not allocate from the LL user heap
+ * either. Loadable drivers are never unregistered, so the kernel RO
+ * area of the LL region, which is never freed, fits.
+ */
+ drv = zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RO, SOF_MEM_FLAG_COHERENT,
+ sizeof(struct comp_driver), 0);
+#else
drv = sof_heap_alloc(drv_heap, SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT,
sizeof(struct comp_driver), 0);
+#endif
if (!drv) {
tr_err(&lib_manager_tr, "failed to allocate comp_driver");
goto cleanup;
@@ -961,8 +974,14 @@ __cold int lib_manager_register_module(const uint32_t component_id)
cleanup:
if (ret < 0) {
rfree(new_drv_info);
+ /* with LL, drv is in a kernel area, which is never freed */
+#if !CONFIG_SOF_USERSPACE_LL || CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP
sof_heap_free(drv_heap, drv);
+#endif
+#if CONFIG_SOF_USERSPACE_USE_DRIVER_HEAP
+ /* only a driver heap of its own, never the shared user heap */
module_driver_heap_remove(drv_heap);
+#endif
}
return ret;
@@ -1010,13 +1029,15 @@ __cold static int lib_manager_dma_init(struct lib_manager_dma_ext *dma_ext, uint
return -ENODEV;
}
- chan_index = dma_request_channel(dma_ext->dma->z_dev, &dma_id);
+ /*
+ * use the sof_dma API: user threads can write dma[], so the Zephyr
+ * DMA API must not be called on dma->z_dev here
+ */
+ chan_index = sof_dma_request_channel(dma_ext->dma, dma_id);
if (chan_index < 0)
return chan_index;
- dma_ext->chan = &dma_ext->dma->chan[chan_index];
- if (!dma_ext->chan)
- return -EINVAL;
+ dma_ext->chan = chan_index;
return 0;
}
@@ -1030,9 +1051,7 @@ __cold static int lib_manager_dma_init(struct lib_manager_dma_ext *dma_ext, uint
static int lib_manager_dma_deinit(struct lib_manager_dma_ext *dma_ext, uint32_t dma_id)
{
if (dma_ext->dma) {
- if (dma_ext->dma->z_dev)
- dma_release_channel(dma_ext->dma->z_dev, dma_id);
-
+ sof_dma_release_channel(dma_ext->dma, dma_id);
sof_dma_put(dma_ext->dma);
}
return 0;
@@ -1051,8 +1070,7 @@ __cold static int lib_manager_load_data_from_host(struct lib_manager_dma_ext *dm
timeout += sof_cycle_get_64();
for (;;) {
- ret = dma_get_status(dma_ext->chan->dma->z_dev,
- dma_ext->chan->index, &stat);
+ ret = sof_dma_get_status(dma_ext->dma, dma_ext->chan, &stat);
if (ret < 0 || stat.pending_length >= size)
return ret;
@@ -1089,7 +1107,7 @@ __cold static int lib_manager_store_data(struct lib_manager_dma_ext *dma_ext,
memcpy_s((__sparse_force uint8_t *)dst_addr + copied_bytes, bytes_to_copy,
(void *)dma_ext->dma_addr, bytes_to_copy);
copied_bytes += bytes_to_copy;
- dma_reload(dma_ext->chan->dma->z_dev, dma_ext->chan->index, 0, 0, bytes_to_copy);
+ sof_dma_reload(dma_ext->dma, dma_ext->chan, bytes_to_copy);
}
return 0;
@@ -1232,8 +1250,8 @@ __cold static int lib_manager_setup(uint32_t dma_id)
if (ret < 0)
goto err_dma_init;
- ret = dma_get_attribute(dma_ext->dma->z_dev, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT,
- &dma_ext->addr_align);
+ ret = sof_dma_get_attribute(dma_ext->dma, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT,
+ &dma_ext->addr_align);
if (ret < 0)
goto err_dma_init;
@@ -1253,11 +1271,11 @@ __cold static int lib_manager_setup(uint32_t dma_id)
goto err_dma_buffer;
#endif
- ret = dma_config(dma_ext->chan->dma->z_dev, dma_ext->chan->index, &config);
+ ret = sof_dma_config(dma_ext->dma, dma_ext->chan, &config);
if (ret < 0)
goto err_dma;
- ret = dma_start(dma_ext->chan->dma->z_dev, dma_ext->chan->index);
+ ret = sof_dma_start(dma_ext->dma, dma_ext->chan);
if (ret < 0)
goto err_dma;
@@ -1339,7 +1357,7 @@ __cold int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t t
#endif /* CONFIG_LIBRARY_AUTH_SUPPORT */
stop_dma:
- ret2 = dma_stop(dma_ext->chan->dma->z_dev, dma_ext->chan->index);
+ ret2 = sof_dma_stop(dma_ext->dma, dma_ext->chan);
if (ret2 < 0) {
tr_err(&lib_manager_tr, "error stopping DMA: %d", ret);
if (!ret)
diff --git a/zephyr/Kconfig b/zephyr/Kconfig
index 55ddc6036936..a29d11d3a76a 100644
--- a/zephyr/Kconfig
+++ b/zephyr/Kconfig
@@ -192,12 +192,14 @@ config SOF_ZEPHYR_LL_KAREA_RW_SIZE
config SOF_ZEPHYR_LL_KAREA_RO_SIZE
hex "Size of the kernel-managed read-only area for LL user threads"
- default 0x1000
+ default 0x2000
depends on SOF_USERSPACE_LL
help
Size of the area in the LL user memory region from which kernel
code allocates objects that the LL and IPC user threads may only
- read. The area is mapped read-only into the LL memory domain, so
+ read, such as the scheduler descriptors and the drivers of the
+ loadable modules. Each loadable module in use takes about 128 bytes.
+ The area is mapped read-only into the LL memory domain, so
kernel code running in that domain, for example in a system call,
cannot write it either. The area is taken from
SOF_ZEPHYR_SYS_USER_HEAP_SIZE. Must be a multiple of the MMU page
diff --git a/zephyr/include/sof/lib/sof_dma.h b/zephyr/include/sof/lib/sof_dma.h
index 86f36c3143a2..b4aee271e70c 100644
--- a/zephyr/include/sof/lib/sof_dma.h
+++ b/zephyr/include/sof/lib/sof_dma.h
@@ -55,58 +55,77 @@ __syscall int sof_dma_suspend(struct sof_dma *dma, uint32_t channel);
__syscall int sof_dma_resume(struct sof_dma *dma, uint32_t channel);
+#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA
+/**
+ * Return the Zephyr device of @p dma from its kernel-only state.
+ *
+ * User threads can write dma[], so kernel code must not call the Zephyr
+ * DMA API on dma->z_dev. @p dma is not dereferenced. Kernel context only.
+ */
+const struct device *sof_dma_kdev(const struct sof_dma *dma);
+#else
+static inline const struct device *sof_dma_kdev(const struct sof_dma *dma)
+{
+ return dma->z_dev;
+}
+#endif
+
+/*
+ * The z_impl_sof_dma_*() functions are only called in kernel context,
+ * the system call validators use the Zephyr DMA API directly
+ */
static inline int z_impl_sof_dma_get_attribute(struct sof_dma *dma, uint32_t type, uint32_t *value)
{
- return dma_get_attribute(dma->z_dev, type, value);
+ return dma_get_attribute(sof_dma_kdev(dma), type, value);
}
static inline int z_impl_sof_dma_request_channel(struct sof_dma *dma, uint32_t stream_tag)
{
- return dma_request_channel(dma->z_dev, &stream_tag);
+ return dma_request_channel(sof_dma_kdev(dma), &stream_tag);
}
static inline void z_impl_sof_dma_release_channel(struct sof_dma *dma,
uint32_t channel)
{
- dma_release_channel(dma->z_dev, channel);
+ dma_release_channel(sof_dma_kdev(dma), channel);
}
static inline int z_impl_sof_dma_config(struct sof_dma *dma, uint32_t channel,
struct dma_config *config)
{
- return dma_config(dma->z_dev, channel, config);
+ return dma_config(sof_dma_kdev(dma), channel, config);
}
static inline int z_impl_sof_dma_start(struct sof_dma *dma, uint32_t channel)
{
- return dma_start(dma->z_dev, channel);
+ return dma_start(sof_dma_kdev(dma), channel);
}
static inline int z_impl_sof_dma_stop(struct sof_dma *dma, uint32_t channel)
{
- return dma_stop(dma->z_dev, channel);
+ return dma_stop(sof_dma_kdev(dma), channel);
}
static inline int z_impl_sof_dma_get_status(struct sof_dma *dma, uint32_t channel,
struct dma_status *stat)
{
- return dma_get_status(dma->z_dev, channel, stat);
+ return dma_get_status(sof_dma_kdev(dma), channel, stat);
}
static inline int z_impl_sof_dma_reload(struct sof_dma *dma, uint32_t channel, size_t size)
{
- return dma_reload(dma->z_dev, channel, 0, 0, size);
+ return dma_reload(sof_dma_kdev(dma), channel, 0, 0, size);
}
static inline int z_impl_sof_dma_suspend(struct sof_dma *dma, uint32_t channel)
{
- return dma_suspend(dma->z_dev, channel);
+ return dma_suspend(sof_dma_kdev(dma), channel);
}
static inline int z_impl_sof_dma_resume(struct sof_dma *dma, uint32_t channel)
{
- return dma_resume(dma->z_dev, channel);
+ return dma_resume(sof_dma_kdev(dma), channel);
}
#ifdef CONFIG_SOF_USERSPACE_INTERFACE_DMA
@@ -134,7 +153,12 @@ struct sof_dma_kinfo {
struct k_spinlock lock; /**< protects sref, heap and chan */
int sref; /**< reference count, under lock */
struct k_heap *heap; /**< heap the channel array came from */
- struct dma_chan_data *chan; /**< channel array to free on last put */
+ /**
+ * channel array, freed on the last put, or with
+ * CONFIG_SOF_USERSPACE_LL kept in the kernel RW area of the LL
+ * region for the next get
+ */
+ struct dma_chan_data *chan;
};
/**
diff --git a/zephyr/lib/dma.c b/zephyr/lib/dma.c
index d9a9642e830b..60d65f582fe2 100644
--- a/zephyr/lib/dma.c
+++ b/zephyr/lib/dma.c
@@ -319,6 +319,16 @@ struct sof_dma_kinfo *sof_dma_kinfo_get(const struct sof_dma *d)
return &dma_kinfo[offset / sizeof(dma[0])];
}
+
+const struct device *sof_dma_kdev(const struct sof_dma *d)
+{
+ const struct sof_dma_kinfo *kinfo = sof_dma_kinfo_get(d);
+
+ /* kernel callers only pass controllers from sof_dma_get() */
+ __ASSERT(kinfo, "%p is not a DMA controller", d);
+
+ return kinfo ? kinfo->z_dev : NULL;
+}
#endif
/* Initialize all platform DMAC's */
diff --git a/zephyr/test/userspace/test_ll_karea.c b/zephyr/test/userspace/test_ll_karea.c
index f7a3221381a4..f46bac9926cf 100644
--- a/zephyr/test/userspace/test_ll_karea.c
+++ b/zephyr/test/userspace/test_ll_karea.c
@@ -7,11 +7,13 @@
* Tests for the kernel-managed areas of the LL memory region: LL user
* threads can write objects in the RW area, only read objects in the RO
* area, such as the user scheduler descriptors, and the kernel allocator
- * state is not in user memory.
+ * state is not in user memory. Also checks the kernel DMA objects that
+ * live in the RW area and the kernel DMA wrappers.
*/
#include
#include
+#include
#include
#include
@@ -174,6 +176,55 @@ ZTEST(userspace_ll_karea, contains_and_limits)
zassert_is_null(zephyr_ll_karea_alloc(ZEPHYR_LL_KAREA_RO, 0, SIZE_MAX, 0));
}
+ZTEST(userspace_ll_karea, dma_channels_in_rw_area)
+{
+ struct sof_dma *dma = sof_dma_get(SOF_DMA_DIR_HMEM_TO_LMEM, 0, SOF_DMA_DEV_HOST,
+ SOF_DMA_ACCESS_SHARED);
+ struct sof_dma_kinfo *kinfo;
+ struct dma_chan_data *chan;
+ int i;
+
+ zassert_not_null(dma, "no host DMA");
+ kinfo = sof_dma_kinfo_get(dma);
+ zassert_not_null(kinfo);
+ chan = kinfo->chan;
+ zassert_true(zephyr_ll_karea_contains(ZEPHYR_LL_KAREA_RW, chan,
+ kinfo->channels * sizeof(*chan)),
+ "channel array %p not in the RW area", chan);
+ sof_dma_put(dma);
+
+ /* the array is kept after the last put and reused */
+ for (i = 0; i < 4; i++) {
+ dma = sof_dma_get(SOF_DMA_DIR_HMEM_TO_LMEM, 0, SOF_DMA_DEV_HOST,
+ SOF_DMA_ACCESS_SHARED);
+ zassert_not_null(dma, "no host DMA");
+ zassert_equal_ptr(sof_dma_kinfo_get(dma)->chan, chan, "channel array reallocated");
+ sof_dma_put(dma);
+ }
+}
+
+ZTEST(userspace_ll_karea, dma_kernel_device)
+{
+ struct sof_dma *dma = sof_dma_get(SOF_DMA_DIR_HMEM_TO_LMEM, 0, SOF_DMA_DEV_HOST,
+ SOF_DMA_ACCESS_SHARED);
+ const struct device *z_dev;
+ uint32_t align = 0;
+ int ret;
+
+ zassert_not_null(dma, "no host DMA");
+ zassert_equal_ptr(sof_dma_kdev(dma), sof_dma_kinfo_get(dma)->z_dev);
+
+ /* user threads can write dma[]: the kernel wrappers must not use it */
+ z_dev = dma->z_dev;
+ dma->z_dev = (const struct device *)&kernel_only_word;
+ ret = sof_dma_get_attribute(dma, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, &align);
+ dma->z_dev = z_dev;
+
+ zassert_equal(ret, 0, "get_attribute failed: %d", ret);
+ zassert_not_equal(align, 0);
+ sof_dma_put(dma);
+}
+
ZTEST_SUITE(userspace_ll_karea, sof_boot_test_standalone_predicate, ll_karea_setup,
NULL, NULL, NULL);