Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions app/ll_userspace_overlay.conf
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@ CONFIG_SOF_BOOT_TEST_ALLOWED=n
CONFIG_INTEL_ADSP_MIC_PRIVACY=n
CONFIG_XRUN_NOTIFICATIONS_ENABLE=n

# IPC gateway: the gateway list is kernel-only but is updated by the IPC
# user thread, and the gateway commands are handled in kernel mode on
# buffers that user threads can write
CONFIG_IPC4_GATEWAY=n

# increase L2 allocation for LL userspace and
# max two isolated DP modules
CONFIG_XTENSA_MMU_NUM_L2_TABLES=70
Expand Down
5 changes: 5 additions & 0 deletions src/audio/chain_dma.c
Original file line number Diff line number Diff line change
Expand Up @@ -348,7 +348,10 @@ static int chain_task_start(struct comp_dev *dev)
goto error_task;
}

#ifndef CONFIG_SOF_USERSPACE_LL
/* with LL in user space, the kernel takes it, see ipc4_route_chain_dma() */
pm_policy_state_lock_get(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES);
#endif

return 0;

Expand Down Expand Up @@ -379,7 +382,9 @@ static int chain_task_pause(struct comp_dev *dev)
ret = ret2;

schedule_task_free(&cd->chain_task);
#ifndef CONFIG_SOF_USERSPACE_LL
pm_policy_state_lock_put(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES);
#endif

return ret;
}
Expand Down
10 changes: 10 additions & 0 deletions src/include/ipc4/handler.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,16 @@ int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct i
int ipc4_user_module_load(const struct comp_driver *drv,
const struct ipc4_module_init_instance *mi);

/**
* \brief Process GLB_CHAIN_DMA in any execution context.
* \ingroup ipc4_envelope_api
* @param[in] ipc4 IPC4 message request structure.
* @param[in] drv Chain DMA component driver, looked up by the kernel.
* @return IPC4 status code (IPC4_SUCCESS on success).
*/
int ipc4_process_chain_dma(struct ipc4_message_request *ipc4,
const struct comp_driver *drv);

/**
* \brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context.
* \ingroup ipc4_envelope_api
Expand Down
4 changes: 3 additions & 1 deletion src/include/sof/ipc/topology.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,9 @@ struct comp_driver;
struct comp_dev *comp_new_ipc4_user(struct ipc4_message_request *ipc4,
const struct comp_driver *drv);
#endif
int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma);
const struct comp_driver *ipc4_chain_dma_drv(void);
int ipc4_chain_manager_create(const struct comp_driver *drv,
const struct ipc4_chain_dma *cdma);
int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma);
int ipc4_create_chain_dma(struct ipc *ipc, struct ipc4_chain_dma *cdma);
int ipc4_trigger_chain_dma(struct ipc *ipc, struct ipc4_chain_dma *cdma, bool *delay);
Expand Down
27 changes: 21 additions & 6 deletions src/include/sof/userspace/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,11 +190,15 @@ calls from LL threads: only kernel threads can write objects there.
Objects in the kernel areas:

- RO: the user scheduler list heads and descriptors (`struct schedule_data`, including the
scheduler ops), and the LL allocation context `ipc->ll_alloc`. Kernel code finds the user
scheduler lists through a kernel-only copy of the per-core list heads.
scheduler ops), the LL allocation context `ipc->ll_alloc`, and the drivers of the loadable
modules (`struct comp_driver`, including the component ops), which are never unregistered.
Kernel code finds the user scheduler lists through a kernel-only copy of the per-core list
heads.
- RW: `struct ipc_user`, the IPC data buffer `ipc->comp_data`, the LL scheduler state
(`struct zephyr_ll`), the LL scheduling domain and its Zephyr domain data, the DP scheduler
data, and the LL tasks that kernel threads create at boot together with their task data.
data, the LL tasks that kernel threads create at boot together with their task data, and
the DMA channel arrays. A channel array is allocated on the first `sof_dma_get()` of its
controller and kept in the kernel-only `struct sof_dma_kinfo` for the next one.

DP modules are created by kernel code, but their allocation context, pin format arrays and
configuration buffers are in the module's own vregion, released with it, and the IPC user
Expand All @@ -205,9 +209,20 @@ The LL timer interrupt and the LL thread setup and teardown use kernel-only copi
domain data pointer, the per-core semaphores and the LL threads, not the user-writable domain
data.

The LL user heap keeps its `sys_heap` metadata in user-writable memory. Kernel code is being
moved off it; `CONFIG_SOF_ZEPHYR_LL_HEAP_KERNEL_TRACE` logs the remaining uses by kernel
threads. Kernel code still follows some pointers stored in user-writable memory, for example
Chain DMA messages are forwarded to the IPC user thread like the pipeline messages, so the
chain DMA component is created, run and freed in user space. The kernel only looks up the
driver, and takes the runtime idle PM lock of a started chain itself, tracked in a kernel-only
mask of host DMA channels. The IPC gateway is not supported and is disabled in the LL overlay.

User threads can write the `dma[]` array, so kernel code drives DMA through the `sof_dma_*()`
wrappers, which take the Zephyr device from `struct sof_dma_kinfo` (`sof_dma_kdev()`), and
never calls the Zephyr DMA API on `dma->z_dev`.

The LL user heap keeps its `sys_heap` metadata in user-writable memory. Kernel threads no
longer allocate from it or free to it; `CONFIG_SOF_ZEPHYR_LL_HEAP_KERNEL_TRACE` logs any use
by a kernel thread. User threads still allocate from it through the `sof_heap_alloc()` and
`sof_heap_free()` system calls, so the allocator still runs in supervisor mode on this
metadata. Kernel code still follows some pointers stored in user-writable memory, for example
`task->sch` and the domain `ops`.

## Memory-partition markers
Expand Down
6 changes: 3 additions & 3 deletions src/ipc/ipc4/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ graph TD
Host[Host IPC] --> Disp["handler-kernel.c<br/>ipc4_process_glb/module_message()"]
Disp -->|kernel only| KOnly["library load, D0ix/Dx"]
Disp --> Routers["handler-user-kernel.c<br/>ipc4_user_process_glb/module_message()"]
Routers -->|kernel only| KRouted["CHAIN_DMA, IPC gateway, GDB,<br/>base firmware LARGE_CONFIG"]
Routers -->|kernel only| KRouted["IPC gateway, GDB,<br/>base firmware LARGE_CONFIG"]
Routers --> Route{"ipc4_route_*()"}
Route -->|CONFIG_SOF_USERSPACE_LL| LLUser["handler-ll-user.c<br/>target core, driver lookup,<br/>module load, multicore sequencing"]
Route -->|otherwise| LLKernel["handler-ll-kernel.c<br/>direct call"]
Expand All @@ -80,14 +80,14 @@ With `CONFIG_SOF_USERSPACE_LL`, the messages are handled as follows:
| Message | Kernel only | Kernel and user | User |
|---|---|---|---|
| `GLB_LOAD_LIBRARY`, `GLB_LOAD_LIBRARY_PREPARE` | ✓ library manager | | |
| `GLB_CHAIN_DMA` | ✓ | | |
| `GLB_IPCGATEWAY_CMD` | ✓ | | |
| `GLB_IPCGATEWAY_CMD` | ✓ (the IPC gateway is disabled in the LL overlay) | | |
| `GLB_ENTER_GDB` | ✓ | | |
| `MOD_SET_D0IX`, `MOD_SET_DX` | ✓ power management | | |
| `MOD_LARGE_CONFIG_GET`/`SET`, base firmware (module_id 0) | ✓ reads the IMR manifest | | |
| `GLB_CREATE_PIPELINE` | | kernel: target core; user: pipeline creation | |
| `GLB_DELETE_PIPELINE` | | kernel: owning core; user: pipeline free | |
| `GLB_SET_PIPELINE_STATE` | | kernel: pipeline list check, multicore prepare/trigger sequencing; user: prepare and trigger | |
| `GLB_CHAIN_DMA` | | kernel: driver lookup, runtime idle PM lock of started chains; user: chain DMA creation, state changes and free | |
| `MOD_INIT_INSTANCE` | | kernel: driver lookup, loadable module setup; user: component creation | |
| `MOD_CONFIG_GET`/`SET` | | | ✓ |
| `MOD_LARGE_CONFIG_GET`/`SET`, modules | | | ✓ |
Expand Down
70 changes: 70 additions & 0 deletions src/ipc/ipc4/handler-user.c
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,66 @@ static int ipc4_set_pipeline_state_phase(struct ipc4_message_request *ipc4, uint
}
#endif /* CONFIG_SOF_USERSPACE_LL */

#if CONFIG_COMP_CHAIN_DMA
/**
* @brief Process GLB_CHAIN_DMA in any execution context.
*
* Creates the chain DMA component on the first message for a host DMA
* channel and changes its state.
*
* @param ipc4 Pointer to the IPC4 message request (primary + extension)
* @param drv Chain DMA component driver, looked up by the kernel
* @return IPC4 status code (0 on success)
*/
__cold int ipc4_process_chain_dma(struct ipc4_message_request *ipc4,
const struct comp_driver *drv)
{
struct ipc_comp_dev *cdma_comp;
struct ipc *ipc = ipc_get();
const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4;
int comp_id;
int ret;

assert_can_be_cold();

comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0);
cdma_comp = ipc_get_comp_by_id(ipc, comp_id);

if (!cdma_comp) {
/*
* Nothing to do when the chainDMA is not allocated and asked to
* be freed
*/
if (!cdma->primary.r.allocate && !cdma->primary.r.enable)
return IPC4_SUCCESS;

ret = ipc4_chain_manager_create(drv, cdma);
if (ret < 0)
return IPC4_FAILURE;

cdma_comp = ipc_get_comp_by_id(ipc, comp_id);
if (!cdma_comp)
return IPC4_FAILURE;

ret = ipc4_chain_dma_state(cdma_comp->cd, cdma);
if (ret < 0) {
comp_free(cdma_comp->cd);
list_item_del(&cdma_comp->list);
sof_heap_free(sof_sys_user_heap_get(), cdma_comp);
return IPC4_FAILURE;
}

return IPC4_SUCCESS;
}

ret = ipc4_chain_dma_state(cdma_comp->cd, cdma);
if (ret < 0)
return IPC4_INVALID_CHAIN_STATE_TRANSITION;

return IPC4_SUCCESS;
}
#endif /* CONFIG_COMP_CHAIN_DMA */

/**
* @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context.
*
Expand Down Expand Up @@ -1178,6 +1238,16 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user)
pipe->primary.r.instance_id);
break;
}
#if CONFIG_COMP_CHAIN_DMA
case SOF_IPC4_GLB_CHAIN_DMA: {
/* driver looked up by the kernel, see ipc4_route_chain_dma() */
const struct comp_driver *drv = ipc_user->init_drv;

ipc_user->init_drv = NULL;
result = ipc4_process_chain_dma(&msg, drv);
break;
}
#endif
case SOF_IPC4_GLB_SET_PIPELINE_STATE:
if (ipc_user->ppl_phase)
result = ipc4_set_pipeline_state_phase(&msg, ipc_user->ppl_phase);
Expand Down
68 changes: 68 additions & 0 deletions src/ipc/ipc4/helper.c
Original file line number Diff line number Diff line change
Expand Up @@ -1247,6 +1247,74 @@ int ipc4_find_dma_config(struct ipc_config_dai *dai, uint8_t *data_buffer, uint3
return IPC4_SUCCESS;
}

#if CONFIG_COMP_CHAIN_DMA
/* Only called from ipc4_process_chain_dma(), which is __cold */
__cold int ipc4_chain_manager_create(const struct comp_driver *drv,
const struct ipc4_chain_dma *cdma)
{
struct comp_dev *dev;

assert_can_be_cold();

if (!drv)
return -EINVAL;

dev = drv->ops.create(drv, NULL, cdma);
if (!dev)
return -EINVAL;

/* differentiate instance by unique ids assignment */
const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id
+ IPC4_MAX_MODULE_COUNT, 0);
dev->ipc_config.id = comp_id;
dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT;

return ipc4_add_comp_dev(dev);
}

/* Only called from ipc4_process_chain_dma(), which is __cold */
__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma)
{
const bool allocate = cdma->primary.r.allocate;
const bool enable = cdma->primary.r.enable;
struct ipc *ipc = ipc_get();
struct ipc_comp_dev *icd;
struct list_item *clist, *_tmp;
int ret;

assert_can_be_cold();

if (!dev)
return -EINVAL;

if (allocate) {
if (enable)
ret = comp_trigger(dev, COMP_TRIGGER_START);
else
ret = comp_trigger(dev, COMP_TRIGGER_PAUSE);
} else {
if (enable)
return -EINVAL;

/* remove chain part */
ret = comp_trigger(dev, COMP_TRIGGER_PAUSE);
if (ret < 0)
return ret;

list_for_item_safe(clist, _tmp, &ipc->comp_list) {
icd = container_of(clist, struct ipc_comp_dev, list);
if (icd->cd != dev)
continue;
list_item_del(&icd->list);
sof_heap_free(sof_sys_user_heap_get(), icd);
break;
}
comp_free(dev);
}
return ret;
}
#endif /* CONFIG_COMP_CHAIN_DMA */

/* Unlike the above ipc4_find_dma_config(), this can find multiple DMA configs.
* For example, a UAOL copier may use two DMA channels: one for audio and one
* for clock feedback. This function can only work when all data in data_buffer
Expand Down
11 changes: 11 additions & 0 deletions src/kernel/ipc/ipc4/handler-ll-kernel.c
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,17 @@

LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL);

__cold int ipc4_route_chain_dma(struct ipc4_message_request *ipc4)
{
assert_can_be_cold();

#if CONFIG_COMP_CHAIN_DMA
return ipc4_process_chain_dma(ipc4, ipc4_chain_dma_drv());
#else
return IPC4_UNAVAILABLE;
#endif
}

__cold int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4)
{
struct ipc *ipc = ipc_get();
Expand Down
60 changes: 60 additions & 0 deletions src/kernel/ipc/ipc4/handler-ll-user.c
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
#include <rtos/kernel.h>
#include <sof/lib_manager.h>

#include <zephyr/pm/policy.h>

#include <errno.h>
#include <stdbool.h>
#include <stddef.h>
Expand Down Expand Up @@ -136,6 +138,64 @@ __cold static int ipc4_user_set_pipeline_state_multicore(struct ipc4_message_req
return IPC4_SUCCESS;
}

#if CONFIG_COMP_CHAIN_DMA
/*
* Host DMA channels of the started chain DMAs, kernel-only. A running chain
* DMA blocks the runtime idle state, and the PM policy can only be changed
* in kernel mode, so the lock is taken here and not by the component.
*/
static uint32_t chain_dma_pm_locked;
#endif

__cold int ipc4_route_chain_dma(struct ipc4_message_request *ipc4)
{
assert_can_be_cold();

#if CONFIG_COMP_CHAIN_DMA
const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4;
const bool start = cdma->primary.r.allocate && cdma->primary.r.enable;
const uint32_t chan = BIT(cdma->primary.r.host_dma_id);
bool locked_here = false;
int ret;

/* hold the lock from before the user thread starts the DMA */
if (start && !(chain_dma_pm_locked & chan)) {
chain_dma_pm_locked |= chan;
pm_policy_state_lock_get(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES);
locked_here = true;
}

/*
* The kernel only looks up the driver: the IPC user thread creates
* and controls the component, like for the other pipeline messages.
*/
ipc_user_get()->init_drv = ipc4_chain_dma_drv();
ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, cpu_get_id());

/*
* A timed-out command can still run in the user thread later, so keep
* the lock: it is only released once the DMA is known to be stopped.
* A later pause or free of the chain releases it.
*/
if (ret == -EAGAIN)
return ret;

/*
* A failed start leaves the DMA stopped, unless the chain was started
* before. Pause and free stop a started chain also when they fail.
*/
if ((start && ret && locked_here) ||
(!cdma->primary.r.enable && (chain_dma_pm_locked & chan))) {
chain_dma_pm_locked &= ~chan;
pm_policy_state_lock_put(PM_STATE_RUNTIME_IDLE, PM_ALL_SUBSTATES);
}

return ret;
#else
return IPC4_UNAVAILABLE;
#endif
}

int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4)
{
const struct ipc4_pipeline_create *create =
Expand Down
1 change: 1 addition & 0 deletions src/kernel/ipc/ipc4/handler-route.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4);
int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4);

/* Global messages */
int ipc4_route_chain_dma(struct ipc4_message_request *ipc4);
int ipc4_route_create_pipeline(struct ipc4_message_request *ipc4);
int ipc4_route_delete_pipeline(struct ipc4_message_request *ipc4);
int ipc4_route_set_pipeline_state(struct ipc4_message_request *ipc4);
Expand Down
Loading
Loading