From 1c0e3c9b576314992a8c8bc17c2fe9d132f5afc1 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Tue, 7 Jul 2026 12:45:00 +0200 Subject: [PATCH 01/12] schedule: userspace: DP: make scheduler_dp_ll_tick() a syscall scheduler_dp_ll_tick(() has to recalculate DP deadlines and reschedule DP threads. Make it a syscall to be able to call it from the userspace LL scheduler. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/dp_schedule.h | 6 +++++- src/schedule/zephyr_dp_schedule.c | 11 ++++++++++- zephyr/CMakeLists.txt | 1 + 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 1b55405d0fb1..9bfd9b06978a 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -78,7 +78,11 @@ int scheduler_dp_task_init(struct task **task, uint16_t core, size_t stack_size, uint32_t options); -void scheduler_dp_ll_tick(void); + +#if defined(__ZEPHYR__) && CONFIG_SOF_FULL_ZEPHYR_APPLICATION +__syscall void scheduler_dp_ll_tick(void); +#include +#endif /** * \brief Extract information about scheduler's tasks diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index fa8bbd285791..b6e09cc2c293 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -224,7 +224,7 @@ static enum task_state scheduler_dp_ll_tick_dummy(void *data) * needed 1.2ms for processing - but the example would be too complicated) */ -void scheduler_dp_ll_tick(void) +void z_impl_scheduler_dp_ll_tick(void) { unsigned int lock_key; struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); @@ -240,6 +240,15 @@ void scheduler_dp_ll_tick(void) scheduler_dp_unlock(lock_key); } +#ifdef CONFIG_USERSPACE +#include +void z_vrfy_scheduler_dp_ll_tick(void) +{ + z_impl_scheduler_dp_ll_tick(); +} +#include +#endif + #if CONFIG_SOF_USERSPACE_APPLICATION static int scheduler_dp_task_cancel(void *data, struct task *task) { diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index c8f49c1f07cb..d036eaeede90 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -625,6 +625,7 @@ zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/cpu.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/audio/module_adapter/module/generic.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/fast-get.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/ipc/ipc_reply.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/dp_schedule.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/schedule/ll_schedule_domain.h) zephyr_syscall_header(${SOF_SRC_PATH}/include/ipc4/handler.h) zephyr_syscall_header(include/rtos/alloc.h) From 5981ff2cbf0482a3fae88dca786d5a1cc7fbe3a9 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 28 Aug 2026 13:55:49 +0200 Subject: [PATCH 02/12] schedule: dp: userspace: make scheduler_dp_internal_free() a syscall Make scheduler_dp_internal_free() a syscall in the "application" DP implementation. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/dp_schedule.h | 1 + src/schedule/zephyr_dp_schedule.h | 4 -- src/schedule/zephyr_dp_schedule_application.c | 47 ++++++++++++++++++- src/schedule/zephyr_dp_schedule_thread.c | 16 ++++++- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/include/sof/schedule/dp_schedule.h b/src/include/sof/schedule/dp_schedule.h index 9bfd9b06978a..449bf2151872 100644 --- a/src/include/sof/schedule/dp_schedule.h +++ b/src/include/sof/schedule/dp_schedule.h @@ -80,6 +80,7 @@ int scheduler_dp_task_init(struct task **task, uint32_t options); #if defined(__ZEPHYR__) && CONFIG_SOF_FULL_ZEPHYR_APPLICATION +__syscall void scheduler_dp_internal_free(struct task *task); __syscall void scheduler_dp_ll_tick(void); #include #endif diff --git a/src/schedule/zephyr_dp_schedule.h b/src/schedule/zephyr_dp_schedule.h index 694bb541f87e..2119e44c9dfc 100644 --- a/src/schedule/zephyr_dp_schedule.h +++ b/src/schedule/zephyr_dp_schedule.h @@ -57,7 +57,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3); unsigned int scheduler_dp_lock(uint16_t core); void scheduler_dp_unlock(unsigned int key); void scheduler_dp_grant(k_tid_t thread_id, uint16_t core); -int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, - const struct task_ops *ops, struct processing_module *mod, - uint16_t core, size_t stack_size, uint32_t options); -void scheduler_dp_internal_free(struct task *task); diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 547823ca6e6f..3183ca95aef3 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -400,7 +401,7 @@ struct scheduler_dp_task_memory { struct ipc4_flat flat; }; -void scheduler_dp_internal_free(struct task *task) +void z_impl_scheduler_dp_internal_free(struct task *task) { struct task_dp_pdata *pdata = task->priv_data; @@ -616,3 +617,47 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, mod_free(mod, task_memory); return ret; } + +#ifdef CONFIG_USERSPACE +#include + +static void scheduler_dp_mod_vrfy(struct processing_module *mod) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); + K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); + + struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc, sizeof(*alloc))); + if (alloc->heap) { + size_t h_size = 0; + uintptr_t h_start; + + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); + } + if (alloc->vreg) { + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(alloc->vreg, &vr_size, &vr_start); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vr_start, vr_size)); + } +} + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); + + struct task_dp_pdata *pdata = task->priv_data; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(pdata, sizeof(*pdata))); + K_OOPS(K_SYSCALL_OBJ(pdata->event, K_OBJ_EVENT)); + K_OOPS(K_SYSCALL_OBJ_INIT(pdata->thread, K_OBJ_THREAD)); + scheduler_dp_mod_vrfy(pdata->mod); + return z_impl_scheduler_dp_internal_free(task); +} +#include +#endif diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index f2c2434bc2b5..43c4b43e35e2 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -340,7 +340,7 @@ int scheduler_dp_task_init(struct task **task, return ret; } -void scheduler_dp_internal_free(struct task *task) +void z_impl_scheduler_dp_internal_free(struct task *task) { struct task_dp_pdata *pdata = task->priv_data; @@ -354,3 +354,17 @@ void scheduler_dp_internal_free(struct task *task) /* task is the first member in task_memory above */ sof_heap_free(pdata->mod->dev->drv->user_heap, task); } + +#ifdef CONFIG_USERSPACE +#include + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + /* + * With the thread DP scheduler variant scheduler_dp_internal_free() is + * never called from the userspace context + */ + K_OOPS(true); +} +#include +#endif From 6ad77b29c23a6ee953d1d492e7323fcbe996b64e Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 8 Jul 2026 16:17:54 +0200 Subject: [PATCH 03/12] schedule: ll: userspace: grant the LL thread rights on DP The LL userspace thread has to interact with the DP one. Grant required rights. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/schedule/ll_schedule_domain.h | 1 + src/schedule/zephyr_domain.c | 12 ++++++++++++ src/schedule/zephyr_dp_schedule.c | 3 ++- src/schedule/zephyr_dp_schedule_application.c | 3 +++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index 03991debb0cb..cfd0043995db 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -330,6 +330,7 @@ struct ll_schedule_domain *zephyr_domain_init(int clk); struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain); struct k_thread *zephyr_domain_thread_tid_for_core(int core); struct k_mem_domain *zephyr_ll_mem_domain(void); +struct k_thread *zephyr_ll_domain_thread(void); #endif /* CONFIG_SOF_USERSPACE_LL */ #ifdef CONFIG_SOF_FULL_ZEPHYR_APPLICATION __syscall int zephyr_ll_task_sem_alloc(struct task *task); diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index b37f91285854..cc1b76600dee 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -525,6 +525,18 @@ struct k_thread *zephyr_domain_thread_tid_for_core(int core) return ll_thread_tid[core]; } +struct k_thread *zephyr_ll_domain_thread(void) +{ + struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); + + if (!ll_domain) + return NULL; + + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); + + return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; +} + #endif /* CONFIG_SOF_USERSPACE_LL */ #if CONFIG_CROSS_CORE_STREAM diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index b6e09cc2c293..ca9f7dced9fc 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -62,7 +62,8 @@ void scheduler_dp_unlock(unsigned int key) void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) { - k_thread_access_grant(thread_id, &dp_lock[core]); + if (thread_id) + k_thread_access_grant(thread_id, &dp_lock[core]); } /* dummy LL task - to start LL on secondary cores */ diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 3183ca95aef3..c01d73964175 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -528,6 +528,9 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, k_thread_access_grant(pdata->thread_id, pdata->event, &dp_sync[core]); scheduler_dp_grant(pdata->thread_id, core); +#if CONFIG_SOF_USERSPACE_LL + scheduler_dp_grant(zephyr_ll_domain_thread(), core); +#endif struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), SOF_MEM_FLAG_COHERENT); From c52e4986ab9eaaaaff6406b3236cdecee7ed9d81 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 10:53:50 +0200 Subject: [PATCH 04/12] lib-manager: add a syscall to handle LLEXT-related work Extract a privileged LLEXT-related part from lib_manager_module_create() into a separate function to be called from kernel context. At the same time lib_manager_mod_free_priv() already executes privileged operations; to make it callable in userspace, convert lib_manager_free_module() to a system call. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/lib_manager.h | 19 +++++ src/library_manager/lib_manager.c | 111 +++++++++++++++++++++--------- zephyr/CMakeLists.txt | 1 + 3 files changed, 98 insertions(+), 33 deletions(-) diff --git a/src/include/sof/lib_manager.h b/src/include/sof/lib_manager.h index 019695be69ef..ceb47e6abe6c 100644 --- a/src/include/sof/lib_manager.h +++ b/src/include/sof/lib_manager.h @@ -220,6 +220,25 @@ void lib_manager_get_instance_bss_address(uint32_t instance_id, */ int lib_manager_load_library(uint32_t dma_id, uint32_t lib_id, uint32_t type); +struct userspace_context; +/* + * \brief Allocate the module and start the agent if needed + */ +int lib_manager_mod_create_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec, void **adapter_priv, + struct userspace_context **userspace, + const struct module_interface **ops); + +#if defined(__ZEPHYR__) && defined(CONFIG_SOF_FULL_ZEPHYR_APPLICATION) +__syscall int lib_manager_free_module(const uint32_t component_id); + +#include +#else +int z_impl_lib_manager_free_module(const uint32_t component_id); +#define lib_manager_free_module z_impl_lib_manager_free_module +#endif + /* * \brief Initialize message * diff --git a/src/library_manager/lib_manager.c b/src/library_manager/lib_manager.c index a672f51133d2..62233cb431a5 100644 --- a/src/library_manager/lib_manager.c +++ b/src/library_manager/lib_manager.c @@ -415,7 +415,7 @@ static uintptr_t lib_manager_allocate_module(const struct sof_man_fw_desc *const * * Function is responsible to free module resources in HP memory. */ -static int lib_manager_free_module(const uint32_t component_id) +int z_impl_lib_manager_free_module(const uint32_t component_id) { const struct sof_man_module *mod; const uint32_t module_id = IPC4_MOD_ID(component_id); @@ -465,7 +465,7 @@ static uintptr_t lib_manager_allocate_module(const struct sof_man_fw_desc *const return 0; } -static int lib_manager_free_module(const uint32_t component_id) +static int z_impl_lib_manager_free_module(const uint32_t component_id) { /* Since we cannot allocate the freeing is not considered to be an error */ tr_warn(&lib_manager_tr, "Dynamic module freeing is not supported"); @@ -646,34 +646,36 @@ static enum buildinfo_mod_type lib_manager_get_module_type(const struct sof_man_ } } -/* - * \brief Load module code, allocate its instance and create a module adapter component. - * \param[in] drv - component driver pointer. - * \param[in] config - component ipc descriptor pointer. - * \param[in] spec - passdowned data from driver. - * - * \return: a pointer to newly created module adapter component on success. NULL on error. - */ -static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, - const struct comp_ipc_config *config, - const void *spec) +/* Error path resource freeing */ +static void lib_manager_mod_free_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + struct userspace_context *userspace) +{ +#if CONFIG_SOF_USERSPACE_PROXY + if (userspace) + userspace_proxy_destroy(drv, userspace); +#endif /* CONFIG_SOF_USERSPACE_PROXY */ + lib_manager_free_module(config->id); +} + +int lib_manager_mod_create_priv(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec, void **adapter_priv, + struct userspace_context **userspace, + const struct module_interface **ops) { const struct sof_man_fw_desc *const desc = lib_manager_get_library_manifest(config->id); const struct ipc_config_process *args = (const struct ipc_config_process *)spec; const uint32_t entry_index = LIB_MANAGER_GET_MODULE_INDEX(config->id); - struct userspace_context *userspace = NULL; - const struct module_interface *ops; const struct sof_man_module *mod; system_agent_start_fn agent; - void *adapter_priv = NULL; const void **agent_iface; - struct comp_dev *dev; int ret; #ifdef CONFIG_SOF_USERSPACE_PROXY if (drv->user_heap && config->proc_domain != COMP_PROCESSING_DOMAIN_DP) { tr_err(&lib_manager_tr, "Userspace supports only DP modules."); - return NULL; + return -EOPNOTSUPP; } #endif @@ -681,12 +683,12 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, if (!desc) { tr_err(&lib_manager_tr, "Error: Couldn't find loadable module with id %u.", config->id); - return NULL; + return -ENOENT; } if (entry_index >= desc->header.num_module_entries) { tr_err(&lib_manager_tr, "Entry index %u out of bounds.", entry_index); - return NULL; + return -EINVAL; } mod = (const struct sof_man_module *) @@ -697,53 +699,96 @@ static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, if (!module_entry_point) { tr_err(&lib_manager_tr, "lib_manager_allocate_module() failed!"); - return NULL; + return -ENOENT; } switch (lib_manager_get_module_type(desc, mod)) { case MOD_TYPE_LLEXT: agent = NULL; - ops = (const struct module_interface *)module_entry_point; + *ops = (const struct module_interface *)module_entry_point; agent_iface = NULL; break; case MOD_TYPE_LMDK: agent = &native_system_agent_start; - agent_iface = (const void **)&ops; + agent_iface = (const void **)ops; break; #if CONFIG_INTEL_MODULES case MOD_TYPE_IADK: agent = &system_agent_start; - ops = &processing_module_adapter_interface; - agent_iface = (const void **)&adapter_priv; + *ops = &processing_module_adapter_interface; + agent_iface = (const void **)adapter_priv; break; #endif case MOD_TYPE_INVALID: + default: + ret = -EINVAL; goto err; } if (agent || IS_ENABLED(CONFIG_SOF_USERSPACE_PROXY)) { /* At this point module resources are allocated and it is moved to L2 memory. */ ret = lib_manager_start_agent(drv, config, mod, args, module_entry_point, agent, - agent_iface, &userspace, &ops); + agent_iface, userspace, ops); if (ret) goto err; } - if (comp_set_adapter_ops(drv, ops) < 0) + ret = comp_set_adapter_ops(drv, *ops); + if (ret < 0) goto err; - dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, NULL); + return 0; + +err: + lib_manager_mod_free_priv(drv, config, *userspace); + return ret; +} + +#ifdef CONFIG_USERSPACE +#include + +static int z_vrfy_lib_manager_free_module(const uint32_t component_id) +{ + return z_impl_lib_manager_free_module(component_id); +} +#include + +#endif /* CONFIG_USERSPACE */ + +/* + * \brief Load module code, allocate its instance and create a module adapter component. + * \param[in] drv - component driver pointer. + * \param[in] config - component ipc descriptor pointer. + * \param[in] spec - passdowned data from driver. + * + * \return: a pointer to newly created module adapter component on success. NULL on error. + */ +static struct comp_dev *lib_manager_module_create(const struct comp_driver *drv, + const struct comp_ipc_config *config, + const void *spec) +{ + struct userspace_context *userspace = NULL; + const struct module_interface *ops = NULL; + void *adapter_priv = NULL; + struct comp_dev *dev; + + if (config->proc_domain == COMP_PROCESSING_DOMAIN_DP || + !IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { + int ret = lib_manager_mod_create_priv(drv, config, spec, &adapter_priv, + &userspace, &ops); + + if (ret < 0) + return NULL; + } + + dev = module_adapter_new_ext(drv, config, spec, adapter_priv, userspace, ops); if (!dev) goto err; return dev; err: -#if CONFIG_SOF_USERSPACE_PROXY - if (userspace) - userspace_proxy_destroy(drv, userspace); -#endif /* CONFIG_SOF_USERSPACE_PROXY */ - lib_manager_free_module(config->id); + lib_manager_mod_free_priv(drv, config, userspace); return NULL; } diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index d036eaeede90..ca76f7eb9966 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -633,6 +633,7 @@ zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_ALLOC syscall/alloc. zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/vregion.h) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_INTERFACE_VREGION syscall/vregion.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib/dai-zephyr.h) +zephyr_syscall_header(${SOF_SRC_PATH}/include/sof/lib_manager.h) zephyr_library_sources_ifdef(CONFIG_USERSPACE syscall/dai.c) zephyr_syscall_header(${SOF_SRC_PATH}/include/user/debug_stream_slot.h) From 2d0d47286e47631dde6f7163230ce7528c87cc12 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 2 Jul 2026 13:52:52 +0200 Subject: [PATCH 05/12] llext: with userspace let LL thread access DP modules too If LL runs in userspace, it needs access to loaded LLEXT modules, running in DP more too. Signed-off-by: Guennadi Liakhovetski --- src/library_manager/llext_manager.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/library_manager/llext_manager.c b/src/library_manager/llext_manager.c index bcbfd290ceb0..30ff461ace7c 100644 --- a/src/library_manager/llext_manager.c +++ b/src/library_manager/llext_manager.c @@ -334,12 +334,10 @@ static int llext_manager_load_module(struct lib_manager_module *mctx) mctx->mapped = true; #ifdef CONFIG_SOF_USERSPACE_LL - if (!mctx->domain_dp) { - ret = llext_manager_add_mod_domain(mctx, zephyr_ll_mem_domain()); - if (ret < 0) { - tr_err(&lib_manager_tr, "failed to add domain: %d", ret); - goto e_data; - } + ret = llext_manager_add_mod_domain(mctx, zephyr_ll_mem_domain()); + if (ret < 0) { + tr_err(&lib_manager_tr, "failed to add domain: %d", ret); + goto e_data; } #endif @@ -426,8 +424,7 @@ static int llext_manager_unload_module(struct lib_manager_module *mctx) mctx->mapped = false; #ifdef CONFIG_SOF_USERSPACE_LL - if (!mctx->domain_dp) - llext_manager_rm_mod_domain(mctx, zephyr_ll_mem_domain()); + llext_manager_rm_mod_domain(mctx, zephyr_ll_mem_domain()); #endif return err; From ccc1e101bc8b58d23339d90906a936f2512f865f Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 10 Jun 2026 16:18:53 +0200 Subject: [PATCH 06/12] schedule: ll: enable multicore userspace Make scheduling LL thread and synchronisation objects per-core and forward IPCs and scheduling events accordingly. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/ipc/common.h | 23 +++++-- src/init/init.c | 12 +++- src/ipc/ipc-common.c | 123 ++++++++++++++++++++++++++--------- src/ipc/ipc4/handler-user.c | 105 ++++++++++++++++++++++++------ uuid-registry.txt | 1 + 5 files changed, 209 insertions(+), 55 deletions(-) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index a910c6d42c92..2726855d4f56 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -56,10 +56,16 @@ extern struct tr_ctx ipc_tr; #define IPC_TASK_SECONDARY_CORE BIT(2) #define IPC_TASK_POWERDOWN BIT(3) +#ifdef CONFIG_CORE_COUNT +#define CORE_COUNT CONFIG_CORE_COUNT +#else +#define CORE_COUNT 1 +#endif + struct ipc_user { - struct k_thread *thread; + struct k_thread *thread[CORE_COUNT]; struct k_sem *sem; - struct k_event *event; + struct k_event *event[CORE_COUNT]; /** @brief Copy of IPC4 message primary word forwarded to user thread */ uint32_t ipc_msg_pri; /** @brief Copy of IPC4 message extension word forwarded to user thread */ @@ -73,9 +79,10 @@ struct ipc_user { /** @brief Reply TX data pointer from user thread (e.g. LARGE_CONFIG_GET result) */ void *reply_tx_data; struct ipc *ipc; - struct k_thread *audio_thread; + struct k_thread *audio_thread[CORE_COUNT]; /** @brief Original kernel driver pointer for restoring dev->drv after create */ const struct comp_driver *init_drv; + bool init_needed[CORE_COUNT]; /** * @brief User-accessible copy of comp_driver + tr_ctx for create(). * @@ -326,7 +333,7 @@ extern bool ipc_enter_gdb; * @param extension Extension message word * @return Result code from user thread processing */ -int ipc_user_forward_cmd(uint32_t primary, uint32_t extension); +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core); /** * @brief Protocol-specific dispatch of a forwarded IPC command. @@ -338,6 +345,14 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension); * @return Result code to report back to the host */ int ipc_user_thread_dispatch(struct ipc_user *ipc_user); + +/** + * @brief Initialize IPC and LL scheduler threads on a booting secondary core. + * + * @param core Secondary core ID + * @return 0 or a negative error code + */ +int ipc_user_init_secondary(unsigned int core); #endif #endif /* __SOF_DRIVERS_IPC_H__ */ diff --git a/src/init/init.c b/src/init/init.c index 5990cfebc2dc..a9acfed5e9ac 100644 --- a/src/init/init.c +++ b/src/init/init.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -110,6 +111,9 @@ static inline int secondary_core_restore(void) { return 0; }; __cold int secondary_core_init(struct sof *sof) { +#if CONFIG_SOF_USERSPACE_LL || CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL + unsigned int core = cpu_get_id(); +#endif int err; struct ll_schedule_domain *dma_domain; @@ -134,6 +138,12 @@ __cold int secondary_core_init(struct sof *sof) if (dma_domain) scheduler_init_ll(dma_domain); +#if CONFIG_SOF_USERSPACE_LL + err = ipc_user_init_secondary(core); + if (err < 0) + return err; +#endif + #if CONFIG_ZEPHYR_DP_SCHEDULER err = scheduler_dp_init(); if (err < 0) @@ -152,7 +162,7 @@ __cold int secondary_core_init(struct sof *sof) return err; #endif #if CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL - err = core_kcps_adjust(cpu_get_id(), SECONDARY_CORE_BASE_CPS_USAGE); + err = core_kcps_adjust(core, SECONDARY_CORE_BASE_CPS_USAGE); if (err < 0) return err; #endif diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index 0e7b344510cd..a49ceffd5b1a 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -318,7 +318,10 @@ void ipc_schedule_process(struct ipc *ipc) #define IPC_USER_EVENT_CMD BIT(0) #define IPC_USER_EVENT_STOP BIT(1) -static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); +SOF_DEFINE_REG_UUID(sec_core_init); + +static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); /** * @brief Forward an IPC command to the user-space thread. @@ -332,7 +335,7 @@ static K_THREAD_STACK_DEFINE(ipc_user_stack, CONFIG_SOF_IPC_USER_THREAD_STACK_SI * @param extension Extension message word * @return Result from user thread processing */ -int ipc_user_forward_cmd(uint32_t primary, uint32_t extension) +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -346,13 +349,22 @@ int ipc_user_forward_cmd(uint32_t primary, uint32_t extension) pdata->ipc_msg_ext = extension; pdata->ipc = ipc; + /* + * Forwarding the first IPC to this core, wait for its userspace IPC + * thread to start + */ + if (pdata->init_needed[core]) { + pdata->init_needed[core] = false; + k_sem_take(pdata->sem, K_FOREVER); + } + /* Prevent host completion until user thread finishes */ key = k_spin_lock(&ipc->lock); ipc->task_mask |= IPC_TASK_IN_THREAD; k_spin_unlock(&ipc->lock, key); /* Wake the user thread */ - k_event_set(pdata->event, IPC_USER_EVENT_CMD); + k_event_set(pdata->event[core], IPC_USER_EVENT_CMD); /* Wait for user thread to complete */ ret = k_sem_take(pdata->sem, K_MSEC(100)); @@ -389,8 +401,8 @@ __weak int ipc_user_thread_dispatch(struct ipc_user *ipc_user) static void ipc_user_thread_fn(void *p1, void *p2, void *p3) { struct ipc_user *ipc_user = p1; + unsigned int core = POINTER_TO_UINT(p2); - ARG_UNUSED(p2); ARG_UNUSED(p3); __ASSERT(k_is_user_context(), "expected user context"); @@ -400,7 +412,7 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) LOG_INF("IPC user-space thread started"); for (;;) { - uint32_t mask = k_event_wait_safe(ipc_user->event, + uint32_t mask = k_event_wait_safe(ipc_user->event[core], IPC_USER_EVENT_CMD | IPC_USER_EVENT_STOP, false, K_FOREVER); @@ -418,7 +430,7 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) } } -__cold static int ipc_user_init_thread(struct ipc_user *ipc_user) +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) { char thread_name[] = "ll_user0"; int ret; @@ -426,47 +438,85 @@ __cold static int ipc_user_init_thread(struct ipc_user *ipc_user) assert_can_be_cold(); /* Allocate kernel objects for the user-space thread */ - ipc_user->event = k_object_alloc(K_OBJ_EVENT); - if (!ipc_user->event) { + ipc_user->event[core] = k_object_alloc(K_OBJ_EVENT); + if (!ipc_user->event[core]) { LOG_ERR("user IPC event alloc failed"); return -ENOMEM; } - k_event_init(ipc_user->event); + k_event_init(ipc_user->event[core]); - ipc_user->thread = k_object_alloc(K_OBJ_THREAD); - if (!ipc_user->thread) { + ipc_user->thread[core] = k_object_alloc(K_OBJ_THREAD); + if (!ipc_user->thread[core]) { LOG_ERR("user IPC thread alloc failed"); ret = -ENOMEM; goto e_event; } - k_thread_create(ipc_user->thread, ipc_user_stack, + k_thread_create(ipc_user->thread[core], ipc_user_stack[core], CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, - ipc_user_thread_fn, ipc_user, NULL, NULL, + ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, -1, K_USER, K_FOREVER); - k_thread_cpu_pin(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); - k_thread_name_set(ipc_user->thread, thread_name); + k_thread_cpu_pin(ipc_user->thread[core], core); + thread_name[sizeof(thread_name) - 2] = '0' + core; + k_thread_name_set(ipc_user->thread[core], thread_name); /* * Each userspace IPC thread must be able to wait on its private event * and signal completion on the primary core semaphore */ - k_thread_access_grant(ipc_user->thread, ipc_user->sem, ipc_user->event); - user_grant_dai_access_all(ipc_user->thread); - user_grant_dma_access_all(ipc_user->thread); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread); - user_ll_grant_access(ipc_user->thread, PLATFORM_PRIMARY_CORE_ID); - pipeline_posn_grant_access(ipc_user->thread); + k_thread_access_grant(ipc_user->thread[core], ipc_user->sem, ipc_user->event[core]); + user_grant_dai_access_all(ipc_user->thread[core]); + user_grant_dma_access_all(ipc_user->thread[core]); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), ipc_user->thread[core]); + user_ll_grant_access(ipc_user->thread[core], core); + pipeline_posn_grant_access(ipc_user->thread[core]); return 0; e_event: - k_object_free(ipc_user->event); + k_object_free(ipc_user->event[core]); return ret; } +__cold int ipc_user_init_secondary(unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *ipc_user = ipc->ipc_user_pdata; + int ret = ipc_user_init_thread(ipc_user, core); + + if (ret < 0) + return ret; + + assert_can_be_cold(); + + k_thread_start(ipc_user->thread[core]); + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("user LL task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, core, 0); + + ipc_user->audio_thread[core] = scheduler_init_context(task); + if (!ipc_user->audio_thread[core]) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + k_thread_access_grant(ipc_user->thread[core], ipc_user->audio_thread[core]); + ipc_user->init_needed[core] = true; + + /* Wait for user thread startup — consumes the initial k_sem_give from thread */ + return 0; +} + +/* Primary core only */ __cold static void ipc_user_init(void) { struct ipc *ipc = ipc_get(); @@ -480,6 +530,8 @@ __cold static void ipc_user_init(void) sof_panic(SOF_IPC_PANIC_IPC); } + assert_can_be_cold(); + ipc_user->sem = k_object_alloc(K_OBJ_SEM); if (!ipc_user->sem) { LOG_ERR("user IPC sem alloc failed"); @@ -530,13 +582,14 @@ __cold static void ipc_user_init(void) k_sem_init(ipc_user->sem, 0, 1); - ret = ipc_user_init_thread(ipc_user); + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); if (ret < 0) { LOG_ERR("user IPC thread initialization failed"); sof_panic(SOF_IPC_PANIC_IPC); } - ret = user_access_to_mailbox(zephyr_ll_mem_domain(), ipc_user->thread); + ret = user_access_to_mailbox(zephyr_ll_mem_domain(), + ipc_user->thread[PLATFORM_PRIMARY_CORE_ID]); if (ret < 0) { LOG_ERR("ipc user: mailbox access grant failed: %d", ret); sof_panic(SOF_IPC_PANIC_IPC); @@ -545,22 +598,32 @@ __cold static void ipc_user_init(void) /* Store references in ipc struct so kernel handler can forward commands */ ipc->ipc_user_pdata = ipc_user; - k_thread_start(ipc_user->thread); - struct task *task = zephyr_ll_task_alloc(); + if (!task) { + LOG_ERR("task allocation failed"); + k_panic(); + } + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, cpu_get_id(), 0); - ipc_user->audio_thread = scheduler_init_context(task); + 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = scheduler_init_context(task); + if (!ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID]) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } /* Grant ipc_user thread permission on the audio thread object. * Needed so user-space dai_common_new() can call * k_thread_access_grant(audio_thread, dai_mutex) from user context. */ - k_thread_access_grant(ipc_user->thread, ipc_user->audio_thread); + k_thread_access_grant(ipc_user->thread[PLATFORM_PRIMARY_CORE_ID], + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID]); + + k_thread_start(ipc_user->thread[PLATFORM_PRIMARY_CORE_ID]); /* Wait for user thread startup — consumes the initial k_sem_give from thread */ - k_sem_take(ipc->ipc_user_pdata->sem, K_FOREVER); + k_sem_take(ipc_user->sem, K_FOREVER); } #else static void ipc_user_init(void) diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index fbb68d809bd2..638356a6c352 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -90,7 +90,31 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( /* * Global IPC Operations. */ -#ifndef CONFIG_SOF_USERSPACE_LL +#ifdef CONFIG_SOF_USERSPACE_LL +/* + * Determine the target core for an IPC4 module message. + * Falls back to current core when no component is bound yet. + */ +static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) +{ + /* + * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, + * struct ipc4_module_delete_instance + */ + struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; + uint32_t module_id = config->primary.r.module_id; + + if (module_id) { + uint32_t instance_id = config->primary.r.instance_id; + struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); + + if (dev) + return dev->ipc_config.core; + } + + return cpu_get_id(); +} +#else __cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) { struct ipc *ipc = ipc_get(); @@ -99,9 +123,7 @@ __cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); } -#endif -#ifndef CONFIG_SOF_USERSPACE_LL __cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) { struct ipc4_pipeline_delete *pipe; @@ -675,6 +697,9 @@ static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply) { +#ifdef CONFIG_SOF_USERSPACE_LL + struct ipc *ipc = ipc_get(); +#endif uint32_t type; int ret; @@ -698,21 +723,57 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, /* pipeline settings */ case SOF_IPC4_GLB_CREATE_PIPELINE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + { + const struct ipc4_pipeline_create *create = + (const struct ipc4_pipeline_create *)ipc4; + + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + create->extension.r.core_id); + } #else ret = ipc4_new_pipeline(ipc4); #endif break; case SOF_IPC4_GLB_DELETE_PIPELINE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + { + const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + del->primary.r.instance_id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); + } #else ret = ipc4_delete_pipeline(ipc4); #endif break; case SOF_IPC4_GLB_SET_PIPELINE_STATE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + { + struct ipc4_pipeline_set_state state = { + .primary.dat = ipc4->primary.dat, + .extension.dat = ipc4->extension.dat, + }; + int id = ipc4_pipeline_id_get(ipc4, &state, NULL, NULL); + if (id < 0) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); + } #else ret = ipc4_set_pipeline_state(ipc4); #endif @@ -1514,14 +1575,11 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, * access to IMR manifest and driver list in kernel memory). * Component creation (drv->ops.create) runs in user thread * so untrusted module code does not execute in kernel context. - * Cross-core creation stays fully in kernel. */ struct ipc *ipc = ipc_get(); uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, mi->primary.r.instance_id); - const struct comp_driver *drv = ipc4_get_comp_drv( - IPC4_MOD_ID(comp_id)); - struct ipc_user *pdata = ipc->ipc_user_pdata; + const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); if (!drv) { ret = IPC4_MOD_NOT_INITIALIZED; @@ -1531,6 +1589,7 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, /* Copy comp_driver and tr_ctx into user-accessible ipc_user buffer * originals are in kernel .rodata/.data and not readable from user mode. */ + struct ipc_user *pdata = ipc->ipc_user_pdata; struct comp_driver *drv_copy = (struct comp_driver *)pdata->init_drv_data; struct tr_ctx *tctx_copy = (struct tr_ctx *)(pdata->init_drv_data + @@ -1547,7 +1606,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, break; pdata->init_drv = drv; - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + mi->extension.r.core_id); #endif } else { /* @@ -1561,7 +1621,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, case SOF_IPC4_MOD_CONFIG_GET: #ifdef CONFIG_SOF_USERSPACE_LL /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); if (!ret) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -1575,7 +1636,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, case SOF_IPC4_MOD_CONFIG_SET: #ifdef CONFIG_SOF_USERSPACE_LL /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_set_get_config_module_instance(ipc4, true); #endif @@ -1588,8 +1650,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, if (config->primary.r.module_id) { /* Module case: forward to user thread */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, - ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); if (!ret) { struct ipc *ipc = ipc_get(); struct ipc_user *pdata = ipc->ipc_user_pdata; @@ -1617,8 +1679,8 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, (const struct ipc4_module_large_config *)ipc4; if (config->primary.r.module_id) { - ret = ipc_user_forward_cmd(ipc4->primary.dat, - ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); } else { /* Base firmware: keep in kernel (IMR access) */ ret = ipc4_set_large_config_module_instance(ipc4); @@ -1630,21 +1692,24 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, break; case SOF_IPC4_MOD_BIND: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_bind_module_instance(ipc4); #endif break; case SOF_IPC4_MOD_UNBIND: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_unbind_module_instance(ipc4); #endif break; case SOF_IPC4_MOD_DELETE_INSTANCE: #ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat); + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); #else ret = ipc4_delete_module_instance(ipc4); #endif diff --git a/uuid-registry.txt b/uuid-registry.txt index 04c4a3881b21..b235311b312c 100644 --- a/uuid-registry.txt +++ b/uuid-registry.txt @@ -148,6 +148,7 @@ d7f6712d-131c-45a7-82ed6aa9dc2291ea pm_runtime 9302adf5-88be-4234-a0a7dca538ef81f4 sai 3dee06de-f25a-4e10-ae1fabc9573873ea schedule 70d223ef-2b91-4aac-b444d89a0db2793a sdma +bdcb1461-34f5-4047-b9cc70fdf8dfb234 sec_core_init 55a88ed5-3d18-46ca-88f10ee6eae9930f selector 32fe92c1-1e17-4fc2-9758c7f3542e980a selector4 cf90d851-68a2-4987-a2de85aed0c8531c sgen_mt8186 From bf3312ba0a8d45dd29f9402cf0673044a2e464cf Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 28 Aug 2026 15:53:59 +0200 Subject: [PATCH 07/12] schedule: dp: application: fix NULL check order Checking pointer for non-NULL after dereferencing it makes no sense. Swap the order. Signed-off-by: Guennadi Liakhovetski --- src/schedule/zephyr_dp_schedule_application.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index c01d73964175..78cbc269a61c 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -155,14 +155,14 @@ static void ipc_thread_unflatten_run(struct processing_module *pmod, struct ipc4 int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, const union scheduler_dp_thread_ipc_param *param) { - struct task_dp_pdata *pdata = pmod->dev->task->priv_data; - int ret; - if (!pmod) { tr_err(&dp_tr, "no thread module"); return -EINVAL; } + struct task_dp_pdata *pdata = pmod->dev->task->priv_data; + int ret; + if (cmd == SOF_IPC4_MOD_INIT_INSTANCE) { /* Wait for the DP thread to start */ ret = k_sem_take(&dp_sync[pmod->dev->task->core], DP_THREAD_IPC_TIMEOUT); From 426d6c13479e0a70e404abe5904b0a0e6741a3d3 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Wed, 8 Jul 2026 15:22:17 +0200 Subject: [PATCH 08/12] schedule: dp: userspace grant IPC thread rights on DP assets In case of userspace LL scheduling the (also userspace) IPC thread needs access rights to DP assets like the thread itself and its stack and synchronisation primitives. Signed-off-by: Guennadi Liakhovetski --- src/include/sof/ipc/common.h | 5 +++++ src/ipc/ipc-common.c | 8 ++++++++ src/schedule/zephyr_dp_schedule_application.c | 12 +++++++++--- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 2726855d4f56..9d318fb59166 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -355,4 +355,9 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user); int ipc_user_init_secondary(unsigned int core); #endif +/** + * \brief get pointer to the userspace IPC thread for core + */ +struct k_thread *ipc_thread_user(unsigned int core); + #endif /* __SOF_DRIVERS_IPC_H__ */ diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index a49ceffd5b1a..dd1a1ee6ee01 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -516,6 +516,14 @@ __cold int ipc_user_init_secondary(unsigned int core) return 0; } +struct k_thread *ipc_thread_user(unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *ipc_user = ipc->ipc_user_pdata; + + return ipc_user->thread[core]; +} + /* Primary core only */ __cold static void ipc_user_init(void) { diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 78cbc269a61c..043d97a8c89b 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -161,18 +162,19 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, } struct task_dp_pdata *pdata = pmod->dev->task->priv_data; + unsigned int core = pmod->dev->task->core; int ret; if (cmd == SOF_IPC4_MOD_INIT_INSTANCE) { /* Wait for the DP thread to start */ - ret = k_sem_take(&dp_sync[pmod->dev->task->core], DP_THREAD_IPC_TIMEOUT); + ret = k_sem_take(&dp_sync[core], DP_THREAD_IPC_TIMEOUT); if (ret < 0) { tr_err(&dp_tr, "Failed waiting for DP thread to start: %d", ret); return ret; } } - unsigned int lock_key = scheduler_dp_lock(pmod->dev->task->core); + unsigned int lock_key = scheduler_dp_lock(core); /* IPCs are serialised */ pdata->flat->ret = -ENOSYS; @@ -185,7 +187,7 @@ int scheduler_dp_thread_ipc(struct processing_module *pmod, unsigned int cmd, if (!ret) { /* Wait for completion */ - ret = k_sem_take(&dp_sync[cpu_get_id()], DP_THREAD_IPC_TIMEOUT); + ret = k_sem_take(&dp_sync[core], DP_THREAD_IPC_TIMEOUT); if (ret < 0) tr_err(&dp_tr, "Failed waiting for DP thread: %d", ret); else @@ -529,6 +531,10 @@ int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, k_thread_access_grant(pdata->thread_id, pdata->event, &dp_sync[core]); scheduler_dp_grant(pdata->thread_id, core); #if CONFIG_SOF_USERSPACE_LL + struct k_thread *thread_ipc = ipc_thread_user(core); + + k_thread_access_grant(thread_ipc, pdata->event, pdata->thread_id, p_stack, &dp_sync[core]); + scheduler_dp_grant(thread_ipc, core); scheduler_dp_grant(zephyr_ll_domain_thread(), core); #endif From 81112c6b7f094f85d2a9cbad7840e719fdd5a792 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 16:04:10 +0200 Subject: [PATCH 09/12] userspace: perform library loading in kernel context When running in syscall context on behalf of a userspace thread dynamically mapped memory doesn't automatically become accessible. To make it accessible it has to be added to the thread memory domain. This is a problem for loadable modules with executable cold sections. To be able to execute them they have to be mapped to threads with the executable bit set. While for linking that memory has to be mapped writable. To solve the problem we perform linking from the kernel IPC context before forwarding to the userspace IPC thread. Signed-off-by: Guennadi Liakhovetski --- src/include/ipc4/handler.h | 8 +++++++ src/ipc/ipc4/handler-user.c | 47 ++++++++++++------------------------- src/ipc/ipc4/helper.c | 33 ++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index d6f839458d54..cb54ecc08db0 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -16,6 +16,14 @@ struct ipc4_message_request; */ int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, struct ipc_msg *reply); +/** + * \brief Load a dynamically loadable module. + * @param[in] drv Component driver. + * @param[in] mi SOF_IPC4_MOD_INIT_INSTANCE data + */ +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi); + /** * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context. * @param[in] ipc4 IPC4 message request. diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 638356a6c352..c14bbe7a7ce5 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -1576,7 +1576,6 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, * Component creation (drv->ops.create) runs in user thread * so untrusted module code does not execute in kernel context. */ - struct ipc *ipc = ipc_get(); uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, mi->primary.r.instance_id); const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); @@ -1586,26 +1585,18 @@ __cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, break; } - /* Copy comp_driver and tr_ctx into user-accessible ipc_user buffer - * originals are in kernel .rodata/.data and not readable from user mode. - */ - struct ipc_user *pdata = ipc->ipc_user_pdata; - struct comp_driver *drv_copy = (struct comp_driver *)pdata->init_drv_data; - struct tr_ctx *tctx_copy = - (struct tr_ctx *)(pdata->init_drv_data + - sizeof(struct comp_driver)); - - ret = memcpy_s(drv_copy, sizeof(*drv_copy), drv, sizeof(*drv)); - if (!ret && drv->tctx) { - ret = memcpy_s(tctx_copy, sizeof(*tctx_copy), - drv->tctx, sizeof(*drv->tctx)); - drv_copy->tctx = tctx_copy; - } + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); - if (ret < 0) - break; + if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { + int err = ipc4_user_module_load(drv, mi); + + if (err < 0) { + ret = IPC4_MOD_NOT_INITIALIZED; + break; + } + } - pdata->init_drv = drv; + ipc_get()->ipc_user_pdata->init_drv = drv; ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, mi->extension.r.core_id); #endif @@ -1779,36 +1770,28 @@ int ipc_user_thread_dispatch(struct ipc_user *ipc_user) * module code does not execute with kernel privileges. * * init_drv = original kernel pointer - * init_drv_data = user-accessible copy */ - const struct comp_driver *orig_drv = ipc_user->init_drv; - const struct comp_driver *drv_copy = - (const struct comp_driver *)ipc_user->init_drv_data; - struct comp_dev *dev; + const struct comp_driver *drv = ipc_user->init_drv; ipc_user->init_drv = NULL; - if (!orig_drv) { + if (!drv) { result = IPC4_MOD_NOT_INITIALIZED; break; } - dev = comp_new_ipc4_user(&msg, drv_copy); + struct comp_dev *dev = comp_new_ipc4_user(&msg, drv); + if (!dev) { result = IPC4_MOD_NOT_INITIALIZED; break; } - /* Restore original kernel driver pointer. comp_init() - * set dev->drv to the copy; runtime code expects the - * canonical kernel address. - */ - dev->drv = orig_drv; - result = ipc4_add_comp_dev(dev); if (result != IPC4_SUCCESS) break; comp_update_ibs_obs_cpc(dev); + result = 0; break; } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 0907bcb03ace..ae6a5d7abd9c 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -236,6 +236,39 @@ __cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *mo } #ifdef CONFIG_SOF_USERSPACE_LL + +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi) +{ + /* + * move a part to the kernel thread: + * the userspace IPC handling thread would call comp_new_ipc4_user() to + * then call library manager .create method lib_manager_module_create(). + * That one calls lib_manager_mod_create_priv(), then + * lib_manager_allocate_module() and eventually + * llext_manager_allocate_module() for LLEXT modules. + */ + struct comp_ipc_config ipc_config; + int ret = ipc4_comp_new_config(&ipc_config, mi); + + if (ret < 0) + return ret; + + const struct ipc_config_process spec = { + .data = ipc4_get_comp_new_data(), + .size = ipc_config.ipc_config_size, + }; + +#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY + sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); +#endif + + struct userspace_context *userspace = NULL; + const struct module_interface *ops = NULL; + + return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace, &ops); +} + /** * comp_new_ipc4_user - Create component in user-space IPC thread context. * From b943a3e36fd67c83a6be0273b9bedc1e2e68431c Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 22 May 2026 16:14:08 +0200 Subject: [PATCH 10/12] schedule: ll: userspace: enable LLEXT LLEXT is now working with userspace LL and can be enabled. Signed-off-by: Guennadi Liakhovetski --- app/overlays/ptl/ll_userspace_overlay.conf | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 53b881320e6f..53c8763d6ce4 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -11,6 +11,7 @@ CONFIG_SOF_USERSPACE_LL=y # make the drivers work in user-space CONFIG_SOF_USERSPACE_INTERFACE_DMA=y CONFIG_DAI_USERSPACE=y +CONFIG_MAX_THREAD_BYTES=4 # Temporary settings that are needed currently to enable user-space LL # -------------------------------------------------------------------- @@ -24,11 +25,6 @@ CONFIG_COLD_STORE_EXECUTE_DEBUG=n CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS=n CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS=n -# disable loadable modules (hits privilege issues in user-space now) -CONFIG_LLEXT_STORAGE_WRITABLE=n -CONFIG_LLEXT_EXPERIMENTAL=n -CONFIG_MODULES=n - # some of current boot tests interfere with user-space setup CONFIG_SOF_BOOT_TEST_ALLOWED=n From cbbda3e988b793785b04c16750caf528abfecb8b Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Mon, 3 Aug 2026 16:14:09 +0200 Subject: [PATCH 11/12] userspace-ll: re-enable DRAM execution and data CONFIG_COLD_STORE_EXECUTE_DRAM can now be re-enabled for userspace LL. Signed-off-by: Guennadi Liakhovetski --- app/overlays/ptl/ll_userspace_overlay.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 53c8763d6ce4..4b89f63a8eb2 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -18,7 +18,6 @@ CONFIG_MAX_THREAD_BYTES=4 # problem with DSP panics due to illegal instruction hit in user-space if cold # store execution is enabled. Disable it for now until rootcause is found. -CONFIG_COLD_STORE_EXECUTE_DRAM=n CONFIG_COLD_STORE_EXECUTE_DEBUG=n # telemetry not yet user-space compatible From 524d832b769c3bd516d20af5df5bb3c445eae961 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Thu, 20 Aug 2026 15:18:17 +0200 Subject: [PATCH 12/12] userspace-ll: re-enable the DP scheduler The DP scheduler can now be user with userspace LL. Signed-off-by: Guennadi Liakhovetski --- app/overlays/ptl/ll_userspace_overlay.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/app/overlays/ptl/ll_userspace_overlay.conf b/app/overlays/ptl/ll_userspace_overlay.conf index 4b89f63a8eb2..90082d26e16f 100644 --- a/app/overlays/ptl/ll_userspace_overlay.conf +++ b/app/overlays/ptl/ll_userspace_overlay.conf @@ -31,4 +31,3 @@ CONFIG_SOF_BOOT_TEST_ALLOWED=n CONFIG_CROSS_CORE_STREAM=n CONFIG_INTEL_ADSP_MIC_PRIVACY=n CONFIG_XRUN_NOTIFICATIONS_ENABLE=n -CONFIG_ZEPHYR_DP_SCHEDULER=n