From 5cd7a780386940d1a49a08cbe272f803d6f5efc8 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Thu, 23 Oct 2025 12:42:00 +0100 Subject: [PATCH 1/4] vpages: Add a simple virtual continuous page allocator Add a simple virtual page allocator that uses the Zephyr memory mapping infrastructure to allocate pages from a virtual region and map them to physical pages. Due to simplicity, the number of allocations is limited to CONFIG_SOF_VPAGE_MAX_ALLOCS Signed-off-by: Liam Girdwood Signed-off-by: Guennadi Liakhovetski --- zephyr/Kconfig | 13 ++ zephyr/include/sof/lib/regions_mm.h | 5 +- zephyr/include/sof/lib/vpage.h | 38 ++++ zephyr/lib/vpage.c | 341 ++++++++++++++++++++++++++++ 4 files changed, 395 insertions(+), 2 deletions(-) create mode 100644 zephyr/include/sof/lib/vpage.h create mode 100644 zephyr/lib/vpage.c diff --git a/zephyr/Kconfig b/zephyr/Kconfig index c744b965a07f..3cbea871fe48 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -145,6 +145,19 @@ config SOF_USERSPACE_APPLICATION Not manually settable. This is effectively a shortcut to replace numerous checks for (CONFIG_USERSPACE && !CONFIG_SOF_USERSPACE_PROXY) +config SOF_VPAGE_MAX_ALLOCS + int "Number of virtual memory page allocations" + default 128 + help + This setting defines the maximum number of virtual memory page + allocations that can be tracked. Each allocation represents a + contiguous block of virtual memory allocated from the virtual memory + region. This API isn't intended for end-user allocations, instead it + should be used by the framework to allocate memory, which is then + used, e.g. to create one or multiple heap allocators in it. Increasing + this number allows for more simultaneous allocations, but also + increases the memory overhead for tracking these allocations. + config ZEPHYR_NATIVE_DRIVERS bool "Use Zephyr native drivers" help diff --git a/zephyr/include/sof/lib/regions_mm.h b/zephyr/include/sof/lib/regions_mm.h index 9e960e304b65..abaaf158d3d4 100644 --- a/zephyr/include/sof/lib/regions_mm.h +++ b/zephyr/include/sof/lib/regions_mm.h @@ -18,8 +18,9 @@ #include /* Attributes for memory regions */ -#define VIRTUAL_REGION_SHARED_HEAP_ATTR 1U /*< region dedicated for shared virtual heap */ -#define VIRTUAL_REGION_LLEXT_LIBRARIES_ATTR 2U /*< region dedicated for LLEXT libraries */ +#define VIRTUAL_REGION_SHARED_HEAP_ATTR 1U /*< region for shared virtual heap */ +#define VIRTUAL_REGION_LLEXT_LIBRARIES_ATTR 2U /*< region for LLEXT libraries */ +#define VIRTUAL_REGION_VPAGES_ATTR 3U /*< region for virtual page allocator */ /* Dependency on ipc/topology.h created due to memory capability definitions * that are defined there diff --git a/zephyr/include/sof/lib/vpage.h b/zephyr/include/sof/lib/vpage.h new file mode 100644 index 000000000000..f3fc8b89e968 --- /dev/null +++ b/zephyr/include/sof/lib/vpage.h @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright(c) 2025 Intel Corporation. + +/* Virtual Page Allocator API */ +#ifndef __SOF_LIB_VPAGE_H__ +#define __SOF_LIB_VPAGE_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Allocate virtual pages + * Allocates a specified number of contiguous virtual memory pages by mapping + * physical pages. + * + * @param[in] pages Number of pages (usually 4kB large) to allocate. + * + * @return Pointer to the allocated virtual memory region, or NULL on failure. + */ +void *vpage_alloc(unsigned int pages); + +/** + * @brief Free virtual pages + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param[in] ptr Pointer to the memory pages to free. + */ +void vpage_free(void *ptr); + +#ifdef __cplusplus +} +#endif + +#endif /* __SOF_LIB_VPAGE_H__ */ diff --git a/zephyr/lib/vpage.c b/zephyr/lib/vpage.c new file mode 100644 index 000000000000..ce0da7b5ac97 --- /dev/null +++ b/zephyr/lib/vpage.c @@ -0,0 +1,341 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. + * + * Author: Liam Girdwood + */ + +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(vpage, CONFIG_SOF_LOG_LEVEL); + +/* Simple Page Allocator. + * + * This allocator manages the allocation and deallocation of virtual memory + * pages from a predefined virtual memory region which is larger than the + * physical memory region. + * + * Both memory regions are divided into pages (usually 4kB) that are represented + * as blocks in a bitmap using the zephyr sys_mem_blocks API. The virtual block + * map tracks the allocation of virtual memory pages while the physical block + * map in the Zephyr MM driver tracks the allocation of physical memory pages. + */ + +/* max number of allocations */ +#define VPAGE_MAX_ALLOCS CONFIG_SOF_VPAGE_MAX_ALLOCS + +/* + * Virtual memory allocation element - tracks allocated virtual page id and size + */ +struct valloc_elem { + unsigned short pages; /* number of pages allocated in contiguous block */ + unsigned short vpage; /* virtual page number from start of region */ +}; + +/* + * Virtual page allocator context + * + * This structure holds all information about virtual memory pages including the + * number of free and total pages, the virtual memory region, the block + * allocator for virtual pages and the allocations. + */ +struct vpage_context { + struct k_mutex lock; + unsigned int free_pages; /* number of free pages */ + unsigned int total_pages; /* total number of pages */ + + /* Virtual memory region information */ + const struct sys_mm_drv_region *virtual_region; + struct sys_mem_blocks vpage_blocks; + sys_bitarray_t bitmap; + + /* allocation elements to track page id to allocation size */ + unsigned int num_elems_in_use; /* number of allocated elements in use*/ + struct valloc_elem velems[VPAGE_MAX_ALLOCS]; +}; + +/* uncache persistent across all cores */ +static struct vpage_context vpage_ctx; + +/** + * @brief Allocate and map virtual memory pages + * + * Allocates memory pages from the virtual page allocator. + * Maps physical memory pages to the virtual region as needed. + * + * @param pages Number of pages to allocate. + * @param ptr Pointer to store the address of allocated pages. + * @retval 0 if successful. + */ +static int vpages_alloc_and_map(unsigned int pages, void **ptr) +{ + void *vaddr; + int ret; + + /* check for valid pages and ptr */ + if (!ptr) + return -EINVAL; + + if (!pages) { + *ptr = NULL; + return 0; + } + + /* quick check for enough free pages */ + if (vpage_ctx.free_pages < pages) { + LOG_ERR("error: not enough free pages %u for requested pages %u", + vpage_ctx.free_pages, pages); + return -ENOMEM; + } + + /* check for allocation elements */ + if (vpage_ctx.num_elems_in_use >= VPAGE_MAX_ALLOCS) { + LOG_ERR("error: max allocation elements reached"); + return -ENOMEM; + } + + /* allocate virtual continuous blocks */ + ret = sys_mem_blocks_alloc_contiguous(&vpage_ctx.vpage_blocks, pages, &vaddr); + if (ret < 0) { + LOG_ERR("error: failed to allocate %u continuous virtual pages, free %u", + pages, vpage_ctx.free_pages); + return ret; + } + + /* map the virtual blocks in virtual region to free physical blocks */ + ret = sys_mm_drv_map_region_safe(vpage_ctx.virtual_region, vaddr, + 0, pages * CONFIG_MM_DRV_PAGE_SIZE, SYS_MM_MEM_PERM_RW); + if (ret < 0) { + LOG_ERR("error: failed to map virtual region %p to physical region %p, error %d", + vaddr, vpage_ctx.virtual_region->addr, ret); + sys_mem_blocks_free_contiguous(&vpage_ctx.vpage_blocks, vaddr, pages); + return ret; + } + + /* success update the free pages */ + vpage_ctx.free_pages -= pages; + + /* Elements are acquired densely, just use the next one */ + vpage_ctx.velems[vpage_ctx.num_elems_in_use].pages = pages; + vpage_ctx.velems[vpage_ctx.num_elems_in_use].vpage = + (POINTER_TO_UINT(vaddr) - + POINTER_TO_UINT(vpage_ctx.virtual_region->addr)) / + CONFIG_MM_DRV_PAGE_SIZE; + vpage_ctx.num_elems_in_use++; + + /* return the virtual address */ + *ptr = vaddr; + + return 0; +} + +/** + * @brief Allocate virtual memory pages + * + * Allocates virtual memory pages from the virtual page allocator. + * + * @param pages Number of pages (usually 4kB large) to allocate. + * @retval NULL on allocation failure. + */ +void *vpage_alloc(unsigned int pages) +{ + void *ptr = NULL; + int ret; + + k_mutex_lock(&vpage_ctx.lock, K_FOREVER); + ret = vpages_alloc_and_map(pages, &ptr); + k_mutex_unlock(&vpage_ctx.lock); + if (ret < 0) + LOG_ERR("vpage_alloc failed %d for %d pages, total %d free %d", + ret, pages, vpage_ctx.total_pages, vpage_ctx.free_pages); + else + LOG_INF("vpage_alloc ptr %p pages %u free %u/%u", ptr, pages, vpage_ctx.free_pages, + vpage_ctx.total_pages); + return ptr; +} + +/** + * @brief Free and unmap virtual memory pages + * + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param ptr Pointer to the memory pages to free. + * @retval 0 if successful. + * @retval -EINVAL if ptr is invalid. + */ +static int vpages_free_and_unmap(uintptr_t *ptr) +{ + unsigned int alloc_idx, elem_idx; + unsigned int pages = 0; + int ret; + + /* check for valid ptr which must be page aligned */ + CHECKIF(!IS_ALIGNED(ptr, CONFIG_MM_DRV_PAGE_SIZE)) { + LOG_ERR("error: invalid non aligned page pointer %p", ptr); + return -EINVAL; + } + + alloc_idx = (POINTER_TO_UINT(ptr) - POINTER_TO_UINT(vpage_ctx.virtual_region->addr)) / + CONFIG_MM_DRV_PAGE_SIZE; + + /* find the allocation element */ + for (elem_idx = 0; elem_idx < VPAGE_MAX_ALLOCS; elem_idx++) { + if (vpage_ctx.velems[elem_idx].pages > 0 && + vpage_ctx.velems[elem_idx].vpage == alloc_idx) { + pages = vpage_ctx.velems[elem_idx].pages; + + LOG_DBG("found allocation element %d pages %u vpage %u for ptr %p", + elem_idx, vpage_ctx.velems[elem_idx].pages, + vpage_ctx.velems[elem_idx].vpage, ptr); + break; + } + } + + /* check we found allocation element */ + CHECKIF(!pages) { + LOG_ERR("error: invalid page pointer %p not found", ptr); + return -EINVAL; + } + + /* unmap the pages from virtual region */ + ret = sys_mm_drv_unmap_region((void *)ptr, pages * CONFIG_MM_DRV_PAGE_SIZE); + if (ret < 0) { + LOG_ERR("error: failed to unmap virtual region %p pages %u, error %d", + ptr, pages, ret); + return ret; + } + + /* free physical blocks */ + ret = sys_mem_blocks_free_contiguous(&vpage_ctx.vpage_blocks, ptr, pages); + if (ret < 0) { + LOG_ERR("error: failed to free %u continuous virtual page blocks at %p, error %d", + pages, ptr, ret); + return ret; + } + + /* move the last element over the released one, clear the last element */ + if (vpage_ctx.num_elems_in_use != elem_idx) + vpage_ctx.velems[elem_idx] = vpage_ctx.velems[vpage_ctx.num_elems_in_use]; + vpage_ctx.velems[vpage_ctx.num_elems_in_use].pages = 0; + vpage_ctx.velems[vpage_ctx.num_elems_in_use].vpage = 0; + vpage_ctx.num_elems_in_use--; + + /* success update the free pages */ + vpage_ctx.free_pages += pages; + + return ret; +} + +/** + * @brief Free virtual pages + * Frees previously allocated virtual memory pages and unmaps them. + * + * @param ptr + */ +void vpage_free(void *ptr) +{ + int ret; + + k_mutex_lock(&vpage_ctx.lock, K_FOREVER); + ret = vpages_free_and_unmap((uintptr_t *)ptr); + k_mutex_unlock(&vpage_ctx.lock); + + if (!ret) + LOG_INF("vptr %p free/total pages %d/%d", ptr, vpage_ctx.free_pages, + vpage_ctx.total_pages); +} + +/** + * @brief Initialize virtual page allocator + * + * Initializes a virtual page allocator that manages a virtual memory region + * using a page table and block structures. + * + * @retval 0 if successful. + * @retval -ENOMEM on creation failure. + */ +static int vpage_init(void) +{ + const struct sys_mm_drv_region *virtual_memory_regions; + const struct sys_mm_drv_region *region; + uint32_t *bundles = NULL; + unsigned int block_count, bitmap_num_bundles; + int ret; + + /* create the virtual memory region and add it to the system */ + size_t remaining_ram = L2_SRAM_BASE + L2_SRAM_SIZE - + (adsp_mm_get_unused_l2_start_aligned() + + CONFIG_SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE + + CONFIG_LIBRARY_REGION_SIZE); + + ret = adsp_add_virtual_memory_region(adsp_mm_get_unused_l2_start_aligned() + + CONFIG_SOF_ZEPHYR_VIRTUAL_HEAP_REGION_SIZE, + remaining_ram, VIRTUAL_REGION_VPAGES_ATTR); + if (ret) + return ret; + + k_mutex_init(&vpage_ctx.lock); + + /* now find the virtual region in all memory regions */ + virtual_memory_regions = sys_mm_drv_query_memory_regions(); + SYS_MM_DRV_MEMORY_REGION_FOREACH(virtual_memory_regions, region) { + if (region->attr == VIRTUAL_REGION_VPAGES_ATTR) { + vpage_ctx.virtual_region = region; + break; + } + } + sys_mm_drv_query_memory_regions_free(virtual_memory_regions); + + /* check for a valid region */ + if (!vpage_ctx.virtual_region) { + LOG_ERR("error: no valid virtual region found"); + k_panic(); + } + + block_count = region->size / CONFIG_MM_DRV_PAGE_SIZE; + if (block_count == 0) { + LOG_ERR("error: virtual region too small %zu", region->size); + k_panic(); + } + + vpage_ctx.total_pages = block_count; + vpage_ctx.free_pages = block_count; + vpage_ctx.num_elems_in_use = 0; + + /* bundles are uint32_t of bits */ + bitmap_num_bundles = SOF_DIV_ROUND_UP(block_count, sizeof(uint32_t) * 8); + + /* allocate memory for bitmap bundles */ + bundles = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + bitmap_num_bundles * sizeof(uint32_t)); + if (!bundles) { + LOG_ERR("error: virtual region bitmap alloc failed"); + k_panic(); + } + + /* Fill allocators data based on config and virtual region data */ + vpage_ctx.vpage_blocks.info.num_blocks = block_count; + vpage_ctx.vpage_blocks.info.blk_sz_shift = ilog2(CONFIG_MM_DRV_PAGE_SIZE); + /* buffer is the start of the virtual memory region */ + vpage_ctx.vpage_blocks.buffer = (uint8_t *)vpage_ctx.virtual_region->addr; + + /* initialize bitmap */ + vpage_ctx.bitmap.num_bits = block_count; + vpage_ctx.bitmap.num_bundles = bitmap_num_bundles; + vpage_ctx.bitmap.bundles = bundles; + vpage_ctx.vpage_blocks.bitmap = &vpage_ctx.bitmap; + + LOG_INF("region %p size %#zx pages %u", + (void *)vpage_ctx.virtual_region->addr, + vpage_ctx.virtual_region->size, block_count); + + return 0; +} + +SYS_INIT(vpage_init, POST_KERNEL, 1); From 7887cfb825786f3c4e58fbf30987d37689b7726b Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Thu, 23 Oct 2025 13:39:49 +0100 Subject: [PATCH 2/4] vregion: Add support for per pipeline/module virtual regions Add support for per pipeline and per module virtual memory regions. The intention is to provide a single virtual memory region per pipeline or per DP module that can simplify module/pipeline memory management. The region takes advantage of the way pipeline and modules are constructed, destroyed and used during their lifetimes. 1) memory tracking - 1 pointer/size per pipeline or DP module. 2) memory read/write/execute permissions 3) cache utilization. Modules and pipelines will allocate from their region only and this will be abstracted via the allocation APIs. Signed-off-by: Liam Girdwood Signed-off-by: Guennadi Liakhovetski --- app/boards/intel_adsp_ace15_mtpm.conf | 2 +- app/boards/intel_adsp_ace20_lnl.conf | 2 +- app/boards/intel_adsp_ace30_ptl.conf | 2 +- app/boards/intel_adsp_ace30_wcl.conf | 2 +- app/boards/intel_adsp_ace40_nvl.conf | 2 +- app/boards/intel_adsp_ace40_nvls.conf | 2 +- zephyr/CMakeLists.txt | 8 +- zephyr/Kconfig | 13 +- zephyr/include/sof/lib/vregion.h | 96 +++++++ zephyr/lib/vregion.c | 355 ++++++++++++++++++++++++++ 10 files changed, 475 insertions(+), 9 deletions(-) create mode 100644 zephyr/include/sof/lib/vregion.h create mode 100644 zephyr/lib/vregion.c diff --git a/app/boards/intel_adsp_ace15_mtpm.conf b/app/boards/intel_adsp_ace15_mtpm.conf index e95b18d97169..4f124eee5285 100644 --- a/app/boards/intel_adsp_ace15_mtpm.conf +++ b/app/boards/intel_adsp_ace15_mtpm.conf @@ -71,7 +71,7 @@ CONFIG_DMA_DW_LLI_POOL_SIZE=50 CONFIG_DMA_INTEL_ADSP_GPDMA=y CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=38400000 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 diff --git a/app/boards/intel_adsp_ace20_lnl.conf b/app/boards/intel_adsp_ace20_lnl.conf index beb441d6e6c6..2aef09fb9a4a 100644 --- a/app/boards/intel_adsp_ace20_lnl.conf +++ b/app/boards/intel_adsp_ace20_lnl.conf @@ -49,7 +49,7 @@ CONFIG_DAI_INIT_PRIORITY=70 CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_SYS_CLOCK_HW_CYCLES_PER_SEC=38400000 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 diff --git a/app/boards/intel_adsp_ace30_ptl.conf b/app/boards/intel_adsp_ace30_ptl.conf index 12aa78162c06..a79ca6e67764 100644 --- a/app/boards/intel_adsp_ace30_ptl.conf +++ b/app/boards/intel_adsp_ace30_ptl.conf @@ -53,7 +53,7 @@ CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_DMA_DW_LLI_POOL_SIZE=50 CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_XTENSA_MMU_NUM_L2_TABLES=128 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 diff --git a/app/boards/intel_adsp_ace30_wcl.conf b/app/boards/intel_adsp_ace30_wcl.conf index 621eb719de9f..999a3c309d41 100644 --- a/app/boards/intel_adsp_ace30_wcl.conf +++ b/app/boards/intel_adsp_ace30_wcl.conf @@ -50,7 +50,7 @@ CONFIG_DAI_DMIC_HAS_MULTIPLE_LINE_SYNC=y CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_MEMORY_WIN_2_SIZE=12288 CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 # Zephyr / power settings diff --git a/app/boards/intel_adsp_ace40_nvl.conf b/app/boards/intel_adsp_ace40_nvl.conf index 0f60dc5569b1..becef1a65f10 100644 --- a/app/boards/intel_adsp_ace40_nvl.conf +++ b/app/boards/intel_adsp_ace40_nvl.conf @@ -42,7 +42,7 @@ CONFIG_DAI_DMIC_HAS_OWNERSHIP=n CONFIG_DAI_DMIC_HAS_MULTIPLE_LINE_SYNC=y CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 # Zephyr / power settings diff --git a/app/boards/intel_adsp_ace40_nvls.conf b/app/boards/intel_adsp_ace40_nvls.conf index 0f60dc5569b1..becef1a65f10 100644 --- a/app/boards/intel_adsp_ace40_nvls.conf +++ b/app/boards/intel_adsp_ace40_nvls.conf @@ -42,7 +42,7 @@ CONFIG_DAI_DMIC_HAS_OWNERSHIP=n CONFIG_DAI_DMIC_HAS_MULTIPLE_LINE_SYNC=y CONFIG_DMA_INTEL_ADSP_GPDMA=n CONFIG_MM_DRV_INTEL_ADSP_TLB_REMAP_UNUSED_RAM=y -CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=2 +CONFIG_MM_DRV_INTEL_VIRTUAL_REGION_COUNT=3 CONFIG_SYS_CLOCK_TICKS_PER_SEC=12000 # Zephyr / power settings diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 233c93f3e070..9a4737858910 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -289,8 +289,12 @@ if (CONFIG_SOC_SERIES_INTEL_ADSP_ACE) ${SOF_PLATFORM_PATH}/novalake/lib/clk.c ) - # Sources for virtual heap management - zephyr_library_sources( + zephyr_library_sources_ifdef(CONFIG_SOF_VREGIONS + lib/vpage.c + lib/vregion.c + ) + + zephyr_library_sources_ifdef(CONFIG_VIRTUAL_HEAP lib/regions_mm.c ) diff --git a/zephyr/Kconfig b/zephyr/Kconfig index 3cbea871fe48..1d02868fd5ca 100644 --- a/zephyr/Kconfig +++ b/zephyr/Kconfig @@ -250,8 +250,19 @@ config SOF_ZEPHYR_NO_SOF_CLOCK Do not use SOF clk.h interface to set the DSP clock frequency. Requires implementation of platform/lib/clk.h. +config SOF_VREGIONS + bool "Enable virtual memory regions" + default y if ACE + depends on ACE + help + Enable the virtual regions memory allocator for pipeline resource management. + This provides a way to manage memory resources for audio pipelines, + including + 1) multiple pipeline static lifetime allocations. + 2) runtime pipeline allocations. + config VIRTUAL_HEAP - bool "Use virtual memory heap to allocate a buffers" + bool "Use virtual memory heap to allocate buffers" default y if ACE depends on ACE help diff --git a/zephyr/include/sof/lib/vregion.h b/zephyr/include/sof/lib/vregion.h new file mode 100644 index 000000000000..135052c51280 --- /dev/null +++ b/zephyr/include/sof/lib/vregion.h @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: BSD-3-Clause +// Copyright(c) 2025 Intel Corporation. + +/* Pre Allocated Contiguous Virtual Region */ +#ifndef __SOF_LIB_VREGION_H__ +#define __SOF_LIB_VREGION_H__ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct vregion; + +/** + * @brief Create a new virtual region instance. + * + * Create a new virtual region instance with specified static and dynamic partitions. + * Total size is the sum of static and dynamic sizes. + * + * @param[in] lifetime_size Size of the virtual region lifetime partition. + * @param[in] interim_size Size of the virtual region interim partition. + * @return struct vregion* Pointer to the new virtual region instance, or NULL on failure. + */ +struct vregion *vregion_create(size_t lifetime_size, size_t interim_size); + +/** + * @brief Destroy a virtual region instance. + * + * Free all associated resources and deallocate the virtual region instance. + * + * @param[in] vr Pointer to the virtual region instance to destroy. + */ +void vregion_destroy(struct vregion *vr); + +/** + * @brief Memory types for virtual region allocations. + * Used to specify the type of memory allocation within a virtual region. + * + * @note + * - interim: allocation that can be freed i.e. get/set large config, kcontrols. + * - lifetime: allocation that cannot be freed i.e. init data, pipeline data. + */ +enum vregion_mem_type { + VREGION_MEM_TYPE_INTERIM, /* interim allocation that can be freed */ + VREGION_MEM_TYPE_LIFETIME, /* lifetime allocation */ +}; + +/** + * @brief Allocate memory from the specified virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Type of memory to allocate (lifetime or interim). + * @param[in] size Size of memory to allocate in bytes. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size); + +/** + * @brief Allocate aligned memory from the specified virtual region. + * + * Allocate aligned memory from the specified virtual region based on the memory type. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Type of memory to allocate (lifetime or interim). + * @param[in] size Size of memory to allocate in bytes. + * @param[in] alignment Alignment of memory to allocate in bytes. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment); + +/** + * @brief Free memory allocated from the specified virtual region. + * + * Free memory previously allocated from the specified virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] ptr Pointer to the memory to free. + */ +void vregion_free(struct vregion *vr, void *ptr); + +/** + * @brief Log virtual region memory usage. + * + * @param[in] vr Pointer to the virtual region instance. + */ +void vregion_info(struct vregion *vr); + +#ifdef __cplusplus +} +#endif + +#endif /* __SOF_LIB_VREGION_H__ */ diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c new file mode 100644 index 000000000000..42e72cb8d15b --- /dev/null +++ b/zephyr/lib/vregion.c @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. + * + * Author: Liam Girdwood + */ + +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_REGISTER(vregion, CONFIG_SOF_LOG_LEVEL); + +/* + * Pre Allocated Contiguous Virtual Memory Region Allocator + * + * This allocator manages a pre-allocated virtual memory region that uses the + * virtual page allocator to allocate and free memory pages. + * + * It is designed for use cases where a contiguous virtual memory region + * is required, such as for batched allocation of audio pipelines and modules. + * + * New pipelines will create a new virtual region and will specify the size of the region + * which can be divided into multiple areas for different allocation lifetimes, permissions + * and sharing requirements. + * + * Advantages: + * + * 1) Contiguous virtual memory region for easier management and tracking of + * pipeline & DP module memory. i.e. we just need to track the vregion pointer. + * 2) Reduced fragmentation and better cache utilization by using a simple linear + * allocator for lifetime objects. + * + * Note: Software must pass in the size of the region areas at pipeline creation time. + */ + +/** + * @brief virtual region memory structure. + * + * This structure represents a virtual memory region, which includes + * information about the base address, size, and allocation status + * of the region. + * + * Currently the virtual region memory can be partitioned into two areas on + * page-aligned boundaries: + * + * 1. Interim Heap: An interim memory area used for multiple temporary + * allocations and frees over the lifetime of the audio processing pipeline. + * E.g. for module kcontrol derived allocations. + * + * 2. Lifetime Allocator: A simple incrementing allocator used for long-term + * static allocations that persist for the lifetime of the audio processing + * pipeline. This allocator compresses allocations for better cache + * utilization. + * + * More types can be added in the future. + * + * * TODO: Pipeline/module reset() could reset the dynamic heap. + */ + + /* linear heap used for lifetime allocations */ +struct vlinear_heap { + uint8_t *base; /* base address of linear allocator */ + uint8_t *ptr; /* current alloc pointer */ + size_t size; /* size of linear allocator in bytes */ + size_t used; /* used bytes in linear allocator */ + int free_count; /* number of frees - tuning only */ +}; + +/* zephyr k_heap for interim allocations. TODO: make lockless for improved performance */ +struct interim_heap { + struct k_heap heap; +}; + +/* Main vregion context, see above intro for more details. + * TODO: Add support to flag which heaps should have their contexts saved and restored. + */ +struct vregion { + /* region context */ + uint8_t *base; /* base address of entire region */ + size_t size; /* size of whole region in bytes */ + unsigned int pages; /* size of whole region in pages */ + + /* interim heap */ + struct interim_heap interim; /* interim heap */ + + /* lifetime heap */ + struct vlinear_heap lifetime; /* lifetime linear heap */ +}; + +/** + * @brief Create a new virtual region instance. + * + * Create a new VIRTUAL REGION instance with specified static and dynamic + * sizes. Total size is their sum. + * + * @param[in] lifetime_size Size of the virtual region lifetime partition. + * @param[in] interim_size Size of the virtual region interim partition. + * @return struct vregion* Pointer to the new virtual region instance, or NULL on failure. + */ +struct vregion *vregion_create(size_t lifetime_size, size_t interim_size) +{ + struct vregion *vr; + unsigned int pages; + size_t total_size; + uint8_t *vregion_base; + + if (!lifetime_size || !interim_size) { + LOG_ERR("error: invalid vregion lifetime size %zu or interim size %zu", + lifetime_size, interim_size); + return NULL; + } + + /* + * Align up lifetime sizes and interim sizes to nearest page, the + * vregion structure is stored in lifetime area so account for its size too. + */ + lifetime_size += sizeof(*vr); + lifetime_size = ALIGN_UP(lifetime_size, CONFIG_MM_DRV_PAGE_SIZE); + interim_size = ALIGN_UP(interim_size, CONFIG_MM_DRV_PAGE_SIZE); + total_size = lifetime_size + interim_size; + + /* allocate pages for vregion */ + pages = total_size / CONFIG_MM_DRV_PAGE_SIZE; + vregion_base = vpage_alloc(pages); + if (!vregion_base) + return NULL; + + /* init vregion - place it at the start of the lifetime region */ + vr = (struct vregion *)(vregion_base + interim_size); + vr->base = vregion_base; + vr->size = total_size; + vr->pages = pages; + + /* set partition sizes */ + vr->interim.heap.heap.init_bytes = interim_size; + vr->lifetime.size = lifetime_size; + + /* set base addresses for partitions */ + vr->interim.heap.heap.init_mem = vr->base; + vr->lifetime.base = vr->base + interim_size; + + /* set alloc ptr addresses for lifetime linear partitions */ + vr->lifetime.ptr = vr->lifetime.base + sizeof(*vr); /* skip vregion struct */ + vr->lifetime.used = sizeof(*vr); + + /* init interim heaps */ + k_heap_init(&vr->interim.heap, vr->interim.heap.heap.init_mem, interim_size); + + /* log the new vregion */ + LOG_INF("new at base %p size %#zx pages %u struct embedded at %p", + (void *)vr->base, total_size, pages, (void *)vr); + LOG_DBG(" interim size %#zx at %p", interim_size, (void *)vr->interim.heap.heap.init_mem); + LOG_DBG(" lifetime size %#zx at %p", lifetime_size, (void *)vr->lifetime.base); + + return vr; +} + +/** + * @brief Destroy a virtual region instance. + * + * @param[in] vr Pointer to the virtual region instance to destroy. + */ +void vregion_destroy(struct vregion *vr) +{ + if (!vr) + return; + + /* log the vregion being destroyed */ + LOG_DBG("destroy %p size %#zx pages %u", (void *)vr->base, vr->size, vr->pages); + LOG_DBG(" lifetime used %zu free count %d", vr->lifetime.used, vr->lifetime.free_count); + vpage_free(vr->base); +} + +/** + * @brief Allocate memory with alignment from the virtual region dynamic heap. + * + * @param[in] heap Pointer to the heap to use. + * @param[in] size Size of the allocation. + * @param[in] align Alignment of the allocation. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +static void *interim_alloc(struct interim_heap *heap, + size_t size, size_t align) +{ + void *ptr; + + ptr = k_heap_aligned_alloc(&heap->heap, align, size, K_NO_WAIT); + if (!ptr) + LOG_ERR("error: interim alloc failed for %d bytes align %d", + size, align); + + return ptr; +} + +/** + * @brief Free memory from the virtual region interim heap. + * + * @param[in] heap Pointer to the heap to use. + * @param[in] ptr Pointer to the memory to free. + */ +static void interim_free(struct interim_heap *heap, void *ptr) +{ + k_heap_free(&heap->heap, ptr); +} + +/** + * @brief Allocate memory from the virtual region lifetime allocator. + * + * @param[in] heap Pointer to the linear heap to use. + * @param[in] size Size of the allocation. + * @param[in] align Alignment of the allocation. + * + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +static void *lifetime_alloc(struct vlinear_heap *heap, + size_t size, size_t align) +{ + void *ptr; + uint8_t *aligned_ptr; + size_t heap_obj_size; + + /* align heap pointer to alignment requested */ + aligned_ptr = UINT_TO_POINTER(ALIGN_UP(POINTER_TO_UINT(heap->ptr), align)); + + /* also align up size to D$ bytes if asked - allocation head and tail aligned */ + if (align == CONFIG_DCACHE_LINE_SIZE) + size = ALIGN_UP(size, CONFIG_DCACHE_LINE_SIZE); + + /* calculate new heap object size for object and alignment */ + heap_obj_size = aligned_ptr - heap->ptr + size; + + /* check we have enough lifetime space left */ + if (heap_obj_size + heap->used > heap->size) { + LOG_ERR("error: lifetime alloc failed for object %d heap %d bytes free %d", + size, heap_obj_size, heap->size - heap->used); + return NULL; + } + + /* allocate memory */ + ptr = aligned_ptr; + heap->ptr += heap_obj_size; + heap->used += heap_obj_size; + + return ptr; +} + +/** + * @brief Free memory from the virtual region lifetime allocator. + * + * @param[in] heap Pointer to the linear heap to use. + * @param[in] ptr Pointer to the memory to free. + */ +static void lifetime_free(struct vlinear_heap *heap, void *ptr) +{ + /* simple free, just increment free count, this is for tuning only */ + heap->free_count++; + + LOG_DBG("lifetime free %p count %d", ptr, heap->free_count); +} + +/** + * @brief Free memory from the virtual region. + * + * @param vr Pointer to the virtual region instance. + * @param ptr Pointer to the memory to free. + */ +void vregion_free(struct vregion *vr, void *ptr) +{ + if (!vr || !ptr) + return; + + /* check if pointer is in interim heap */ + if (ptr >= (void *)vr->interim.heap.heap.init_mem && + ptr < (void *)((uint8_t *)vr->interim.heap.heap.init_mem + + vr->interim.heap.heap.init_bytes)) { + interim_free(&vr->interim, ptr); + return; + } + + /* check if pointer is in lifetime heap */ + if (ptr >= (void *)vr->lifetime.base && + ptr < (void *)(vr->lifetime.base + vr->lifetime.size)) { + lifetime_free(&vr->lifetime, ptr); + return; + } + + LOG_ERR("error: vregion free invalid pointer %p", ptr); +} +EXPORT_SYMBOL(vregion_free); + +/** + * @brief Allocate memory type from the virtual region. + * + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Memory type to allocate. + * @param[in] size Size of the allocation. + * @param[in] alignment Alignment of the allocation. + * + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc_align(struct vregion *vr, enum vregion_mem_type type, + size_t size, size_t alignment) +{ + if (!vr || !size) + return NULL; + + if (!alignment) + alignment = 4; /* default align 4 bytes */ + + switch (type) { + case VREGION_MEM_TYPE_INTERIM: + return interim_alloc(&vr->interim, size, alignment); + case VREGION_MEM_TYPE_LIFETIME: + return lifetime_alloc(&vr->lifetime, size, alignment); + default: + LOG_ERR("error: invalid memory type %d", type); + return NULL; + } +} +EXPORT_SYMBOL(vregion_alloc_align); + +/** + * @brief Allocate memory from the virtual region. + * @param[in] vr Pointer to the virtual region instance. + * @param[in] type Memory type to allocate. + * @param[in] size Size of the allocation. + * @return void* Pointer to the allocated memory, or NULL on failure. + */ +void *vregion_alloc(struct vregion *vr, enum vregion_mem_type type, size_t size) +{ + return vregion_alloc_align(vr, type, size, 0); +} +EXPORT_SYMBOL(vregion_alloc); + +/** + * @brief Log virtual region memory usage. + * + * @param[in] vr Pointer to the virtual region instance. + */ +void vregion_info(struct vregion *vr) +{ + if (!vr) + return; + + LOG_INF("base %p size %#zx pages %u", + (void *)vr->base, vr->size, vr->pages); + LOG_INF("lifetime used %#zx free count %d", + vr->lifetime.used, vr->lifetime.free_count); +} +EXPORT_SYMBOL(vregion_info); From 7ec12693511d44f9f6d67bec5a6ce174cdbf811e Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 20 Feb 2026 12:51:44 +0100 Subject: [PATCH 3/4] debug: ztest: reduce delay between tests Currently ztest delay is set to 100ms, which adds that time between each too ztest runs. This very quickly adds up to cause an IPC timeout in the kernel driver. That timeout isn't needed for SOF, set it to 1ms. Signed-off-by: Guennadi Liakhovetski --- app/debug_overlay.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/app/debug_overlay.conf b/app/debug_overlay.conf index 4cb2bc767023..e99910ebfff1 100644 --- a/app/debug_overlay.conf +++ b/app/debug_overlay.conf @@ -3,6 +3,7 @@ CONFIG_ASSERT=y CONFIG_ZTEST_NO_YIELD=n CONFIG_ZTEST_SUMMARY=n +CONFIG_ZTEST_TEST_DELAY_MS=1 CONFIG_SOF_BOOT_TEST_ALLOWED=y CONFIG_TEST_EXTRA_STACK_SIZE=7168 From dfbc8081344a9968c23ee3c543ed5d696a196273 Mon Sep 17 00:00:00 2001 From: Guennadi Liakhovetski Date: Fri, 20 Mar 2026 14:42:40 +0100 Subject: [PATCH 4/4] test: add vpage and vregion ztests Add boot-time tests for basic vpage and vregion functionality. Signed-off-by: Guennadi Liakhovetski --- zephyr/test/CMakeLists.txt | 3 ++ zephyr/test/vpage.c | 36 +++++++++++++++ zephyr/test/vregion.c | 92 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 zephyr/test/vpage.c create mode 100644 zephyr/test/vregion.c diff --git a/zephyr/test/CMakeLists.txt b/zephyr/test/CMakeLists.txt index c5b66c83bbaa..f548c98c5e73 100644 --- a/zephyr/test/CMakeLists.txt +++ b/zephyr/test/CMakeLists.txt @@ -2,6 +2,9 @@ if(CONFIG_SOF_BOOT_TEST) zephyr_library_sources_ifdef(CONFIG_VIRTUAL_HEAP vmh.c ) + zephyr_library_sources_ifdef(CONFIG_SOF_VREGIONS + vpage.c vregion.c + ) zephyr_library_sources_ifdef(CONFIG_USERSPACE userspace/ksem.c ) diff --git a/zephyr/test/vpage.c b/zephyr/test/vpage.c new file mode 100644 index 000000000000..038e299eba70 --- /dev/null +++ b/zephyr/test/vpage.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2026 Intel Corporation. + */ + +#include +#include +#include + +#include +#include + +#include +#include + +LOG_MODULE_DECLARE(sof_boot_test, CONFIG_SOF_LOG_LEVEL); + +ZTEST(sof_boot, vpage) +{ + void *p1 = vpage_alloc(1); + + zassert_not_null(p1); + + void *p2 = vpage_alloc(2); + + zassert_not_null(p2); + + vpage_free(p1); + vpage_free(p2); + + p1 = vpage_alloc(2); + + zassert_not_null(p1); + + vpage_free(p1); +} diff --git a/zephyr/test/vregion.c b/zephyr/test/vregion.c new file mode 100644 index 000000000000..65586d690494 --- /dev/null +++ b/zephyr/test/vregion.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2026 Intel Corporation. + */ + +#include +#include +#include + +#include +#include + +#include +#include + +LOG_MODULE_DECLARE(sof_boot_test, CONFIG_SOF_LOG_LEVEL); + +static struct vregion *test_vreg_create(void) +{ + struct vregion *vreg = vregion_create(CONFIG_MM_DRV_PAGE_SIZE - 100, + CONFIG_MM_DRV_PAGE_SIZE); + + zassert_not_null(vreg); + + return vreg; +} + +static void test_vreg_alloc_lifet(struct vregion *vreg) +{ + void *ptr = vregion_alloc(vreg, VREGION_MEM_TYPE_LIFETIME, 2000); + + zassert_not_null(ptr); + + void *ptr_align = vregion_alloc_align(vreg, VREGION_MEM_TYPE_LIFETIME, 2000, 16); + + zassert_not_null(ptr_align); + zassert_equal((uintptr_t)ptr_align & 15, 0); + + void *ptr_nomem = vregion_alloc(vreg, VREGION_MEM_TYPE_LIFETIME, 2000); + + zassert_is_null(ptr_nomem); + + vregion_free(vreg, ptr_align); + vregion_free(vreg, ptr); + + /* Freeing isn't possible with LIFETIME */ + ptr_nomem = vregion_alloc(vreg, VREGION_MEM_TYPE_LIFETIME, 2000); + + zassert_is_null(ptr_nomem); +} + +static void test_vreg_alloc_tmp(struct vregion *vreg) +{ + void *ptr = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, 20); + + zassert_not_null(ptr); + + void *ptr_align = vregion_alloc_align(vreg, VREGION_MEM_TYPE_INTERIM, 2000, 16); + + zassert_not_null(ptr_align); + zassert_equal((uintptr_t)ptr_align & 15, 0); + + void *ptr_nomem = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, 2000); + + zassert_is_null(ptr_nomem); + + vregion_free(vreg, ptr_align); + vregion_free(vreg, ptr); + + /* Should be possible to allocate again */ + ptr = vregion_alloc(vreg, VREGION_MEM_TYPE_INTERIM, 2000); + + zassert_not_null(ptr); +} + +static void test_vreg_destroy(struct vregion *vreg) +{ + vregion_info(vreg); + vregion_destroy(vreg); +} + +ZTEST(sof_boot, vregion) +{ + struct vregion *vreg = test_vreg_create(); + + /* Test interim allocations */ + test_vreg_alloc_tmp(vreg); + /* Test lifetime allocations */ + test_vreg_alloc_lifet(vreg); + + test_vreg_destroy(vreg); +}