From a5fb8e3e679839e6889f57b7784d8963e50f8a49 Mon Sep 17 00:00:00 2001 From: Arkadiusz Jedrzejewski Date: Fri, 10 Jul 2026 15:51:28 +0200 Subject: [PATCH 1/2] allocator: add arena allocator - Add arena allocator implementation. - Internally ref-counted and Send+Sync. - Add mmap-based memory region provider. - Add loom tests support for `allocator`. - `AllocationError`: - Add `InvalidLayout`. - Add `PartialEq` and `Eq` to make it comparable. --- Cargo.lock | 310 ++++++++++++++++++ score/allocator/BUILD | 17 + score/allocator/Cargo.toml | 4 + score/allocator/allocator_traits.rs | 5 +- score/allocator/arena_allocator.rs | 486 ++++++++++++++++++++++++++++ score/allocator/atomic.rs | 20 ++ score/allocator/lib.rs | 5 + score/allocator/memory_region.rs | 163 ++++++++++ score/pal/lib.rs | 32 ++ 9 files changed, 1041 insertions(+), 1 deletion(-) create mode 100644 score/allocator/arena_allocator.rs create mode 100644 score/allocator/atomic.rs create mode 100644 score/allocator/memory_region.rs diff --git a/Cargo.lock b/Cargo.lock index 613455cd26..4050ee2d66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,13 +2,40 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "allocator" version = "0.0.1" dependencies = [ + "loom", + "pal", "score_log", ] +[[package]] +name = "cc" +version = "1.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d6cac793997bd970000024b2934968efe83b382de4fdcf4fcb46b6ee4ad996" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + [[package]] name = "containers" version = "0.0.1" @@ -17,6 +44,51 @@ dependencies = [ "score_log", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + [[package]] name = "log_builtin" version = "0.0.1" @@ -32,6 +104,51 @@ dependencies = [ "score_log", ] +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "serde", + "serde_json", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "pal" version = "0.0.1" @@ -39,6 +156,12 @@ dependencies = [ "score_log", ] +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "proc-macro2" version = "1.0.106" @@ -57,6 +180,35 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "score_log" version = "0.0.1" @@ -89,6 +241,70 @@ dependencies = [ "syn", ] +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + [[package]] name = "stdout_logger" version = "0.0.1" @@ -123,8 +339,102 @@ dependencies = [ "score_log", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + [[package]] name = "unicode-ident" version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/score/allocator/BUILD b/score/allocator/BUILD index 87e481d316..9c55e980e6 100644 --- a/score/allocator/BUILD +++ b/score/allocator/BUILD @@ -20,6 +20,7 @@ rust_library( visibility = ["//visibility:public"], deps = [ "//score/log_rust/score_log", + "//score/pal", ], ) @@ -31,3 +32,19 @@ rust_test( "ut", ], ) + +rust_test( + name = "loom_tests", + crate = "allocator", + rustc_flags = [ + "--cfg", + "loom", + ], + tags = [ + "loom", + ], + target_compatible_with = ["@platforms//os:linux"], + deps = [ + "@score_crates//:loom", + ], +) diff --git a/score/allocator/Cargo.toml b/score/allocator/Cargo.toml index 7218bc2521..9abfb7b7ee 100644 --- a/score/allocator/Cargo.toml +++ b/score/allocator/Cargo.toml @@ -23,7 +23,11 @@ license-file.workspace = true path = "lib.rs" [dependencies] +pal.workspace = true score_log.workspace = true +[target.'cfg(loom)'.dependencies] +loom = { version = "0.7.2", features = ["checkpoint"] } + [lints] workspace = true diff --git a/score/allocator/allocator_traits.rs b/score/allocator/allocator_traits.rs index 676c80a763..f087670384 100644 --- a/score/allocator/allocator_traits.rs +++ b/score/allocator/allocator_traits.rs @@ -16,7 +16,7 @@ use core::ptr::NonNull; use score_log::ScoreDebug; /// Allocation errors. -#[derive(Debug, ScoreDebug, Clone, Copy)] +#[derive(Debug, ScoreDebug, Clone, Copy, PartialEq, Eq)] pub enum AllocationError { /// Memory allocation failed due to insufficient memory. OutOfMemory, @@ -24,6 +24,9 @@ pub enum AllocationError { /// Memory allocation requested for zero size. ZeroSizeAllocation, + /// Invalid layout provided. + InvalidLayout, + /// Fatal failure inside the allocator. Internal, } diff --git a/score/allocator/arena_allocator.rs b/score/allocator/arena_allocator.rs new file mode 100644 index 0000000000..39f5bc7790 --- /dev/null +++ b/score/allocator/arena_allocator.rs @@ -0,0 +1,486 @@ +// ******************************************************************************* +// Copyright (c) 2026 Contributors to the Eclipse Foundation +// +// See the NOTICE file(s) distributed with this work for additional +// information regarding copyright ownership. +// +// This program and the accompanying materials are made available under the +// terms of the Apache License Version 2.0 which is available at +// +// +// SPDX-License-Identifier: Apache-2.0 +// ******************************************************************************* + +//! Arena allocator backed by a [`MemoryRegion`]. + +// TODO: safety clauses. + +use crate::atomic::{fence, AtomicUsize, Ordering}; +use crate::memory_region::{align_to, MemoryRegion}; +use crate::{AllocationError, BasicAllocator}; +use core::alloc::Layout; +use core::fmt as core_fmt; +use core::mem::ManuallyDrop; +use core::ptr::NonNull; +use score_log::fmt as score_fmt; + +/// Arena header, keeps track of living object references and current pointer position. +struct ArenaHeader { + ref_count: AtomicUsize, + offset: AtomicUsize, +} + +/// Arena allocator backed by a [`MemoryRegion`]. +/// +/// Allocated chunks are from a continuous chunk of memory provided by the memory region. +/// Deallocation is a no-op. +/// +/// Internal storage is reference counted. +/// Cloning provides a new handle to same data. +pub struct ArenaAllocator { + /// Memory region, contains [`ArenaHeader`] at its start. + /// It is wrapped in [`ManuallyDrop`] to ensure it is dropped once. + memory_region: ManuallyDrop, +} + +// SAFETY: only arena header contains mutable state, and it's always accessed using atomics. +unsafe impl Send for ArenaAllocator {} +unsafe impl Sync for ArenaAllocator {} + +impl ArenaAllocator { + /// Create a new allocator using given memory region. + /// Allocator own data is written at the region start. + pub fn new(memory_region: R) -> Result { + // Disallow memory region smaller than size of `ArenaHeader`. + if memory_region.layout().size() < size_of::() { + return Err(AllocationError::InvalidLayout); + } + // Disallow memory region alignments causing an offset. + if memory_region.ptr().as_ptr().align_offset(align_of::()) != 0 { + return Err(AllocationError::InvalidLayout); + } + + // Initialize the header at the start of the memory region. + // Offset is set with regard to header size. + // SAFETY: bounds and alignment checked above. + unsafe { + memory_region.ptr().cast::().as_ptr().write(ArenaHeader { + ref_count: AtomicUsize::new(1), + offset: AtomicUsize::new(size_of::()), + }); + } + + Ok(Self { + memory_region: ManuallyDrop::new(memory_region), + }) + } + + /// Access the header. + fn header(&self) -> &ArenaHeader { + // SAFETY: header remains valid for the whole lifetime of the object. + unsafe { self.memory_region.ptr().cast::().as_ref() } + } + + /// Capacity of the allocator. + pub fn capacity(&self) -> usize { + self.memory_region.layout().size() + } +} + +impl Clone for ArenaAllocator { + fn clone(&self) -> Self { + // Increase reference count. + self.header().ref_count.fetch_add(1, Ordering::Relaxed); + // SAFETY: `MemoryRegion` implementations are obliged to be bitwise copyable. + let memory_region = ManuallyDrop::new(unsafe { core::ptr::read(&*self.memory_region) }); + Self { memory_region } + } +} + +impl Drop for ArenaAllocator { + fn drop(&mut self) { + // Release the region on last reference dropping. + if self.header().ref_count.fetch_sub(1, Ordering::Release) == 1 { + fence(Ordering::Acquire); + // Drop the header before its memory is removed. + // SAFETY: header remains valid for the whole lifetime of the object. + unsafe { core::ptr::drop_in_place(self.memory_region.ptr().cast::().as_ptr()) }; + // SAFETY: memory region is dropped exactly once. + unsafe { ManuallyDrop::drop(&mut self.memory_region) }; + } + } +} + +impl BasicAllocator for ArenaAllocator { + fn allocate(&self, layout: Layout) -> Result, AllocationError> { + // Disallow zero size allocation. + if layout.size() == 0 { + return Err(AllocationError::ZeroSizeAllocation); + } + + // Get aligned size. + let aligned_size = align_to(layout.size(), layout.align()); + + // Bump the offset with compare-and-swap loop. + // Concurrent allocations must receive non-overlapping regions. + let offset = &self.header().offset; + let mut current_offset = offset.load(Ordering::Relaxed); + let ptr_as_non_null = loop { + // Align current position and check the allocation will not exceed capacity. + let aligned_offset = align_to(current_offset, layout.align()); + let new_offset = aligned_offset + aligned_size; + if new_offset >= self.capacity() { + return Err(AllocationError::OutOfMemory); + } + + match offset.compare_exchange(current_offset, new_offset, Ordering::Relaxed, Ordering::Relaxed) { + Ok(_) => { + break unsafe { + let ptr_with_offset = self.memory_region.ptr().as_ptr().byte_offset(aligned_offset as isize); + NonNull::new_unchecked(ptr_with_offset) + }; + }, + Err(actual) => current_offset = actual, + } + }; + + Ok(ptr_as_non_null) + } + + unsafe fn deallocate(&self, _ptr: NonNull, _layout: Layout) {} +} + +impl core_fmt::Debug for ArenaAllocator { + fn fmt(&self, f: &mut core_fmt::Formatter<'_>) -> core_fmt::Result { + f.debug_struct("ArenaAllocator") + .field("capacity", &self.capacity()) + .finish() + } +} + +impl score_fmt::ScoreDebug for ArenaAllocator { + fn fmt(&self, f: score_fmt::Writer, spec: &score_fmt::FormatSpec) -> score_fmt::Result { + score_fmt::DebugStruct::new(f, spec, "ArenaAllocator") + .field("capacity", &self.capacity()) + .finish() + } +} + +#[cfg(all(test, not(loom)))] +mod tests { + use crate::arena_allocator::ArenaHeader; + use crate::atomic::Ordering; + use crate::memory_region::{MemoryRegion, MmapMemoryRegion}; + use crate::{page_size, AllocationError, ArenaAllocator, BasicAllocator}; + use core::alloc::Layout; + use core::unimplemented; + use std::collections::BTreeSet; + use std::sync::Arc; + use std::thread; + + const HEADER_SIZE: usize = size_of::(); + + #[test] + fn test_new_ok() { + let capacity = page_size(); + let memory_region = MmapMemoryRegion::new(capacity).unwrap(); + let result = ArenaAllocator::new(memory_region); + assert!(result.is_ok()); + } + + #[test] + fn test_new_zero() { + struct MockMemoryRegion; + + unsafe impl MemoryRegion for MockMemoryRegion { + fn ptr(&self) -> core::ptr::NonNull { + unimplemented!() + } + + fn layout(&self) -> Layout { + Layout::from_size_align(0, 4).unwrap() + } + } + + let memory_region = MockMemoryRegion; + let result = ArenaAllocator::new(memory_region); + assert!(result.is_err_and(|e| e == AllocationError::InvalidLayout)); + } + + #[test] + fn test_new_unaligned() { + struct MockMemoryRegion; + + unsafe impl MemoryRegion for MockMemoryRegion { + fn ptr(&self) -> core::ptr::NonNull { + // Address is deliberately misaligned for `ArenaHeader`. + // It is never dereferenced. + #[allow(clippy::manual_dangling_ptr, reason = "pointer is deliberately misaligned")] + unsafe { + core::ptr::NonNull::new_unchecked(1 as *mut u8) + } + } + + fn layout(&self) -> Layout { + // Large enough to pass the size check and reach the alignment check. + Layout::from_size_align(HEADER_SIZE, 1).unwrap() + } + } + + let memory_region = MockMemoryRegion; + let result = ArenaAllocator::new(memory_region); + assert!(result.is_err_and(|e| e == AllocationError::InvalidLayout)); + } + + #[test] + fn test_capacity_ok() { + let capacity = page_size(); + let memory_region = MmapMemoryRegion::new(capacity).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + assert_eq!(allocator.capacity(), capacity); + } + + #[test] + fn test_allocate_single() { + let capacity = 0x1000; + let memory_region = MmapMemoryRegion::new(capacity).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + + // Allocate. + let layout = Layout::from_size_align(253, 4).unwrap(); + let result = allocator.allocate(layout); + assert!(result.is_ok()); + + // Check offset. + assert_eq!(allocator.header().offset.load(Ordering::Relaxed), HEADER_SIZE + 256); + } + + #[test] + fn test_allocate_multiple() { + let capacity = 0x1000; + let memory_region = MmapMemoryRegion::new(capacity).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + + // Cases to check, contains: + // - layout + // - allocation pointer offset from memory region start + // - internal allocator offset + let cases = vec![ + (Layout::from_size_align(253, 1).unwrap(), HEADER_SIZE, HEADER_SIZE + 253), + ( + Layout::from_size_align(557, 2).unwrap(), + HEADER_SIZE + 254, + HEADER_SIZE + 812, + ), + ( + Layout::from_size_align(39, 8).unwrap(), + HEADER_SIZE + 816, + HEADER_SIZE + 856, + ), + ]; + + let memory_region_ptr = allocator.memory_region.ptr().as_ptr(); + for (layout, exp_ptr_offset, exp_allocator_offset) in cases { + // Allocate. + let alloc_ptr = allocator.allocate(layout).unwrap().as_ptr(); + + // Check position relative to memory region and allocator offset. + let allocation_ptr_offset = unsafe { alloc_ptr.offset_from(memory_region_ptr) }; + assert_eq!(allocation_ptr_offset, exp_ptr_offset as isize); + assert_eq!(allocator.header().offset.load(Ordering::Relaxed), exp_allocator_offset); + } + } + + #[test] + fn test_allocate_oom() { + let capacity = page_size(); + let memory_region = MmapMemoryRegion::new(capacity).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + + // Allocate. + let layout = Layout::from_size_align(2 * page_size(), 4).unwrap(); + let result = allocator.allocate(layout); + assert!(result.is_err_and(|e| e == AllocationError::OutOfMemory)); + + // Check offset. + assert_eq!(allocator.header().offset.load(Ordering::Relaxed), HEADER_SIZE); + } + + #[test] + fn test_allocate_zero() { + let capacity = page_size(); + let memory_region = MmapMemoryRegion::new(capacity).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + + // Allocate. + let layout = Layout::from_size_align(0, 4).unwrap(); + let result = allocator.allocate(layout); + assert!(result.is_err_and(|e| e == AllocationError::ZeroSizeAllocation)); + + // Check offset. + assert_eq!(allocator.header().offset.load(Ordering::Relaxed), HEADER_SIZE); + } + + #[test] + fn test_deallocate_panic() { + let capacity = page_size(); + let memory_region = MmapMemoryRegion::new(capacity).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + + // Allocate. + let layout = Layout::from_size_align(253, 4).unwrap(); + let ptr = allocator.allocate(layout).unwrap(); + + // Deallocate - no-op. + unsafe { allocator.deallocate(ptr, layout) }; + } + + #[test] + fn test_clone_shared_offset() { + // Create allocator and its clone. + let capacity = page_size(); + let memory_region = MmapMemoryRegion::new(capacity).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + let clone = allocator.clone(); + + // Allocate using cloned object. + let layout = Layout::from_size_align(253, 4).unwrap(); + clone.allocate(layout).unwrap(); + + // Check offset is bumped for both objects. + let expected = HEADER_SIZE + 256; + assert_eq!(clone.header().offset.load(Ordering::Relaxed), expected); + assert_eq!(allocator.header().offset.load(Ordering::Relaxed), expected); + } + + #[test] + fn test_clone_ref_count() { + let memory_region = MmapMemoryRegion::new(page_size()).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + assert_eq!(allocator.header().ref_count.load(Ordering::Relaxed), 1); + + let clone = allocator.clone(); + assert_eq!(allocator.header().ref_count.load(Ordering::Relaxed), 2); + + drop(clone); + assert_eq!(allocator.header().ref_count.load(Ordering::Relaxed), 1); + } + + #[test] + fn test_clone_single_unmap() { + // Check no panic happens when both instances are dropped. + let memory_region = MmapMemoryRegion::new(page_size()).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + let clone = allocator.clone(); + drop(allocator); + drop(clone); + } + + #[test] + fn test_concurrent_allocate_distinct_regions() { + // Perform allocations from multiple threads to check every region is distinct and in-bounds. + const THREADS: usize = 8; + const PER_THREAD: usize = 32; + let capacity = 4 * page_size(); + let memory_region = MmapMemoryRegion::new(capacity).unwrap(); + let allocator = Arc::new(ArenaAllocator::new(memory_region).unwrap()); + let layout = Layout::from_size_align(16, 8).unwrap(); + + let handles: Vec<_> = (0..THREADS) + .map(|_| { + let allocator = allocator.clone(); + thread::spawn(move || { + let mut ptrs = Vec::with_capacity(PER_THREAD); + for _ in 0..PER_THREAD { + ptrs.push(allocator.allocate(layout).unwrap().as_ptr() as usize); + } + ptrs + }) + }) + .collect(); + + let base = allocator.memory_region.ptr().as_ptr() as usize; + let mut all = BTreeSet::new(); + for handle in handles { + for addr in handle.join().unwrap() { + // In-bounds and (with size 16) non-overlapping given distinct start addresses. + assert!(addr >= base && addr < base + capacity); + assert!(all.insert(addr), "region handed out twice: {addr:#x}"); + } + } + assert_eq!(all.len(), THREADS * PER_THREAD); + } + + #[test] + fn test_concurrent_clone_drop_single_unmap() { + // Perform clones and drops in multiple threads. + // The original is dropped last. + let memory_region = MmapMemoryRegion::new(page_size()).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + let shared = Arc::new(allocator.clone()); + + let handles: Vec<_> = (0..8) + .map(|_| { + let shared = shared.clone(); + thread::spawn(move || { + for _ in 0..64 { + let c = (*shared).clone(); + drop(c); + } + }) + }) + .collect(); + for handle in handles { + handle.join().unwrap(); + } + + drop(Arc::try_unwrap(shared).ok().unwrap()); + assert_eq!(allocator.header().ref_count.load(Ordering::Relaxed), 1); + } +} + +#[cfg(all(test, loom))] +mod loom_tests { + use crate::memory_region::MmapMemoryRegion; + use crate::{page_size, ArenaAllocator, BasicAllocator}; + use core::alloc::Layout; + use loom::thread; + + #[test] + fn loom_concurrent_allocate_distinct() { + // Perform two concurrent allocations on a shared arena. + loom::model(|| { + let memory_region = MmapMemoryRegion::new(page_size()).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + let layout = Layout::from_size_align(16, 8).unwrap(); + + let a = allocator.clone(); + let b = allocator.clone(); + let h1 = thread::spawn(move || a.allocate(layout).unwrap().as_ptr() as usize); + let h2 = thread::spawn(move || b.allocate(layout).unwrap().as_ptr() as usize); + + let p1 = h1.join().unwrap(); + let p2 = h2.join().unwrap(); + assert_ne!(p1, p2); + // Size is 16, so distinct start addresses cannot overlap. + assert!(p1.abs_diff(p2) >= 16); + }); + } + + #[test] + fn loom_concurrent_clone_drop() { + // Perform two concurrent drops on a shared handle. + loom::model(|| { + let memory_region = MmapMemoryRegion::new(page_size()).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + + let a = allocator.clone(); + let b = allocator.clone(); + let h1 = thread::spawn(move || drop(a)); + let h2 = thread::spawn(move || drop(b)); + h1.join().unwrap(); + h2.join().unwrap(); + // `allocator` drops here as the last reference. + }); + } +} diff --git a/score/allocator/atomic.rs b/score/allocator/atomic.rs new file mode 100644 index 0000000000..d5a2b491da --- /dev/null +++ b/score/allocator/atomic.rs @@ -0,0 +1,20 @@ +// ******************************************************************************* +// Copyright (c) 2026 Contributors to the Eclipse Foundation +// +// See the NOTICE file(s) distributed with this work for additional +// information regarding copyright ownership. +// +// This program and the accompanying materials are made available under the +// terms of the Apache License Version 2.0 which is available at +// +// +// SPDX-License-Identifier: Apache-2.0 +// ******************************************************************************* + +//! Atomic primitives re-export. +//! Required for testing with Loom. + +#[cfg(not(loom))] +pub(crate) use core::sync::atomic::{fence, AtomicUsize, Ordering}; +#[cfg(loom)] +pub(crate) use loom::sync::atomic::{fence, AtomicUsize, Ordering}; diff --git a/score/allocator/lib.rs b/score/allocator/lib.rs index 0b04b933b6..0feadfdd70 100644 --- a/score/allocator/lib.rs +++ b/score/allocator/lib.rs @@ -14,7 +14,12 @@ //! Custom allocator support. mod allocator_traits; +mod arena_allocator; +mod atomic; mod heap_allocator; +mod memory_region; pub use allocator_traits::{AllocationError, BasicAllocator}; +pub use arena_allocator::ArenaAllocator; pub use heap_allocator::{HeapAllocator, GLOBAL_ALLOCATOR}; +pub use memory_region::{page_size, MemoryRegion, MmapMemoryRegion}; diff --git a/score/allocator/memory_region.rs b/score/allocator/memory_region.rs new file mode 100644 index 0000000000..772776f281 --- /dev/null +++ b/score/allocator/memory_region.rs @@ -0,0 +1,163 @@ +// ******************************************************************************* +// Copyright (c) 2026 Contributors to the Eclipse Foundation +// +// See the NOTICE file(s) distributed with this work for additional +// information regarding copyright ownership. +// +// This program and the accompanying materials are made available under the +// terms of the Apache License Version 2.0 which is available at +// +// +// SPDX-License-Identifier: Apache-2.0 +// ******************************************************************************* + +//! Memory region. + +use crate::AllocationError; +use core::alloc::Layout; +use core::ptr::{null_mut, NonNull}; +use pal::{errno, mmap, munmap, sysconf, MAP_ANONYMOUS, MAP_PRIVATE, PROT_READ, PROT_WRITE, _SC_PAGESIZE}; + +/// Get current page size. +pub fn page_size() -> usize { + // SAFETY: both `sysconf` and `_SC_PAGESIZE` are ensured to be valid. + let result = unsafe { sysconf(_SC_PAGESIZE) }; + assert!(result >= 1, "page size must not be less than 1"); + result as usize +} + +/// Get aligned value - rounded up to a multiplication of the second value. +pub(crate) fn align_to(value: usize, align: usize) -> usize { + let mut v_div = value / align; + if !value.is_multiple_of(align) { + v_div += 1; + } + v_div * align +} + +/// Memory region interface. +/// +/// # Safety +/// +/// - Object must be safe to bitwise-copy. +/// - `ptr()` and `layout()` must return same values over the lifetime of the object. +/// - Underlying storage must be released on drop. +pub unsafe trait MemoryRegion { + /// Pointer to region start. + fn ptr(&self) -> NonNull; + /// Layout of the region. + fn layout(&self) -> Layout; +} + +/// mmap-backed memory region. +pub struct MmapMemoryRegion { + ptr: NonNull, + layout: Layout, +} + +impl MmapMemoryRegion { + /// Create new memory region using mmap. + /// Capacity of the region is rounded up to the nearest multiplication of a page size. + pub fn new(capacity: usize) -> Result { + // Disallow zero size allocation. + if capacity == 0 { + return Err(AllocationError::ZeroSizeAllocation); + } + + // Determine final capacity with regard to page size. + let page_size = page_size(); + let capacity = align_to(capacity, page_size); + + // Create a memory mapping. + let prot = PROT_READ | PROT_WRITE; + let flags = MAP_PRIVATE | MAP_ANONYMOUS; + let fd = -1; + // SAFETY: parameters are ensured to be valid. + let ptr: *mut u8 = unsafe { mmap(null_mut(), capacity, prot, flags, fd, 0) }.cast(); + if ptr as isize == -1 || ptr.is_null() { + // TODO: information about errno is missing. + return Err(AllocationError::Internal); + } + + // SAFETY: checked against null above. + let ptr = unsafe { NonNull::new_unchecked(ptr) }; + // Create layout - aligned to page size, capacity is rounded up to fill the page. + let layout = Layout::from_size_align(capacity, page_size).map_err(|_| AllocationError::Internal)?; + + Ok(Self { ptr, layout }) + } +} + +impl Drop for MmapMemoryRegion { + fn drop(&mut self) { + // SAFETY: `ptr` and `layout` come from a successful `mmap` in `new`. + let rc = unsafe { munmap(self.ptr.as_ptr().cast(), self.layout.size()) }; + if rc != 0 { + let errno = errno(); + panic!("munmap failed, rc: {rc}, errno: {errno}"); + } + } +} + +unsafe impl MemoryRegion for MmapMemoryRegion { + fn ptr(&self) -> NonNull { + self.ptr + } + + fn layout(&self) -> Layout { + self.layout + } +} + +// SAFETY: region contains process global memory, it's safe to send and sync as long as object is alive. +unsafe impl Send for MmapMemoryRegion {} +unsafe impl Sync for MmapMemoryRegion {} + +#[cfg(all(test, not(loom)))] +mod tests { + use crate::memory_region::{page_size, MemoryRegion, MmapMemoryRegion}; + use crate::AllocationError; + + #[test] + fn test_page_size_ok() { + // Check no panic, exact size is platform-specific. + let _ = page_size(); + } + + #[test] + fn test_new_ok() { + let region = MmapMemoryRegion::new(page_size()).unwrap(); + assert_eq!(region.layout().size(), page_size()); + assert_eq!(region.layout().align(), page_size()); + } + + #[test] + fn test_new_zero() { + let result = MmapMemoryRegion::new(0); + assert!(result.is_err_and(|e| e == AllocationError::ZeroSizeAllocation)); + } + + #[test] + fn test_new_capacity_aligned_to_page_size() { + let capacity = 4 * page_size(); + let region = MmapMemoryRegion::new(capacity).unwrap(); + assert_eq!(region.layout().size(), capacity); + } + + #[test] + fn test_new_capacity_rounds_up() { + // Slightly over one page size - rounds up to two page sizes. + let region = MmapMemoryRegion::new(page_size() + 1).unwrap(); + assert_eq!(region.layout().size(), 2 * page_size()); + + // A single byte - rounds up to a full page size. + let region = MmapMemoryRegion::new(1).unwrap(); + assert_eq!(region.layout().size(), page_size()); + } + + #[test] + fn test_send_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + } +} diff --git a/score/pal/lib.rs b/score/pal/lib.rs index 8397f844a7..92d66360c3 100644 --- a/score/pal/lib.rs +++ b/score/pal/lib.rs @@ -27,6 +27,7 @@ pub type c_long = core::ffi::c_long; pub type c_ulong = core::ffi::c_ulong; pub type size_t = usize; pub type c_void = core::ffi::c_void; +pub type off_t = i64; pub type pid_t = i32; pub type pthread_t = c_ulong; @@ -61,6 +62,8 @@ pub struct sched_param { } extern "C" { + pub fn mmap(addr: *mut c_void, len: size_t, prot: c_int, flags: c_int, fd: c_int, offset: off_t) -> *mut c_void; + pub fn munmap(addr: *mut c_void, len: size_t) -> c_int; pub fn sysconf(__name: c_int) -> c_long; pub fn pthread_self() -> pthread_t; pub fn pthread_join(native: pthread_t, value: *mut *mut c_void) -> c_int; @@ -98,3 +101,32 @@ pub const PTHREAD_INHERIT_SCHED: c_int = 0; pub const PTHREAD_EXPLICIT_SCHED: c_int = 1; #[cfg(target_os = "nto")] pub const PTHREAD_EXPLICIT_SCHED: c_int = 2; + +#[cfg(target_os = "linux")] +pub const _SC_PAGESIZE: c_int = 30; +#[cfg(target_os = "nto")] +pub const _SC_PAGESIZE: c_int = 11; + +#[cfg(target_os = "linux")] +pub const PROT_NONE: c_int = 0; +#[cfg(target_os = "linux")] +pub const PROT_READ: c_int = 1; +#[cfg(target_os = "linux")] +pub const PROT_WRITE: c_int = 2; +#[cfg(target_os = "linux")] +pub const PROT_EXEC: c_int = 4; + +#[cfg(target_os = "nto")] +pub const PROT_NONE: c_int = 0x00000000; +#[cfg(target_os = "nto")] +pub const PROT_READ: c_int = 0x00000100; +#[cfg(target_os = "nto")] +pub const PROT_WRITE: c_int = 0x00000200; +#[cfg(target_os = "nto")] +pub const PROT_EXEC: c_int = 0x00000400; + +pub const MAP_PRIVATE: c_int = 0x0002; +#[cfg(target_os = "linux")] +pub const MAP_ANONYMOUS: c_int = 0x0020; +#[cfg(target_os = "nto")] +pub const MAP_ANONYMOUS: c_int = 0x00080000; From 3a9349a53959e788d284ef85d00055cd8d8e56ce Mon Sep 17 00:00:00 2001 From: Arkadiusz Jedrzejewski Date: Tue, 14 Jul 2026 09:15:30 +0200 Subject: [PATCH 2/2] allocator: memory pool - Fixed capacity pool of equally-sized elements. - Add `score_log` support for `Layout` and `LayoutError`. --- score/allocator/atomic.rs | 4 +- score/allocator/lib.rs | 2 + score/allocator/memory_pool.rs | 658 +++++++++++++++++++++++ score/log_rust/score_log_fmt/fmt_impl.rs | 36 ++ 4 files changed, 698 insertions(+), 2 deletions(-) create mode 100644 score/allocator/memory_pool.rs diff --git a/score/allocator/atomic.rs b/score/allocator/atomic.rs index d5a2b491da..4fa4aee561 100644 --- a/score/allocator/atomic.rs +++ b/score/allocator/atomic.rs @@ -15,6 +15,6 @@ //! Required for testing with Loom. #[cfg(not(loom))] -pub(crate) use core::sync::atomic::{fence, AtomicUsize, Ordering}; +pub(crate) use core::sync::atomic::{fence, AtomicU8, AtomicUsize, Ordering}; #[cfg(loom)] -pub(crate) use loom::sync::atomic::{fence, AtomicUsize, Ordering}; +pub(crate) use loom::sync::atomic::{fence, AtomicU8, AtomicUsize, Ordering}; diff --git a/score/allocator/lib.rs b/score/allocator/lib.rs index 0feadfdd70..106fb1ba08 100644 --- a/score/allocator/lib.rs +++ b/score/allocator/lib.rs @@ -17,9 +17,11 @@ mod allocator_traits; mod arena_allocator; mod atomic; mod heap_allocator; +mod memory_pool; mod memory_region; pub use allocator_traits::{AllocationError, BasicAllocator}; pub use arena_allocator::ArenaAllocator; pub use heap_allocator::{HeapAllocator, GLOBAL_ALLOCATOR}; +pub use memory_pool::MemoryPool; pub use memory_region::{page_size, MemoryRegion, MmapMemoryRegion}; diff --git a/score/allocator/memory_pool.rs b/score/allocator/memory_pool.rs new file mode 100644 index 0000000000..1d7e8952db --- /dev/null +++ b/score/allocator/memory_pool.rs @@ -0,0 +1,658 @@ +// ******************************************************************************* +// Copyright (c) 2026 Contributors to the Eclipse Foundation +// +// See the NOTICE file(s) distributed with this work for additional +// information regarding copyright ownership. +// +// This program and the accompanying materials are made available under the +// terms of the Apache License Version 2.0 which is available at +// +// +// SPDX-License-Identifier: Apache-2.0 +// ******************************************************************************* + +//! Memory pool. + +use crate::atomic::{fence, AtomicU8, AtomicUsize, Ordering}; +use crate::{AllocationError, BasicAllocator}; +use core::alloc::Layout; +use core::fmt as core_fmt; +use core::ops::Drop; +use core::ptr::NonNull; +use score_log::fmt as score_fmt; +use score_log::ScoreDebug; + +#[derive(Debug, ScoreDebug, Eq, PartialEq)] +#[repr(u8)] +enum NodeState { + Free = 0, + Occupied, +} + +/// Fixed-capacity pool of equally-sized elements backed by an allocator implementing [`BasicAllocator`]. +/// +/// Internal storage is reference counted. +/// Cloning provides a new handle to same data. +pub struct MemoryPool { + /// Allocator. + /// - Allocates on creation. + /// - Deallocates on drop. + allocator: A, + + /// Free list storage - one atomic slot per element, holding a [`NodeState`] discriminant. + free_list_storage: NonNull, + /// Free list layout. + free_list_layout: Layout, + + /// Shared reference count storage. + ref_count_storage: NonNull, + /// Reference count layout. + ref_count_layout: Layout, + + /// Internal pool storage. + pool_storage: NonNull, + /// Internal pool layout. + pool_layout: Layout, + + /// Layout of a single element. + /// Used for pre-condition checks. + element_layout: Layout, + /// Layout of a single element - padded. + /// Used for actual work. + element_layout_padded: Layout, + /// Number of pre-allocated elements. + capacity: usize, +} + +impl MemoryPool { + /// Create new memory pool. + /// + /// Internally three allocations are performed: + /// - free list block + /// - reference count + /// - pool storage + /// + /// # Parameters + /// * `element_layout`: Layout of a single element. + /// * `capacity`: Number of elements to be allocated. + /// * `allocator`: Memory allocator. + pub fn new(element_layout: Layout, capacity: usize, allocator: A) -> Result { + // Disallow zero size allocation. + if capacity == 0 || element_layout.size() == 0 { + return Err(AllocationError::ZeroSizeAllocation); + } + + // Free list and pool storages have a layout of an array. + // An array of `[T; N]` has a size of `size_of::() * N` and the same alignment of `T`. + // https://doc.rust-lang.org/reference/type-layout.html#r-layout.array + + // Get internal free list storage layout - one atomic slot per element. + let free_list_size = size_of::() + .checked_mul(capacity) + .ok_or(AllocationError::Internal)?; + let free_list_align = align_of::(); + let free_list_layout = + Layout::from_size_align(free_list_size, free_list_align).map_err(|_| AllocationError::Internal)?; + // Allocate internal free list storage. + let free_list_storage: NonNull = allocator.allocate(free_list_layout)?.cast(); + // Initialize every slot to `Free`. + // Performed element-wise to ensure correct representation of each element. + for index in 0..capacity { + // SAFETY: `free_list_storage` is allocated with `capacity` size. + unsafe { + free_list_storage + .as_ptr() + .add(index) + .write(AtomicU8::new(NodeState::Free as u8)) + }; + } + + // Allocate and initialize the reference count. + let ref_count_layout = Layout::new::(); + let ref_count_storage: NonNull = allocator.allocate(ref_count_layout)?.cast(); + // SAFETY: `ref_count_storage` is allocated with same layout as `AtomicUsize`. + unsafe { ref_count_storage.as_ptr().write(AtomicUsize::new(1)) }; + + // Get internal pool storage layout. + let element_layout_padded = element_layout.pad_to_align(); + let pool_size = element_layout_padded + .size() + .checked_mul(capacity) + .ok_or(AllocationError::Internal)?; + let pool_align = element_layout_padded.align(); + let pool_layout = Layout::from_size_align(pool_size, pool_align).map_err(|_| AllocationError::Internal)?; + // Allocate internal pool storage. + let pool_storage = allocator.allocate(pool_layout)?; + + Ok(Self { + allocator, + ref_count_storage, + ref_count_layout, + free_list_storage, + free_list_layout, + pool_storage, + pool_layout, + element_layout, + element_layout_padded, + capacity, + }) + } + + /// Access the shared reference count. + fn ref_count(&self) -> &AtomicUsize { + // SAFETY: `ref_count_storage` is initialized in `new`. + unsafe { self.ref_count_storage.as_ref() } + } + + /// Access the free-list slot for the given element index. + fn free_list_slot(&self, index: usize) -> &AtomicU8 { + assert!(index < self.capacity); + // SAFETY: + // `free_list_storage` is initialized in `new` with `capacity` number of slots. + // `index < capacity` check is performed. + unsafe { &*self.free_list_storage.as_ptr().add(index) } + } + + /// Return element capacity of the memory pool. + pub fn capacity(&self) -> usize { + self.capacity + } + + /// Return layout of the element. + pub fn element_layout(&self) -> Layout { + self.element_layout + } +} + +impl BasicAllocator for MemoryPool { + /// Allocates a block of memory as described by `layout`. + /// Provided layout must match the one provided at construction. + fn allocate(&self, layout: Layout) -> Result, AllocationError> { + // Check layout is matching. + // NOTE: zero size allocation is not rechecked here. + if layout != self.element_layout { + return Err(AllocationError::InvalidLayout); + } + + // Scan for a free slot and atomically claim it. + for index in 0..self.capacity { + if self + .free_list_slot(index) + .compare_exchange( + NodeState::Free as u8, + NodeState::Occupied as u8, + Ordering::Acquire, + Ordering::Relaxed, + ) + .is_ok() + { + // Get pointer in pool memory with correct offset. + let offset = self.element_layout_padded.size() * index; + // SAFETY: `pool_storage` is initialized in `new` with `capacity` number of slots. + return Ok(unsafe { self.pool_storage.byte_add(offset) }); + } + } + + Err(AllocationError::OutOfMemory) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + // Check layout is matching. + if layout != self.element_layout { + panic!("invalid layout provided to deallocation: {:?}", layout); + } + + // Get index based on offset. + let offset = ptr.byte_offset_from(self.pool_storage) as usize; + let index = offset / self.element_layout_padded.size(); + + // Set node state to free. + self.free_list_slot(index) + .store(NodeState::Free as u8, Ordering::Release); + } +} + +// SAFETY: all shared mutable state is accessed using atomics. +unsafe impl Send for MemoryPool {} +unsafe impl Sync for MemoryPool {} + +impl Clone for MemoryPool { + fn clone(&self) -> Self { + // Increase reference count. + self.ref_count().fetch_add(1, Ordering::Relaxed); + Self { + allocator: self.allocator.clone(), + ref_count_storage: self.ref_count_storage, + ref_count_layout: self.ref_count_layout, + free_list_storage: self.free_list_storage, + free_list_layout: self.free_list_layout, + pool_storage: self.pool_storage, + pool_layout: self.pool_layout, + element_layout: self.element_layout, + element_layout_padded: self.element_layout_padded, + capacity: self.capacity, + } + } +} + +impl Drop for MemoryPool { + fn drop(&mut self) { + // Release allocations on last reference dropping. + if self.ref_count().fetch_sub(1, Ordering::Release) == 1 { + fence(Ordering::Acquire); + // SAFETY: deallocation is ensured to be done once. + unsafe { + self.allocator + .deallocate(self.free_list_storage.cast(), self.free_list_layout); + self.allocator.deallocate(self.pool_storage, self.pool_layout); + self.allocator + .deallocate(self.ref_count_storage.cast(), self.ref_count_layout); + } + } + } +} + +impl core_fmt::Debug for MemoryPool { + fn fmt(&self, f: &mut core_fmt::Formatter<'_>) -> core_fmt::Result { + f.debug_struct("MemoryPool") + .field("element_layout", &self.element_layout()) + .field("capacity", &self.capacity()) + .finish() + } +} + +impl score_fmt::ScoreDebug for MemoryPool { + fn fmt(&self, f: score_fmt::Writer, spec: &score_fmt::FormatSpec) -> score_fmt::Result { + score_fmt::DebugStruct::new(f, spec, "MemoryPool") + .field("element_layout", &self.element_layout()) + .field("capacity", &self.capacity()) + .finish() + } +} + +#[cfg(all(test, not(loom)))] +mod tests { + use crate::atomic::Ordering; + use crate::memory_region::MmapMemoryRegion; + use crate::{page_size, AllocationError, ArenaAllocator, BasicAllocator, HeapAllocator, MemoryPool}; + use core::alloc::Layout; + use std::collections::BTreeSet; + use std::sync::Arc; + use std::thread; + + #[test] + fn test_new_ok() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let result = MemoryPool::new(element_layout, capacity, allocator); + assert!(result.is_ok()); + } + + #[test] + fn test_new_zero_cap() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0; + let result = MemoryPool::new(element_layout, capacity, allocator); + assert!(result.is_err_and(|e| e == AllocationError::ZeroSizeAllocation)); + } + + #[test] + fn test_new_zero_element_size() { + let allocator = HeapAllocator; + let element_layout = Layout::from_size_align(0, 4).unwrap(); + let capacity = 0x20; + let result = MemoryPool::new(element_layout, capacity, allocator); + assert!(result.is_err_and(|e| e == AllocationError::ZeroSizeAllocation)); + } + + #[test] + fn test_new_internal_allocator_oom() { + let internal_allocator_capacity = page_size(); + let memory_region = MmapMemoryRegion::new(internal_allocator_capacity).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + + let element_layout = Layout::new::(); + let capacity = 0x2000; + let result = MemoryPool::new(element_layout, capacity, allocator); + assert!(result.is_err_and(|e| e == AllocationError::OutOfMemory)); + } + + #[test] + fn test_capacity_ok() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + assert_eq!(memory_pool.capacity(), capacity); + } + + #[test] + fn test_element_layout_ok() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + assert_eq!(memory_pool.element_layout(), element_layout); + } + + #[test] + fn test_allocate_single() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + + let result = memory_pool.allocate(element_layout); + assert!(result.is_ok()); + } + + #[test] + fn test_allocate_multiple() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + + let first_ptr = memory_pool.allocate(element_layout).unwrap(); + for index in 1..capacity { + let ptr = memory_pool.allocate(element_layout).unwrap(); + let offset = unsafe { ptr.byte_offset_from(first_ptr) } as usize; + let calculated_index = offset / element_layout.size(); + assert_eq!(index, calculated_index); + } + } + + #[test] + fn test_allocate_full_capacity() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + + // Fill the capacity. + for _ in 0..capacity { + let _ = memory_pool.allocate(element_layout); + } + + // Do one allocation too many. + let result = memory_pool.allocate(element_layout); + assert!(result.is_err_and(|e| e == AllocationError::OutOfMemory)); + } + + #[test] + fn test_allocate_invalid_layout() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + + let invalid_layout = Layout::new::(); + let result = memory_pool.allocate(invalid_layout); + assert!(result.is_err_and(|e| e == AllocationError::InvalidLayout)); + } + + #[test] + fn test_deallocate_single() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + + let ptr = memory_pool.allocate(element_layout).unwrap(); + unsafe { memory_pool.deallocate(ptr, element_layout) }; + } + + #[test] + fn test_deallocate_multiple() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + + // Allocate. + let mut allocations = vec![]; + for _ in 0..capacity { + let ptr = memory_pool.allocate(element_layout).unwrap(); + allocations.push(ptr) + } + + // Deallocate. + for ptr in allocations { + unsafe { memory_pool.deallocate(ptr, element_layout) }; + } + } + + #[test] + #[should_panic(expected = "invalid layout provided to deallocation")] + fn test_deallocate_invalid_layout() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + + let ptr = memory_pool.allocate(element_layout).unwrap(); + let invalid_layout = Layout::new::(); + unsafe { memory_pool.deallocate(ptr, invalid_layout) }; + } + + #[test] + fn test_deallocate_already_deallocated() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + + let ptr = memory_pool.allocate(element_layout).unwrap(); + // Double deallocation does nothing wrong - state is cleared twice. + unsafe { memory_pool.deallocate(ptr, element_layout) }; + unsafe { memory_pool.deallocate(ptr, element_layout) }; + } + + #[test] + fn test_allocate_deallocate_middle_element() { + // Allocate until full, deallocate some element in the middle, try to allocate again. + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + + // Allocate. + let mut allocations = vec![]; + for _ in 0..capacity { + let ptr = memory_pool.allocate(element_layout).unwrap(); + allocations.push(ptr) + } + + // Take some element in the middle. + let ptr_middle = allocations[5]; + unsafe { memory_pool.deallocate(ptr_middle, element_layout) }; + + // Allocate again and compare pointer. + let ptr_new = memory_pool.allocate(element_layout).unwrap(); + + assert_eq!(ptr_middle, ptr_new); + } + + #[test] + fn test_clone_ref_count() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + assert_eq!(memory_pool.ref_count().load(Ordering::Relaxed), 1); + + let clone = memory_pool.clone(); + assert_eq!(memory_pool.ref_count().load(Ordering::Relaxed), 2); + + // Dropping the clone decrements the count without releasing the storage. + drop(clone); + assert_eq!(memory_pool.ref_count().load(Ordering::Relaxed), 1); + } + + #[test] + fn test_clone_shares_pool() { + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + let clone = memory_pool.clone(); + + // Exhaust the whole capacity through the clone. + for _ in 0..capacity { + clone.allocate(element_layout).unwrap(); + } + + // The original observes the exhausted pool - the free list is shared, not copied. + let result = memory_pool.allocate(element_layout); + assert!(result.is_err_and(|e| e == AllocationError::OutOfMemory)); + } + + #[test] + fn test_clone_frees_once() { + // Create, clone, then drop both handles. + // The storage must be released exactly once. + let allocator = HeapAllocator; + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + let clone = memory_pool.clone(); + drop(memory_pool); + drop(clone); + } + + #[test] + fn test_clone_with_mmap_allocator() { + // Nested reference counting. + // The pool's count and the arena's count must both release cleanly, unmapping the arena exactly once. + let memory_region = MmapMemoryRegion::new(page_size()).unwrap(); + let allocator = ArenaAllocator::new(memory_region).unwrap(); + let element_layout = Layout::new::(); + let capacity = 0x20; + let memory_pool = MemoryPool::new(element_layout, capacity, allocator).unwrap(); + let clone = memory_pool.clone(); + drop(memory_pool); + drop(clone); + } + + #[test] + fn test_concurrent_allocate_distinct_slots() { + // Perform allocations from multiple threads to check every claimed slot is distinct. + const THREADS: usize = 8; + const PER_THREAD: usize = 16; + let element_layout = Layout::new::(); + let pool = Arc::new(MemoryPool::new(element_layout, THREADS * PER_THREAD, HeapAllocator).unwrap()); + + let handles: Vec<_> = (0..THREADS) + .map(|_| { + let pool = pool.clone(); + thread::spawn(move || { + (0..PER_THREAD) + .map(|_| pool.allocate(element_layout).unwrap().as_ptr() as usize) + .collect::>() + }) + }) + .collect(); + + let mut all = BTreeSet::new(); + for handle in handles { + for addr in handle.join().unwrap() { + assert!(all.insert(addr), "slot handed out twice: {addr:#x}"); + } + } + assert_eq!(all.len(), THREADS * PER_THREAD); + } + + #[test] + fn test_concurrent_allocate_contended_capacity() { + // Perform more allocations than available slots. + // Success until capacity is reached, the rest observe `OutOfMemory`. + const THREADS: usize = 16; + const CAPACITY: usize = 4; + let element_layout = Layout::new::(); + let pool = Arc::new(MemoryPool::new(element_layout, CAPACITY, HeapAllocator).unwrap()); + + let handles: Vec<_> = (0..THREADS) + .map(|_| { + let pool = pool.clone(); + thread::spawn(move || pool.allocate(element_layout).map(|p| p.as_ptr() as usize).ok()) + }) + .collect(); + + let mut claimed = BTreeSet::new(); + for handle in handles { + if let Some(addr) = handle.join().unwrap() { + assert!(claimed.insert(addr), "slot handed out twice: {addr:#x}"); + } + } + assert_eq!(claimed.len(), CAPACITY); + } + + #[test] + fn test_concurrent_clone_drop_single_release() { + // Perform clones and drops in multiple threads. + // The original is dropped last. + let element_layout = Layout::new::(); + let pool = MemoryPool::new(element_layout, 0x20, HeapAllocator).unwrap(); + let shared = Arc::new(pool.clone()); + + let handles: Vec<_> = (0..8) + .map(|_| { + let shared = shared.clone(); + thread::spawn(move || { + for _ in 0..64 { + drop((*shared).clone()); + } + }) + }) + .collect(); + for handle in handles { + handle.join().unwrap(); + } + + drop(Arc::try_unwrap(shared).ok().unwrap()); + assert_eq!(pool.ref_count().load(Ordering::Relaxed), 1); + } +} + +#[cfg(all(test, loom))] +mod loom_tests { + use crate::{BasicAllocator, HeapAllocator, MemoryPool}; + use core::alloc::Layout; + use loom::thread; + + #[test] + fn loom_concurrent_allocate_distinct() { + // Perform two concurrent allocations on a shared pool. + loom::model(|| { + let element_layout = Layout::new::(); + let pool = MemoryPool::new(element_layout, 2, HeapAllocator).unwrap(); + + let p1 = pool.clone(); + let p2 = pool.clone(); + let h1 = thread::spawn(move || p1.allocate(element_layout).unwrap().as_ptr() as usize); + let h2 = thread::spawn(move || p2.allocate(element_layout).unwrap().as_ptr() as usize); + + let a = h1.join().unwrap(); + let b = h2.join().unwrap(); + assert_ne!(a, b); + }); + } + + #[test] + fn loom_concurrent_clone_drop() { + // Perform two concurrent drops on a shared handle. + loom::model(|| { + let element_layout = Layout::new::(); + let pool = MemoryPool::new(element_layout, 2, HeapAllocator).unwrap(); + + let p1 = pool.clone(); + let p2 = pool.clone(); + let h1 = thread::spawn(move || drop(p1)); + let h2 = thread::spawn(move || drop(p2)); + h1.join().unwrap(); + h2.join().unwrap(); + }); + } +} diff --git a/score/log_rust/score_log_fmt/fmt_impl.rs b/score/log_rust/score_log_fmt/fmt_impl.rs index d516f8ec32..c6a48fc5e5 100644 --- a/score/log_rust/score_log_fmt/fmt_impl.rs +++ b/score/log_rust/score_log_fmt/fmt_impl.rs @@ -259,6 +259,30 @@ impl } } +impl ScoreDebug for core::alloc::LayoutError { + fn fmt(&self, f: Writer, spec: &FormatSpec) -> FmtResult { + DebugStruct::new(f, spec, "LayoutError").finish() + } +} + +impl ScoreDebug for core::alloc::Layout { + fn fmt(&self, f: Writer, spec: &FormatSpec) -> FmtResult { + // `align` field format must be recreated, as `Debug` is using non-stable `Alignment`. + let align_base10 = self.align(); + let align_log2 = align_base10.trailing_zeros(); + let align_string = format!("{:?} (1 << {:?})", align_base10, align_log2); + + // Make a custom format spec to hide "". + let mut spec_clone = spec.clone(); + let align_string_spec = spec_clone.display_hint(DisplayHint::NoHint); + + DebugStruct::new(f, spec, "Layout") + .field("size", &self.size()) + .field_with("align", |f| align_string.fmt(f, align_string_spec)) + .finish() + } +} + #[cfg(test)] mod tests { use crate::test_utils::common_test_debug; @@ -441,4 +465,16 @@ mod tests { )); common_test_debug(("a", "b", (r"0x64", 10, false), "0.1", "true")); } + + #[test] + fn test_layout_error_debug() { + let layout_error = core::alloc::Layout::from_size_align(1, 3).unwrap_err(); + common_test_debug(layout_error); + } + + #[test] + fn test_layout_debug() { + let layout = core::alloc::Layout::from_size_align(123, 4).unwrap(); + common_test_debug(layout); + } }