Skip to content

refactor: new rawsmallvec representation #590

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alejandro-vaz wants to merge 15 commits into from
+154 −123
Closed

refactor: new rawsmallvec representation #590

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
4623ef6
feat: simplified inline representation
alejandro-vaz Sep 5, 2026
f7615e5
refactor: remade rawsmallvec representation
alejandro-vaz Sep 6, 2026
d93722a
fix: unchecked pointer accesses
alejandro-vaz Sep 7, 2026
52969b2
fix: style formatting
alejandro-vaz Sep 7, 2026
24693be
fix: field one heap use
alejandro-vaz Sep 7, 2026
7c43b32
fix: drop mechanic
alejandro-vaz Sep 7, 2026
f2ac596
fix: style formatting
alejandro-vaz Sep 7, 2026
73f0eea
fix: drop behavior
alejandro-vaz Sep 7, 2026
a211920
fix: style formatting
alejandro-vaz Sep 7, 2026
4fea3ac
fix: try_grow_raw capacity handling
alejandro-vaz Sep 7, 2026
05ce30c
fix: style formatting
alejandro-vaz Sep 7, 2026
00c7db9
refactor: added erasure
alejandro-vaz Sep 11, 2026
1ced971
refactor: apply some erasures
alejandro-vaz Sep 11, 2026
eafb4d2
refactor: more erasures
alejandro-vaz Sep 11, 2026
bac4085
reafactor: all erasures
alejandro-vaz Sep 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 81 additions & 80 deletions src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -615,8 +615,8 @@ impl<T, const N: usize> IntoIter<T, N> {
// `self.begin..end` are all initialized. So the pointer arithmetic is
// valid, and so is the construction of the slice
unsafe {
let ptr = self.raw.as_ptr(on_heap);
core::slice::from_raw_parts(ptr.add(self.begin), end - self.begin)
let ptr = self.raw.as_erased(on_heap).as_ptr();
core::slice::from_raw_parts(ptr.add(self.begin).cast(), end - self.begin)
}
}

Expand All @@ -625,8 +625,8 @@ impl<T, const N: usize> IntoIter<T, N> {
let (end, on_heap) = self.end.parts();
// SAFETY: see above
unsafe {
let ptr = self.raw.as_mut_ptr(on_heap);
core::slice::from_raw_parts_mut(ptr.add(self.begin), end - self.begin)
let ptr = self.raw.as_mut_erased(on_heap).as_mut_ptr();
core::slice::from_raw_parts_mut(ptr.add(self.begin).cast(), end - self.begin)
}
}
}
Expand All @@ -642,8 +642,8 @@ impl<T, const N: usize> Iterator for IntoIter<T, N> {
} else {
// SAFETY: see above
unsafe {
let ptr = self.raw.as_mut_ptr(on_heap);
let value = ptr.add(self.begin).read();
let reference = self.raw.as_erased(on_heap);
let value = reference[self.begin].assume_init_read();
self.begin += 1;
Some(value)
}
Expand All @@ -666,9 +666,9 @@ impl<T, const N: usize> DoubleEndedIterator for IntoIter<T, N> {
} else {
// SAFETY: see above
unsafe {
let ptr = self.raw.as_mut_ptr(on_heap);
let reference = self.raw.as_erased(on_heap);
self.end.sub(1);
let value = ptr.add(end - 1).read();
let value = reference[end - 1].assume_init_read();
Some(value)
}
}
Expand Down Expand Up @@ -713,7 +713,7 @@ impl<T, const N: usize> SmallVec<T, N> {
// Although we create a new buffer, since S and N are known at compile
// time, even with `-C opt-level=1`, it gets optimized as best
// as it could be. (Checked with <godbolt.org>)
let mut buf: MaybeUninit<[T; N]> = MaybeUninit::uninit();
let mut buf: [MaybeUninit<T>; N] = [const { MaybeUninit::uninit() }; N];

// SAFETY: buf and elements do not overlap, are aligned and have space
// for at least S elements since S <= N.
Expand All @@ -739,22 +739,16 @@ impl<T, const N: usize> SmallVec<T, N> {
// SAFETY: all the members in 0..len are initialized
let mut vec = Self {
len: TaggedLen::new(len, false),
raw: RawSmallVec::new_inline(MaybeUninit::new(buf)),
raw: RawSmallVec::new_inline(buf.map(MaybeUninit::new)),
_marker: PhantomData
};
// Deallocate the remaining elements so no memory is leaked.
unsafe {
// SAFETY: both the input and output pointers are in range of the
// stack allocation
let remainder_ptr = vec.raw.as_mut_ptr_inline().add(len);
let remainder_len = N - len;

// SAFETY: the values are initialized, so dropping them here is
// fine.
core::ptr::drop_in_place(core::ptr::slice_from_raw_parts_mut(
remainder_ptr,
remainder_len
));
core::ptr::slice_from_raw_parts_mut(
vec.raw.as_mut_inline().as_mut_ptr().add(len),
N - len
)
.drop_in_place();
}

vec
Expand All @@ -773,7 +767,7 @@ impl<T, const N: usize> SmallVec<T, N> {
/// };
///
/// let buf = [1, 2, 3, 4, 5, 0, 0, 0];
/// let small_vec = unsafe { SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf), 5) };
/// let small_vec = unsafe { SmallVec::from_buf_and_len_unchecked(MaybeUninit::new(buf).into(), 5) };
///
/// assert_eq!(&*small_vec, &[1, 2, 3, 4, 5]);
/// ```
Expand All @@ -782,7 +776,7 @@ impl<T, const N: usize> SmallVec<T, N> {
///
/// `len <= N`, and all the elements in `buf[..len]` must be initialized
#[inline]
pub const unsafe fn from_buf_and_len_unchecked(buf: MaybeUninit<[T; N]>, len: usize) -> Self {
pub const unsafe fn from_buf_and_len_unchecked(buf: [MaybeUninit<T>; N], len: usize) -> Self {
debug_assert!(len <= N);
Self {
len: TaggedLen::new(len, false),
Expand Down Expand Up @@ -825,11 +819,16 @@ impl<T, const N: usize> SmallVec<T, N> {
let cap = vec.capacity();
// SAFETY: vec.capacity is not `0` (checked above), so the pointer
// can not dangle and thus specifically cannot be null.
let ptr = unsafe { NonNull::new_unchecked(vec.as_mut_ptr()) };
let ptr = unsafe {
NonNull::new_unchecked(core::ptr::slice_from_raw_parts_mut(
vec.as_mut_ptr().cast(),
cap
))
};

Self {
len: TaggedLen::new(len, true),
raw: RawSmallVec::new_heap(ptr, cap),
raw: RawSmallVec::new_heap(ptr),
_marker: PhantomData
}
}
Expand Down Expand Up @@ -1197,26 +1196,25 @@ impl<T, const N: usize> SmallVec<T, N> {
unsafe { self.set_on_heap() };
}
result
} else {
} else if on_heap {
// new_capacity <= Self::inline_size()
if on_heap {
unsafe {
// SAFETY: heap member is active
let (ptr, old_cap) = self.raw.heap;
// inline member is now active

// SAFETY: len <= new_capacity <= Self::inline_size()
// so the copy is within bounds of the inline member
copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len);
drop(DropDealloc {
ptr: ptr.cast(),
size_bytes: old_cap * size_of::<T>(),
align: align_of::<T>()
});
self.set_inline();
}
unsafe {
// SAFETY: heap member is active
let ptr = self.raw.heap;
// inline member is now active
// SAFETY: len <= new_capacity <= Self::inline_size()
// so the copy is within bounds of the inline member
copy_nonoverlapping(ptr.as_ptr().cast(), self.raw.as_mut_inline(), len);
drop(DropDealloc {
ptr: ptr.cast(),
size_bytes: ptr.len() * size_of::<T>(),
align: align_of::<T>()
});
self.set_inline();
}
Ok(())
} else {
Ok(())
}
}

Expand Down Expand Up @@ -1283,13 +1281,14 @@ impl<T, const N: usize> SmallVec<T, N> {
if len <= Self::inline_size() {
// SAFETY: on_heap is true, so we're on the heap
unsafe {
let (ptr, capacity) = self.raw.heap;
self.raw = RawSmallVec::new_inline(MaybeUninit::uninit());
copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len);
// let (ptr, capacity) = self.raw.heap;
let ptr = self.raw.heap;
self.raw = RawSmallVec::new();
copy_nonoverlapping(ptr.as_ptr().cast::<T>(), self.raw.as_mut_inline().as_mut_ptr().cast(), len);
self.set_inline();
alloc::alloc::dealloc(
ptr.cast().as_ptr(),
Layout::from_size_align_unchecked(capacity * size_of::<T>(), align_of::<T>())
Layout::from_size_align_unchecked(ptr.len() * size_of::<T>(), align_of::<T>())
);
}
} else if len < self.capacity() {
Expand All @@ -1307,25 +1306,22 @@ impl<T, const N: usize> SmallVec<T, N> {
return;
}
// SAFETY: the vector is on the heap
let capacity = unsafe { self.raw.heap.1 };
if capacity > min_capacity {
let ptr = unsafe { self.raw.heap };
let cap = ptr.len();
if cap > min_capacity {
let target = core::cmp::max(len, min_capacity);
if target <= Self::inline_size() {
// SAFETY: on_heap is true, so we're on the heap
unsafe {
let (ptr, capacity) = self.raw.heap;
self.raw = RawSmallVec::new_inline(MaybeUninit::uninit());
copy_nonoverlapping(ptr.as_ptr(), self.raw.as_mut_ptr_inline(), len);
self.raw = RawSmallVec::new();
copy_nonoverlapping(ptr.as_ptr().cast(), self.raw.as_mut_inline(), len);
self.set_inline();
alloc::alloc::dealloc(
ptr.cast().as_ptr(),
Layout::from_size_align_unchecked(
capacity * size_of::<T>(),
align_of::<T>()
)
Layout::from_size_align_unchecked(cap * size_of::<T>(), align_of::<T>())
);
}
} else if target < capacity {
} else if target < cap {
// SAFETY: len > Self::inline_size() >= 0
// so new capacity is non zero, it is equal to the length
// T can't be a ZST because SmallVec<ZST, N> is never spilled.
Expand Down Expand Up @@ -1476,26 +1472,33 @@ impl<T, const N: usize> SmallVec<T, N> {
pub const fn as_slice(&self) -> &[T] {
let (len, on_heap) = self.len.parts();
// SAFETY: all the elements in `..len` are initialized
unsafe { core::slice::from_raw_parts(self.raw.as_ptr(on_heap), len) }
// unsafe { core::slice::from_raw_parts(self.raw.as_ptr(on_heap), len) }
unsafe {
let ptr = self.raw.as_erased(on_heap).as_ptr();
core::slice::from_raw_parts(ptr.cast(), len)
}
}

#[inline]
pub const fn as_mut_slice(&mut self) -> &mut [T] {
let (len, on_heap) = self.len.parts();
// SAFETY: see above
unsafe { core::slice::from_raw_parts_mut(self.raw.as_mut_ptr(on_heap), len) }
unsafe {
let ptr = self.raw.as_mut_erased(on_heap).as_mut_ptr();
core::slice::from_raw_parts_mut(ptr.cast(), len)
}
}

#[inline]
pub const fn as_ptr(&self) -> *const T {
// SAFETY: the tag tells which member is active
unsafe { self.raw.as_ptr(self.len.on_heap()) }
self.raw.as_erased(self.len.on_heap()).as_ptr().cast()
}

#[inline]
pub const fn as_mut_ptr(&mut self) -> *mut T {
// SAFETY: see above
unsafe { self.raw.as_mut_ptr(self.len.on_heap()) }
self.raw.as_mut_erased(self.len.on_heap()).as_mut_ptr().cast()
}

#[inline]
Expand All @@ -1509,7 +1512,7 @@ impl<T, const N: usize> SmallVec<T, N> {
// the length we don't drop the elements we previously
// held
unsafe {
copy_nonoverlapping(this.raw.as_ptr_inline(), vec.as_mut_ptr(), len);
copy_nonoverlapping(this.raw.as_inline().as_ptr().cast(), vec.as_mut_ptr(), len);
vec.set_len(len);
}
vec
Expand All @@ -1523,8 +1526,8 @@ impl<T, const N: usize> SmallVec<T, N> {
// - the first `len` entries are proper `T`-values
// - the allocation is not larger than `isize::MAX`
unsafe {
let (ptr, cap) = this.raw.heap;
Vec::from_raw_parts(ptr.as_ptr(), len, cap)
let ptr = this.raw.heap;
Vec::from_raw_parts(ptr.as_ptr().cast(), len, ptr.len())
}
}
}
Expand Down Expand Up @@ -1726,7 +1729,7 @@ impl<T, const N: usize> SmallVec<T, N> {
);
}
let mut me = ManuallyDrop::new(self);
unsafe { core::slice::from_raw_parts_mut(me.raw.as_mut_ptr(true), len) }
unsafe { core::slice::from_raw_parts_mut(me.raw.as_mut_heap().as_mut_ptr().cast(), len) }
}

/// Returns the remaining spare capacity of the vector as a slice of
Expand All @@ -1740,10 +1743,8 @@ impl<T, const N: usize> SmallVec<T, N> {
let (len, on_heap) = self.len.parts();
unsafe {
let capacity = self.raw.capacity(on_heap);
core::slice::from_raw_parts_mut(
self.raw.as_mut_ptr(on_heap).add(len) as *mut MaybeUninit<T>,
capacity - len
)
let ptr = self.raw.as_mut_erased(on_heap).as_mut_ptr();
core::slice::from_raw_parts_mut(ptr.add(len) as *mut MaybeUninit<T>, capacity - len)
}
}

Expand Down Expand Up @@ -1826,7 +1827,7 @@ impl<T, const N: usize> SmallVec<T, N> {

SmallVec {
len: TaggedLen::new(length, true),
raw: RawSmallVec::new_heap(ptr, capacity),
raw: RawSmallVec::new_heap(NonNull::slice_from_raw_parts(ptr.cast(), capacity)),
_marker: PhantomData
}
}
Expand Down Expand Up @@ -1991,12 +1992,12 @@ impl Drop for DropDealloc {
unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec<T, N> {
fn drop(&mut self) {
let (len, on_heap) = self.len.parts();
let ptr = unsafe { self.raw.as_mut_ptr(on_heap) };
let ptr = self.raw.as_mut_erased(on_heap).as_mut_ptr();
// SAFETY: we first drop the elements, then `_drop_dealloc` is dropped,
// releasing memory we used to own
unsafe {
let _drop_dealloc = if on_heap {
let capacity = self.raw.heap.1;
let capacity = self.raw.heap.len();
Some(DropDealloc {
ptr: NonNull::new_unchecked(ptr as *mut u8),
size_bytes: capacity * size_of::<T>(),
Expand All @@ -2005,7 +2006,7 @@ unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec<T, N> {
} else {
None
};
core::ptr::slice_from_raw_parts_mut(ptr, len).drop_in_place();
core::ptr::slice_from_raw_parts_mut(ptr.cast::<T>(), len).drop_in_place();
}
}
}
Expand All @@ -2015,11 +2016,11 @@ impl<T, const N: usize> Drop for SmallVec<T, N> {
fn drop(&mut self) {
let (len, on_heap) = self.len.parts();
// SAFETY: the tag tells which member is active
let ptr = unsafe { self.raw.as_mut_ptr(on_heap) };
let ptr = self.raw.as_mut_erased(on_heap).as_mut_ptr();
// SAFETY: see above
unsafe {
let _drop_dealloc = if on_heap {
let capacity = self.raw.heap.1;
let capacity = self.raw.heap.len();
Some(DropDealloc {
ptr: NonNull::new_unchecked(ptr as *mut u8),
size_bytes: capacity * size_of::<T>(),
Expand All @@ -2028,7 +2029,7 @@ impl<T, const N: usize> Drop for SmallVec<T, N> {
} else {
None
};
core::ptr::slice_from_raw_parts_mut(ptr, len).drop_in_place();
core::ptr::slice_from_raw_parts_mut(ptr.cast::<T>(), len).drop_in_place();
}
}
}
Expand All @@ -2039,9 +2040,9 @@ impl<T, const N: usize> Drop for IntoIter<T, N> {
unsafe {
let (end, on_heap) = self.end.parts();
let begin = self.begin;
let ptr = self.raw.as_mut_ptr(on_heap);
let ptr = self.raw.as_mut_erased(on_heap).as_mut_ptr();
let _drop_dealloc = if on_heap {
let capacity = self.raw.heap.1;
let capacity = self.raw.heap.len();
Some(DropDealloc {
ptr: NonNull::new_unchecked(ptr as *mut u8),
size_bytes: capacity * size_of::<T>(),
Expand All @@ -2050,7 +2051,7 @@ impl<T, const N: usize> Drop for IntoIter<T, N> {
} else {
None
};
core::ptr::slice_from_raw_parts_mut(ptr.add(begin), end - begin).drop_in_place();
core::ptr::slice_from_raw_parts_mut(ptr.add(begin).cast::<T>(), end - begin).drop_in_place();
}
}
}
Expand Down Expand Up @@ -2094,7 +2095,7 @@ impl<T, const N: usize> SmallVec<T, N> {
let mut result = Self::new();

if n > 0 {
let ptr = result.raw.as_mut_ptr_inline();
let ptr = unsafe { result.raw.as_mut_inline().as_mut_ptr().cast::<T>() };
let mut guard = DropGuard {
ptr,
len: 0
Expand Down Expand Up @@ -2483,7 +2484,7 @@ unsafe impl<const N: usize> BufMut for SmallVec<u8, N> {
let (len, on_heap) = self.len.parts();
// SAFETY: the tag tells which member is active
let cap = unsafe { self.raw.capacity(on_heap) };
let ptr = unsafe { self.raw.as_mut_ptr(on_heap) };
let ptr = self.raw.as_mut_erased(on_heap).as_mut_ptr().cast::<u8>();
// SAFETY: Since `ptr` is valid for `cap` bytes, `ptr.add(len)` must be
// valid for `cap - len` bytes. The subtraction will not underflow since
// `len <= cap`.
Expand Down
Loading