diff --git a/src/borsh.rs b/src/borsh.rs index 139588a..b492e77 100644 --- a/src/borsh.rs +++ b/src/borsh.rs @@ -24,7 +24,7 @@ use { impl BorshSerialize for SmallVec { fn serialize(&self, writer: &mut Writer) -> Serial<()> { - (self.len.value() as u64).serialize(writer)?; + (self.len() as u64).serialize(writer)?; for element in self { element.serialize(writer)?; } diff --git a/src/lib.rs b/src/lib.rs index 3e3e83d..2044784 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -608,45 +608,25 @@ unsafe impl Send for IntoIter where T: Send {} unsafe impl Sync for IntoIter where T: Sync {} impl IntoIter { - #[inline] - const fn as_ptr(&self) -> *const T { - let on_heap = self.end.on_heap(); - if on_heap { - // SAFETY: vector is on the heap - unsafe { self.raw.as_ptr_heap() } - } else { - self.raw.as_ptr_inline() - } - } - - #[inline] - const fn as_mut_ptr(&mut self) -> *mut T { - let on_heap = self.end.on_heap(); - if on_heap { - // SAFETY: vector is on the heap - unsafe { self.raw.as_mut_ptr_heap() } - } else { - self.raw.as_mut_ptr_inline() - } - } - #[inline] pub const fn as_slice(&self) -> &[T] { - // SAFETY: The members in self.begin..self.end.value() are all - // initialized So the pointer arithmetic is valid, and so is the - // construction of the slice + let (end, on_heap) = self.end.parts(); + // SAFETY: `end` tells which buffer is active, and the members in + // `self.begin..end` are all initialized. So the pointer arithmetic is + // valid, and so is the construction of the slice unsafe { - let ptr = self.as_ptr(); - core::slice::from_raw_parts(ptr.add(self.begin), self.end.value() - self.begin) + let ptr = self.raw.as_ptr(on_heap); + core::slice::from_raw_parts(ptr.add(self.begin), end - self.begin) } } #[inline] pub const fn as_mut_slice(&mut self) -> &mut [T] { + let (end, on_heap) = self.end.parts(); // SAFETY: see above unsafe { - let ptr = self.as_mut_ptr(); - core::slice::from_raw_parts_mut(ptr.add(self.begin), self.end.value() - self.begin) + let ptr = self.raw.as_mut_ptr(on_heap); + core::slice::from_raw_parts_mut(ptr.add(self.begin), end - self.begin) } } } @@ -656,12 +636,13 @@ impl Iterator for IntoIter { #[inline] fn next(&mut self) -> Option { - if self.begin == self.end.value() { + let (end, on_heap) = self.end.parts(); + if self.begin == end { None } else { // SAFETY: see above unsafe { - let ptr = self.as_mut_ptr(); + let ptr = self.raw.as_mut_ptr(on_heap); let value = ptr.add(self.begin).read(); self.begin += 1; Some(value) @@ -671,7 +652,7 @@ impl Iterator for IntoIter { #[inline] fn size_hint(&self) -> (usize, Option) { - let size = self.end.value() - self.begin; + let size = self.end.len() - self.begin; (size, Some(size)) } } @@ -679,17 +660,15 @@ impl Iterator for IntoIter { impl DoubleEndedIterator for IntoIter { #[inline] fn next_back(&mut self) -> Option { - let mut end = self.end.value(); + let (end, on_heap) = self.end.parts(); if self.begin == end { None } else { // SAFETY: see above unsafe { - let ptr = self.as_mut_ptr(); - let on_heap = self.end.on_heap(); - end -= 1; - self.end = TaggedLen::new(end, on_heap); - let value = ptr.add(end).read(); + let ptr = self.raw.as_mut_ptr(on_heap); + self.end.sub(1); + let value = ptr.add(end - 1).read(); Some(value) } } @@ -881,18 +860,17 @@ impl SmallVec { #[inline] pub unsafe fn set_len(&mut self, new_len: usize) { debug_assert!(new_len <= self.capacity()); - let on_heap = self.len.on_heap(); - self.len = TaggedLen::new(new_len, on_heap); + self.len = TaggedLen::new(new_len, self.len.on_heap()); } #[inline] pub const fn inline_size() -> usize { - if Self::IS_ZST { usize::MAX } else { N } + RawSmallVec::::INLINE_CAP } #[inline] pub const fn len(&self) -> usize { - self.len.value() + self.len.len() } #[must_use] @@ -903,12 +881,8 @@ impl SmallVec { #[inline] pub const fn capacity(&self) -> usize { - if self.len.on_heap() { - // SAFETY: raw.heap is active - unsafe { self.raw.heap.1 } - } else { - Self::inline_size() - } + // SAFETY: the tag tells which member is active + unsafe { self.raw.capacity(self.len.on_heap()) } } #[inline] @@ -1202,7 +1176,7 @@ impl SmallVec { return Ok(()); } - let len = self.len(); + let (len, on_heap) = self.len.parts(); assert!(new_capacity >= len); if new_capacity > Self::inline_size() { @@ -1217,7 +1191,7 @@ impl SmallVec { result } else { // new_capacity <= Self::inline_size() - if self.spilled() { + if on_heap { unsafe { // SAFETY: heap member is active let (ptr, old_cap) = self.raw.heap; @@ -1294,12 +1268,12 @@ impl SmallVec { #[inline] pub fn shrink_to_fit(&mut self) { - if !self.spilled() { + let (len, on_heap) = self.len.parts(); + if !on_heap { return; } - let len = self.len(); if len <= Self::inline_size() { - // SAFETY: self.spilled() is true, so we're on the heap + // SAFETY: on_heap is true, so we're on the heap unsafe { let (ptr, capacity) = self.raw.heap; self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); @@ -1320,14 +1294,16 @@ impl SmallVec { #[inline] pub fn shrink_to(&mut self, min_capacity: usize) { - if !self.spilled() { + let (len, on_heap) = self.len.parts(); + if !on_heap { return; } - if self.capacity() > min_capacity { - let len = self.len(); + // SAFETY: the vector is on the heap + let capacity = unsafe { self.raw.heap.1 }; + if capacity > min_capacity { let target = core::cmp::max(len, min_capacity); if target <= Self::inline_size() { - // SAFETY: self.spilled() is true, so we're on the heap + // SAFETY: on_heap is true, so we're on the heap unsafe { let (ptr, capacity) = self.raw.heap; self.raw = RawSmallVec::new_inline(MaybeUninit::uninit()); @@ -1341,7 +1317,7 @@ impl SmallVec { ) ); } - } else if target < self.capacity() { + } else if target < capacity { // SAFETY: len > Self::inline_size() >= 0 // so new capacity is non zero, it is equal to the length // T can't be a ZST because SmallVec is never spilled. @@ -1490,44 +1466,34 @@ impl SmallVec { #[inline] pub const fn as_slice(&self) -> &[T] { - let len = self.len(); - let ptr = self.as_ptr(); + let (len, on_heap) = self.len.parts(); // SAFETY: all the elements in `..len` are initialized - unsafe { core::slice::from_raw_parts(ptr, len) } + unsafe { core::slice::from_raw_parts(self.raw.as_ptr(on_heap), len) } } #[inline] pub const fn as_mut_slice(&mut self) -> &mut [T] { - let len = self.len(); - let ptr = self.as_mut_ptr(); + let (len, on_heap) = self.len.parts(); // SAFETY: see above - unsafe { core::slice::from_raw_parts_mut(ptr, len) } + unsafe { core::slice::from_raw_parts_mut(self.raw.as_mut_ptr(on_heap), len) } } #[inline] pub const fn as_ptr(&self) -> *const T { - if self.len.on_heap() { - // SAFETY: heap member is active - unsafe { self.raw.as_ptr_heap() } - } else { - self.raw.as_ptr_inline() - } + // SAFETY: the tag tells which member is active + unsafe { self.raw.as_ptr(self.len.on_heap()) } } #[inline] pub const fn as_mut_ptr(&mut self) -> *mut T { - if self.len.on_heap() { - // SAFETY: see above - unsafe { self.raw.as_mut_ptr_heap() } - } else { - self.raw.as_mut_ptr_inline() - } + // SAFETY: see above + unsafe { self.raw.as_mut_ptr(self.len.on_heap()) } } #[inline] pub fn into_vec(self) -> Vec { - let len = self.len(); - if !self.spilled() { + let (len, on_heap) = self.len.parts(); + if !on_heap { let mut vec = Vec::with_capacity(len); let this = ManuallyDrop::new(self); // SAFETY: we create a new vector with sufficient capacity, copy our @@ -1745,13 +1711,14 @@ impl SmallVec { } pub fn leak<'a>(self) -> &'a mut [T] { - if !self.spilled() { + let (len, on_heap) = self.len.parts(); + if !on_heap { panic!( "SmallVec::leak() called on inline (stack) SmallVec, which cannot be safely leaked" ); } let mut me = ManuallyDrop::new(self); - unsafe { core::slice::from_raw_parts_mut(me.as_mut_ptr(), me.len()) } + unsafe { core::slice::from_raw_parts_mut(me.raw.as_mut_ptr(true), len) } } /// Returns the remaining spare capacity of the vector as a slice of @@ -1762,10 +1729,12 @@ impl SmallVec { /// [`set_len`](Self::set_len) method. #[inline] pub fn spare_capacity_mut(&mut self) -> &mut [MaybeUninit] { + let (len, on_heap) = self.len.parts(); unsafe { + let capacity = self.raw.capacity(on_heap); core::slice::from_raw_parts_mut( - self.as_mut_ptr().add(self.len()) as *mut MaybeUninit, - self.capacity() - self.len() + self.raw.as_mut_ptr(on_heap).add(len) as *mut MaybeUninit, + capacity - len ) } } @@ -2013,14 +1982,13 @@ impl Drop for DropDealloc { #[cfg(feature = "may_dangle")] unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { fn drop(&mut self) { - let on_heap = self.spilled(); - let len = self.len(); - let ptr = self.as_mut_ptr(); + let (len, on_heap) = self.len.parts(); + let ptr = unsafe { self.raw.as_mut_ptr(on_heap) }; // SAFETY: we first drop the elements, then `_drop_dealloc` is dropped, // releasing memory we used to own unsafe { let _drop_dealloc = if on_heap { - let capacity = self.capacity(); + let capacity = self.raw.heap.1; Some(DropDealloc { ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), @@ -2037,13 +2005,13 @@ unsafe impl<#[may_dangle] T, const N: usize> Drop for SmallVec { #[cfg(not(feature = "may_dangle"))] impl Drop for SmallVec { fn drop(&mut self) { - let on_heap = self.spilled(); - let len = self.len(); - let ptr = self.as_mut_ptr(); + let (len, on_heap) = self.len.parts(); + // SAFETY: the tag tells which member is active + let ptr = unsafe { self.raw.as_mut_ptr(on_heap) }; // SAFETY: see above unsafe { let _drop_dealloc = if on_heap { - let capacity = self.capacity(); + let capacity = self.raw.heap.1; Some(DropDealloc { ptr: NonNull::new_unchecked(ptr as *mut u8), size_bytes: capacity * size_of::(), @@ -2061,10 +2029,9 @@ impl Drop for IntoIter { fn drop(&mut self) { // SAFETY: see above unsafe { - let on_heap = self.end.on_heap(); + let (end, on_heap) = self.end.parts(); let begin = self.begin; - let end = self.end.value(); - let ptr = self.as_mut_ptr(); + let ptr = self.raw.as_mut_ptr(on_heap); let _drop_dealloc = if on_heap { let capacity = self.raw.heap.1; Some(DropDealloc { @@ -2505,10 +2472,10 @@ unsafe impl BufMut for SmallVec { self.reserve(64); // Grow the smallvec } - let cap = self.capacity(); - let len = self.len(); - - let ptr = self.as_mut_ptr(); + let (len, on_heap) = self.len.parts(); + // SAFETY: the tag tells which member is active + let cap = unsafe { self.raw.capacity(on_heap) }; + let ptr = unsafe { self.raw.as_mut_ptr(on_heap) }; // SAFETY: Since `ptr` is valid for `cap` bytes, `ptr.add(len)` must be // valid for `cap - len` bytes. The subtraction will not underflow since // `len <= cap`. diff --git a/src/rawsmallvec.rs b/src/rawsmallvec.rs index 977c601..4d279cd 100644 --- a/src/rawsmallvec.rs +++ b/src/rawsmallvec.rs @@ -35,6 +35,7 @@ impl Default for RawSmallVec { } impl RawSmallVec { + pub const INLINE_CAP: usize = if Self::IS_ZST { usize::MAX } else { N }; const IS_ZST: bool = size_of::() == 0; #[inline] @@ -74,18 +75,38 @@ impl RawSmallVec { /// # Safety /// - /// The vector must be on the heap - #[inline] - pub const unsafe fn as_ptr_heap(&self) -> *const T { - unsafe { self.heap.0.as_ptr() } + /// `on_heap` must be true if and only if `self.heap` is the active member. + #[inline(always)] + pub const unsafe fn as_ptr(&self, on_heap: bool) -> *const T { + if on_heap { + unsafe { self.heap.0.as_ptr() } + } else { + self.as_ptr_inline() + } } /// # Safety /// - /// The vector must be on the heap - #[inline] - pub const unsafe fn as_mut_ptr_heap(&mut self) -> *mut T { - unsafe { self.heap.0.as_ptr() } + /// `on_heap` must be true if and only if `self.heap` is the active member. + #[inline(always)] + pub const unsafe fn as_mut_ptr(&mut self, on_heap: bool) -> *mut T { + if on_heap { + unsafe { self.heap.0.as_ptr() } + } else { + self.as_mut_ptr_inline() + } + } + + /// # Safety + /// + /// `on_heap` must be true if and only if `self.heap` is the active member. + #[inline(always)] + pub const unsafe fn capacity(&self, on_heap: bool) -> usize { + if on_heap { + unsafe { self.heap.1 } + } else { + Self::INLINE_CAP + } } /// # Safety @@ -101,17 +122,12 @@ impl RawSmallVec { alloc, realloc }; + let (len, was_on_heap) = len.parts(); debug_assert!(!Self::IS_ZST); - debug_assert!(new_capacity > 0); - debug_assert!(new_capacity >= len.value()); + debug_assert!(new_capacity > 0 && new_capacity >= len); - let was_on_heap = len.on_heap(); - let ptr = if was_on_heap { - unsafe { self.as_mut_ptr_heap() } - } else { - self.as_mut_ptr_inline() - }; - let len = len.value(); + // SAFETY: the tag tells which member is active + let ptr = unsafe { self.as_mut_ptr(was_on_heap) }; let new_layout = Layout::array::(new_capacity).map_err(|_| CollectionAllocErr::CapacityOverflow)?; diff --git a/src/specialization.rs b/src/specialization.rs index f726225..f02f23b 100644 --- a/src/specialization.rs +++ b/src/specialization.rs @@ -126,7 +126,7 @@ impl SpecExtend> for SmallV } // Mark the iterator as fully consumed. - iter.begin = iter.end.value(); + iter.begin = iter.end.len(); } } diff --git a/src/taggedlen.rs b/src/taggedlen.rs index e7d3b6f..70b1419 100644 --- a/src/taggedlen.rs +++ b/src/taggedlen.rs @@ -23,49 +23,53 @@ impl Clone for TaggedLen { impl Copy for TaggedLen {} +#[allow(clippy::len_without_is_empty)] impl TaggedLen { - const IS_ZST: bool = size_of::() == 0; + const MAX_LEN: usize = usize::MAX >> Self::SHIFT; + const SHIFT: u32 = (size_of::() != 0) as u32; + const TAG: usize = Self::SHIFT as usize; - #[inline] + #[inline(always)] pub const fn new(len: usize, on_heap: bool) -> Self { - if Self::IS_ZST { - debug_assert!(!on_heap); - Self(len, PhantomData) - } else { - debug_assert!(len < isize::MAX as usize); - Self((len << 1) | on_heap as usize, PhantomData) - } + debug_assert!(len < Self::MAX_LEN); + debug_assert!(!on_heap || Self::TAG != 0); + Self( + (len << Self::SHIFT) | ((on_heap as usize) & Self::TAG), + PhantomData + ) } - #[inline] + #[inline(always)] + pub const fn len(self) -> usize { + self.0 >> Self::SHIFT + } + + #[inline(always)] #[must_use] pub const fn on_heap(self) -> bool { - if Self::IS_ZST { - false - } else { - (self.0 & 1_usize) == 1 - } + self.0 & Self::TAG != 0 } - #[inline] - pub const fn value(self) -> usize { - if Self::IS_ZST { self.0 } else { self.0 >> 1 } + #[inline(always)] + pub const fn parts(self) -> (usize, bool) { + (self.0 >> Self::SHIFT, (self.0 & Self::TAG) != 0) } - #[inline] + /// # Safety + /// + /// current len+n must be smaller than MAX_LEN + #[inline(always)] pub const unsafe fn add(&mut self, n: usize) { - self.0 += if Self::IS_ZST { - n - } else { - debug_assert!(self.value() + n < isize::MAX as usize); - n << 1 - } + debug_assert!(self.len() + n < Self::MAX_LEN); + self.0 += n << Self::SHIFT; } - #[inline] + /// # Safety + /// + /// current len must be greater equal than n + #[inline(always)] pub const unsafe fn sub(&mut self, n: usize) { - debug_assert!(self.value() >= n); - - self.0 -= if Self::IS_ZST { n } else { n << 1 }; + debug_assert!(self.len() >= n); + self.0 -= n << Self::SHIFT; } }