diff --git a/src/bytes.rs b/src/bytes.rs index 40c902f..2484a1e 100644 --- a/src/bytes.rs +++ b/src/bytes.rs @@ -106,18 +106,22 @@ pub struct Bytes { vtable: &'static Vtable, } +// `data` is passed by value (`*mut ()` instead of `&mut AtomicPtr<()>`) +// when `&mut self` or `self` is consumed. +// This allows the optimizer to see that the address of the `Bytes` is not +// captured by the indirect call, enabling further optimizations. pub(crate) struct Vtable { /// fn(data, ptr, len) pub clone: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Bytes, /// fn(data, ptr, len) /// /// `into_*` consumes the `Bytes`, returning the respective value. - pub into_vec: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Vec, - pub into_mut: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> BytesMut, + pub into_vec: unsafe fn(*mut (), *const u8, usize) -> Vec, + pub into_mut: unsafe fn(*mut (), *const u8, usize) -> BytesMut, /// fn(data) pub is_unique: unsafe fn(&AtomicPtr<()>) -> bool, /// fn(data, ptr, len) - pub drop: unsafe fn(&mut AtomicPtr<()>, *const u8, usize), + pub drop: unsafe fn(*mut (), *const u8, usize), } impl Bytes { @@ -644,6 +648,11 @@ impl Bytes { self.len -= by; self.ptr = self.ptr.add(by); } + + #[inline] + fn data_mut(&mut self) -> *mut () { + self.data.with_mut(|p| *p) + } } // Vtable must enforce this behavior @@ -653,7 +662,8 @@ unsafe impl Sync for Bytes {} impl Drop for Bytes { #[inline] fn drop(&mut self) { - unsafe { (self.vtable.drop)(&mut self.data, self.ptr, self.len) } + let data = self.data_mut(); + unsafe { (self.vtable.drop)(data, self.ptr, self.len) } } } @@ -1013,8 +1023,9 @@ impl From for BytesMut { /// assert_eq!(BytesMut::from(bytes), BytesMut::from(&b"hello"[..])); /// ``` fn from(bytes: Bytes) -> Self { - let bytes = ManuallyDrop::new(bytes); - unsafe { (bytes.vtable.into_mut)(&bytes.data, bytes.ptr, bytes.len) } + let mut bytes = ManuallyDrop::new(bytes); + let data = bytes.data_mut(); + unsafe { (bytes.vtable.into_mut)(data, bytes.ptr, bytes.len) } } } @@ -1026,8 +1037,9 @@ impl From for Bytes { impl From for Vec { fn from(bytes: Bytes) -> Vec { - let bytes = ManuallyDrop::new(bytes); - unsafe { (bytes.vtable.into_vec)(&bytes.data, bytes.ptr, bytes.len) } + let mut bytes = ManuallyDrop::new(bytes); + let data = bytes.data_mut(); + unsafe { (bytes.vtable.into_vec)(data, bytes.ptr, bytes.len) } } } @@ -1057,12 +1069,12 @@ unsafe fn static_clone(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { Bytes::from_static(slice) } -unsafe fn static_to_vec(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec { +unsafe fn static_to_vec(_: *mut (), ptr: *const u8, len: usize) -> Vec { let slice = slice::from_raw_parts(ptr, len); slice.to_vec() } -unsafe fn static_to_mut(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { +unsafe fn static_to_mut(_: *mut (), ptr: *const u8, len: usize) -> BytesMut { let slice = slice::from_raw_parts(ptr, len); BytesMut::from(slice) } @@ -1071,7 +1083,7 @@ fn static_is_unique(_: &AtomicPtr<()>) -> bool { false } -unsafe fn static_drop(_: &mut AtomicPtr<()>, _: *const u8, _: usize) { +unsafe fn static_drop(_: *mut (), _: *const u8, _: usize) { // nothing to drop for &'static [u8] } @@ -1108,15 +1120,15 @@ unsafe fn owned_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> By } } -unsafe fn owned_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec { +unsafe fn owned_to_vec(owned: *mut (), ptr: *const u8, len: usize) -> Vec { let slice = slice::from_raw_parts(ptr, len); let vec = slice.to_vec(); - owned_drop_impl::(data.load(Ordering::Relaxed)); + owned_drop_impl::(owned); vec } -unsafe fn owned_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { - BytesMut::from_vec(owned_to_vec::(data, ptr, len)) +unsafe fn owned_to_mut(owned: *mut (), ptr: *const u8, len: usize) -> BytesMut { + BytesMut::from_vec(owned_to_vec::(owned, ptr, len)) } unsafe fn owned_is_unique(_data: &AtomicPtr<()>) -> bool { @@ -1141,9 +1153,8 @@ unsafe fn owned_drop_impl(owned: *mut ()) { drop(Box::>::from_raw(owned.cast())); } -unsafe fn owned_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) { - let owned = data.load(Ordering::Relaxed); - owned_drop_impl::(owned); +unsafe fn owned_drop(data: *mut (), _ptr: *const u8, _len: usize) { + owned_drop_impl::(data); } // ===== impl PromotableVtable ===== @@ -1178,12 +1189,11 @@ unsafe fn promotable_even_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize } unsafe fn promotable_to_vec( - data: &AtomicPtr<()>, + shared: *mut (), ptr: *const u8, len: usize, f: fn(*mut ()) -> *mut u8, ) -> Vec { - let shared = data.load(Ordering::Acquire); let kind = shared as usize & KIND_MASK; if kind == KIND_ARC { @@ -1204,12 +1214,11 @@ unsafe fn promotable_to_vec( } unsafe fn promotable_to_mut( - data: &AtomicPtr<()>, + shared: *mut (), ptr: *const u8, len: usize, f: fn(*mut ()) -> *mut u8, ) -> BytesMut { - let shared = data.load(Ordering::Acquire); let kind = shared as usize & KIND_MASK; if kind == KIND_ARC { @@ -1232,31 +1241,28 @@ unsafe fn promotable_to_mut( } } -unsafe fn promotable_even_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec { - promotable_to_vec(data, ptr, len, |shared| { +unsafe fn promotable_even_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec { + promotable_to_vec(shared, ptr, len, |shared| { ptr_map(shared.cast(), |addr| addr & !KIND_MASK) }) } -unsafe fn promotable_even_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { - promotable_to_mut(data, ptr, len, |shared| { +unsafe fn promotable_even_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut { + promotable_to_mut(shared, ptr, len, |shared| { ptr_map(shared.cast(), |addr| addr & !KIND_MASK) }) } -unsafe fn promotable_even_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) { - data.with_mut(|shared| { - let shared = *shared; - let kind = shared as usize & KIND_MASK; +unsafe fn promotable_even_drop(shared: *mut (), ptr: *const u8, len: usize) { + let kind = shared as usize & KIND_MASK; - if kind == KIND_ARC { - release_shared(shared.cast()); - } else { - debug_assert_eq!(kind, KIND_VEC); - let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK); - free_boxed_slice(buf, ptr, len); - } - }); + if kind == KIND_ARC { + release_shared(shared.cast()); + } else { + debug_assert_eq!(kind, KIND_VEC); + let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK); + free_boxed_slice(buf, ptr, len); + } } unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { @@ -1271,27 +1277,24 @@ unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) } } -unsafe fn promotable_odd_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec { - promotable_to_vec(data, ptr, len, |shared| shared.cast()) +unsafe fn promotable_odd_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec { + promotable_to_vec(shared, ptr, len, |shared| shared.cast()) } -unsafe fn promotable_odd_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { - promotable_to_mut(data, ptr, len, |shared| shared.cast()) +unsafe fn promotable_odd_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut { + promotable_to_mut(shared, ptr, len, |shared| shared.cast()) } -unsafe fn promotable_odd_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) { - data.with_mut(|shared| { - let shared = *shared; - let kind = shared as usize & KIND_MASK; +unsafe fn promotable_odd_drop(shared: *mut (), ptr: *const u8, len: usize) { + let kind = shared as usize & KIND_MASK; - if kind == KIND_ARC { - release_shared(shared.cast()); - } else { - debug_assert_eq!(kind, KIND_VEC); + if kind == KIND_ARC { + release_shared(shared.cast()); + } else { + debug_assert_eq!(kind, KIND_VEC); - free_boxed_slice(shared.cast(), ptr, len); - } - }); + free_boxed_slice(shared.cast(), ptr, len); + } } unsafe fn promotable_is_unique(data: &AtomicPtr<()>) -> bool { @@ -1378,8 +1381,8 @@ unsafe fn shared_to_vec_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> } } -unsafe fn shared_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec { - shared_to_vec_impl(data.load(Ordering::Relaxed).cast(), ptr, len) +unsafe fn shared_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec { + shared_to_vec_impl(shared.cast(), ptr, len) } unsafe fn shared_to_mut_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> BytesMut { @@ -1417,8 +1420,8 @@ unsafe fn shared_to_mut_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> } } -unsafe fn shared_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { - shared_to_mut_impl(data.load(Ordering::Relaxed).cast(), ptr, len) +unsafe fn shared_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut { + shared_to_mut_impl(shared.cast(), ptr, len) } pub(crate) unsafe fn shared_is_unique(data: &AtomicPtr<()>) -> bool { @@ -1427,10 +1430,8 @@ pub(crate) unsafe fn shared_is_unique(data: &AtomicPtr<()>) -> bool { ref_cnt == 1 } -unsafe fn shared_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) { - data.with_mut(|shared| { - release_shared(shared.cast()); - }); +unsafe fn shared_drop(shared: *mut (), _ptr: *const u8, _len: usize) { + release_shared(shared.cast()); } unsafe fn shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes { diff --git a/src/bytes_mut.rs b/src/bytes_mut.rs index 78c5fd6..9c3ee72 100644 --- a/src/bytes_mut.rs +++ b/src/bytes_mut.rs @@ -1878,8 +1878,8 @@ unsafe fn shared_v_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> By Bytes::with_vtable(ptr, len, data, &SHARED_VTABLE) } -unsafe fn shared_v_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec { - let shared: *mut Shared = data.load(Ordering::Relaxed).cast(); +unsafe fn shared_v_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec { + let shared: *mut Shared = shared.cast(); if (*shared).is_unique() { let shared = &mut *shared; @@ -1900,8 +1900,8 @@ unsafe fn shared_v_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> V } } -unsafe fn shared_v_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { - let shared: *mut Shared = data.load(Ordering::Relaxed).cast(); +unsafe fn shared_v_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut { + let shared: *mut Shared = shared.cast(); if (*shared).is_unique() { let shared = &mut *shared; @@ -1935,10 +1935,8 @@ unsafe fn shared_v_is_unique(data: &AtomicPtr<()>) -> bool { ref_count == 1 } -unsafe fn shared_v_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) { - data.with_mut(|shared| { - release_shared(*shared as *mut Shared); - }); +unsafe fn shared_v_drop(shared: *mut (), _ptr: *const u8, _len: usize) { + release_shared(shared.cast()); } // compile-fails