Pass vtable data by value (#826)

src/bytes.rs

@@ -106,18 +106,22 @@ pub struct Bytes {
     vtable: &'static Vtable,
 }
 
+// `data` is passed by value (`*mut ()` instead of `&mut AtomicPtr<()>`)
+// when `&mut self` or `self` is consumed.
+// This allows the optimizer to see that the address of the `Bytes` is not
+// captured by the indirect call, enabling further optimizations.
 pub(crate) struct Vtable {
     /// fn(data, ptr, len)
     pub clone: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Bytes,
     /// fn(data, ptr, len)
     ///
     /// `into_*` consumes the `Bytes`, returning the respective value.
-    pub into_vec: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Vec<u8>,
-    pub into_mut: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> BytesMut,
+    pub into_vec: unsafe fn(*mut (), *const u8, usize) -> Vec<u8>,
+    pub into_mut: unsafe fn(*mut (), *const u8, usize) -> BytesMut,
     /// fn(data)
     pub is_unique: unsafe fn(&AtomicPtr<()>) -> bool,
     /// fn(data, ptr, len)
-    pub drop: unsafe fn(&mut AtomicPtr<()>, *const u8, usize),
+    pub drop: unsafe fn(*mut (), *const u8, usize),
 }
 
 impl Bytes {
@@ -644,6 +648,11 @@ impl Bytes {
         self.len -= by;
         self.ptr = self.ptr.add(by);
     }
+
+    #[inline]
+    fn data_mut(&mut self) -> *mut () {
+        self.data.with_mut(|p| *p)
+    }
 }
 
 // Vtable must enforce this behavior
@@ -653,7 +662,8 @@ unsafe impl Sync for Bytes {}
 impl Drop for Bytes {
     #[inline]
     fn drop(&mut self) {
-        unsafe { (self.vtable.drop)(&mut self.data, self.ptr, self.len) }
+        let data = self.data_mut();
+        unsafe { (self.vtable.drop)(data, self.ptr, self.len) }
     }
 }
 
@@ -1013,8 +1023,9 @@ impl From<Bytes> for BytesMut {
     /// assert_eq!(BytesMut::from(bytes), BytesMut::from(&b"hello"[..]));
     /// ```
     fn from(bytes: Bytes) -> Self {
-        let bytes = ManuallyDrop::new(bytes);
-        unsafe { (bytes.vtable.into_mut)(&bytes.data, bytes.ptr, bytes.len) }
+        let mut bytes = ManuallyDrop::new(bytes);
+        let data = bytes.data_mut();
+        unsafe { (bytes.vtable.into_mut)(data, bytes.ptr, bytes.len) }
     }
 }
 
@@ -1026,8 +1037,9 @@ impl From<String> for Bytes {
 
 impl From<Bytes> for Vec<u8> {
     fn from(bytes: Bytes) -> Vec<u8> {
-        let bytes = ManuallyDrop::new(bytes);
-        unsafe { (bytes.vtable.into_vec)(&bytes.data, bytes.ptr, bytes.len) }
+        let mut bytes = ManuallyDrop::new(bytes);
+        let data = bytes.data_mut();
+        unsafe { (bytes.vtable.into_vec)(data, bytes.ptr, bytes.len) }
     }
 }
 
@@ -1057,12 +1069,12 @@ unsafe fn static_clone(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
     Bytes::from_static(slice)
 }
 
-unsafe fn static_to_vec(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
+unsafe fn static_to_vec(_: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
     let slice = slice::from_raw_parts(ptr, len);
     slice.to_vec()
 }
 
-unsafe fn static_to_mut(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut {
+unsafe fn static_to_mut(_: *mut (), ptr: *const u8, len: usize) -> BytesMut {
     let slice = slice::from_raw_parts(ptr, len);
     BytesMut::from(slice)
 }
@@ -1071,7 +1083,7 @@ fn static_is_unique(_: &AtomicPtr<()>) -> bool {
     false
 }
 
-unsafe fn static_drop(_: &mut AtomicPtr<()>, _: *const u8, _: usize) {
+unsafe fn static_drop(_: *mut (), _: *const u8, _: usize) {
     // nothing to drop for &'static [u8]
 }
 
@@ -1108,15 +1120,15 @@ unsafe fn owned_clone<T>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> By
     }
 }
 
-unsafe fn owned_to_vec<T>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
+unsafe fn owned_to_vec<T>(owned: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
     let slice = slice::from_raw_parts(ptr, len);
     let vec = slice.to_vec();
-    owned_drop_impl::<T>(data.load(Ordering::Relaxed));
+    owned_drop_impl::<T>(owned);
     vec
 }
 
-unsafe fn owned_to_mut<T>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut {
-    BytesMut::from_vec(owned_to_vec::<T>(data, ptr, len))
+unsafe fn owned_to_mut<T>(owned: *mut (), ptr: *const u8, len: usize) -> BytesMut {
+    BytesMut::from_vec(owned_to_vec::<T>(owned, ptr, len))
 }
 
 unsafe fn owned_is_unique(_data: &AtomicPtr<()>) -> bool {
@@ -1141,9 +1153,8 @@ unsafe fn owned_drop_impl<T>(owned: *mut ()) {
     drop(Box::<Owned<T>>::from_raw(owned.cast()));
 }
 
-unsafe fn owned_drop<T>(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) {
-    let owned = data.load(Ordering::Relaxed);
-    owned_drop_impl::<T>(owned);
+unsafe fn owned_drop<T>(data: *mut (), _ptr: *const u8, _len: usize) {
+    owned_drop_impl::<T>(data);
 }
 
 // ===== impl PromotableVtable =====
@@ -1178,12 +1189,11 @@ unsafe fn promotable_even_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize
 }
 
 unsafe fn promotable_to_vec(
-    data: &AtomicPtr<()>,
+    shared: *mut (),
     ptr: *const u8,
     len: usize,
     f: fn(*mut ()) -> *mut u8,
 ) -> Vec<u8> {
-    let shared = data.load(Ordering::Acquire);
     let kind = shared as usize & KIND_MASK;
 
     if kind == KIND_ARC {
@@ -1204,12 +1214,11 @@ unsafe fn promotable_to_vec(
 }
 
 unsafe fn promotable_to_mut(
-    data: &AtomicPtr<()>,
+    shared: *mut (),
     ptr: *const u8,
     len: usize,
     f: fn(*mut ()) -> *mut u8,
 ) -> BytesMut {
-    let shared = data.load(Ordering::Acquire);
     let kind = shared as usize & KIND_MASK;
 
     if kind == KIND_ARC {
@@ -1232,31 +1241,28 @@ unsafe fn promotable_to_mut(
     }
 }
 
-unsafe fn promotable_even_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
-    promotable_to_vec(data, ptr, len, |shared| {
+unsafe fn promotable_even_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
+    promotable_to_vec(shared, ptr, len, |shared| {
         ptr_map(shared.cast(), |addr| addr & !KIND_MASK)
     })
 }
 
-unsafe fn promotable_even_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut {
-    promotable_to_mut(data, ptr, len, |shared| {
+unsafe fn promotable_even_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
+    promotable_to_mut(shared, ptr, len, |shared| {
         ptr_map(shared.cast(), |addr| addr & !KIND_MASK)
     })
 }
 
-unsafe fn promotable_even_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) {
-    data.with_mut(|shared| {
-        let shared = *shared;
-        let kind = shared as usize & KIND_MASK;
+unsafe fn promotable_even_drop(shared: *mut (), ptr: *const u8, len: usize) {
+    let kind = shared as usize & KIND_MASK;
 
-        if kind == KIND_ARC {
-            release_shared(shared.cast());
-        } else {
-            debug_assert_eq!(kind, KIND_VEC);
-            let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK);
-            free_boxed_slice(buf, ptr, len);
-        }
-    });
+    if kind == KIND_ARC {
+        release_shared(shared.cast());
+    } else {
+        debug_assert_eq!(kind, KIND_VEC);
+        let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK);
+        free_boxed_slice(buf, ptr, len);
+    }
 }
 
 unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
@@ -1271,27 +1277,24 @@ unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize)
     }
 }
 
-unsafe fn promotable_odd_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
-    promotable_to_vec(data, ptr, len, |shared| shared.cast())
+unsafe fn promotable_odd_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
+    promotable_to_vec(shared, ptr, len, |shared| shared.cast())
 }
 
-unsafe fn promotable_odd_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut {
-    promotable_to_mut(data, ptr, len, |shared| shared.cast())
+unsafe fn promotable_odd_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
+    promotable_to_mut(shared, ptr, len, |shared| shared.cast())
 }
 
-unsafe fn promotable_odd_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) {
-    data.with_mut(|shared| {
-        let shared = *shared;
-        let kind = shared as usize & KIND_MASK;
+unsafe fn promotable_odd_drop(shared: *mut (), ptr: *const u8, len: usize) {
+    let kind = shared as usize & KIND_MASK;
 
-        if kind == KIND_ARC {
-            release_shared(shared.cast());
-        } else {
-            debug_assert_eq!(kind, KIND_VEC);
+    if kind == KIND_ARC {
+        release_shared(shared.cast());
+    } else {
+        debug_assert_eq!(kind, KIND_VEC);
 
-            free_boxed_slice(shared.cast(), ptr, len);
-        }
-    });
+        free_boxed_slice(shared.cast(), ptr, len);
+    }
 }
 
 unsafe fn promotable_is_unique(data: &AtomicPtr<()>) -> bool {
@@ -1378,8 +1381,8 @@ unsafe fn shared_to_vec_impl(shared: *mut Shared, ptr: *const u8, len: usize) ->
     }
 }
 
-unsafe fn shared_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
-    shared_to_vec_impl(data.load(Ordering::Relaxed).cast(), ptr, len)
+unsafe fn shared_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
+    shared_to_vec_impl(shared.cast(), ptr, len)
 }
 
 unsafe fn shared_to_mut_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> BytesMut {
@@ -1417,8 +1420,8 @@ unsafe fn shared_to_mut_impl(shared: *mut Shared, ptr: *const u8, len: usize) ->
     }
 }
 
-unsafe fn shared_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut {
-    shared_to_mut_impl(data.load(Ordering::Relaxed).cast(), ptr, len)
+unsafe fn shared_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
+    shared_to_mut_impl(shared.cast(), ptr, len)
 }
 
 pub(crate) unsafe fn shared_is_unique(data: &AtomicPtr<()>) -> bool {
@@ -1427,10 +1430,8 @@ pub(crate) unsafe fn shared_is_unique(data: &AtomicPtr<()>) -> bool {
     ref_cnt == 1
 }
 
-unsafe fn shared_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) {
-    data.with_mut(|shared| {
-        release_shared(shared.cast());
-    });
+unsafe fn shared_drop(shared: *mut (), _ptr: *const u8, _len: usize) {
+    release_shared(shared.cast());
 }
 
 unsafe fn shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes {

src/bytes_mut.rs

@@ -1878,8 +1878,8 @@ unsafe fn shared_v_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> By
     Bytes::with_vtable(ptr, len, data, &SHARED_VTABLE)
 }
 
-unsafe fn shared_v_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> {
-    let shared: *mut Shared = data.load(Ordering::Relaxed).cast();
+unsafe fn shared_v_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
+    let shared: *mut Shared = shared.cast();
 
     if (*shared).is_unique() {
         let shared = &mut *shared;
@@ -1900,8 +1900,8 @@ unsafe fn shared_v_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> V
     }
 }
 
-unsafe fn shared_v_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut {
-    let shared: *mut Shared = data.load(Ordering::Relaxed).cast();
+unsafe fn shared_v_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
+    let shared: *mut Shared = shared.cast();
 
     if (*shared).is_unique() {
         let shared = &mut *shared;
@@ -1935,10 +1935,8 @@ unsafe fn shared_v_is_unique(data: &AtomicPtr<()>) -> bool {
     ref_count == 1
 }
 
-unsafe fn shared_v_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) {
-    data.with_mut(|shared| {
-        release_shared(*shared as *mut Shared);
-    });
+unsafe fn shared_v_drop(shared: *mut (), _ptr: *const u8, _len: usize) {
+    release_shared(shared.cast());
 }
 
 // compile-fails