tokio-rs/bytes
1
0
Fork 0
mirror of https://github.com/tokio-rs/bytes.git synced 2026-08-07 00:00:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

Pass vtable data by value (#826)

Browse Source
This commit is contained in:
DaniPopes
2026-04-29 07:28:59 +02:00
committed by GitHub
parent 00cc5ff2bd
commit 245adff079
2 changed files with 68 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/bytes.rs
+62 -61
View File
@@ -106,18 +106,22 @@ pub struct Bytes {
vtable: &'static Vtable, vtable: &'static Vtable,
} }
// `data` is passed by value (`*mut ()` instead of `&mut AtomicPtr<()>`)
// when `&mut self` or `self` is consumed.
// This allows the optimizer to see that the address of the `Bytes` is not
// captured by the indirect call, enabling further optimizations.
pub(crate) struct Vtable { pub(crate) struct Vtable {
/// fn(data, ptr, len) /// fn(data, ptr, len)
pub clone: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Bytes, pub clone: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Bytes,
/// fn(data, ptr, len) /// fn(data, ptr, len)
/// ///
/// `into_*` consumes the `Bytes`, returning the respective value. /// `into_*` consumes the `Bytes`, returning the respective value.
pub into_vec: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> Vec<u8>, pub into_vec: unsafe fn(*mut (), *const u8, usize) -> Vec<u8>,
pub into_mut: unsafe fn(&AtomicPtr<()>, *const u8, usize) -> BytesMut, pub into_mut: unsafe fn(*mut (), *const u8, usize) -> BytesMut,
/// fn(data) /// fn(data)
pub is_unique: unsafe fn(&AtomicPtr<()>) -> bool, pub is_unique: unsafe fn(&AtomicPtr<()>) -> bool,
/// fn(data, ptr, len) /// fn(data, ptr, len)
pub drop: unsafe fn(&mut AtomicPtr<()>, *const u8, usize), pub drop: unsafe fn(*mut (), *const u8, usize),
} }
impl Bytes { impl Bytes {
@@ -644,6 +648,11 @@ impl Bytes {
self.len -= by; self.len -= by;
self.ptr = self.ptr.add(by); self.ptr = self.ptr.add(by);
} }
#[inline]
fn data_mut(&mut self) -> *mut () {
self.data.with_mut(|p| *p)
}
} }
// Vtable must enforce this behavior // Vtable must enforce this behavior
@@ -653,7 +662,8 @@ unsafe impl Sync for Bytes {}
impl Drop for Bytes { impl Drop for Bytes {
#[inline] #[inline]
fn drop(&mut self) { fn drop(&mut self) {
unsafe { (self.vtable.drop)(&mut self.data, self.ptr, self.len) } let data = self.data_mut();
unsafe { (self.vtable.drop)(data, self.ptr, self.len) }
} }
} }
@@ -1013,8 +1023,9 @@ impl From<Bytes> for BytesMut {
/// assert_eq!(BytesMut::from(bytes), BytesMut::from(&b"hello"[..])); /// assert_eq!(BytesMut::from(bytes), BytesMut::from(&b"hello"[..]));
/// ``` /// ```
fn from(bytes: Bytes) -> Self { fn from(bytes: Bytes) -> Self {
let bytes = ManuallyDrop::new(bytes); let mut bytes = ManuallyDrop::new(bytes);
unsafe { (bytes.vtable.into_mut)(&bytes.data, bytes.ptr, bytes.len) } let data = bytes.data_mut();
unsafe { (bytes.vtable.into_mut)(data, bytes.ptr, bytes.len) }
} }
} }
@@ -1026,8 +1037,9 @@ impl From<String> for Bytes {
impl From<Bytes> for Vec<u8> { impl From<Bytes> for Vec<u8> {
fn from(bytes: Bytes) -> Vec<u8> { fn from(bytes: Bytes) -> Vec<u8> {
let bytes = ManuallyDrop::new(bytes); let mut bytes = ManuallyDrop::new(bytes);
unsafe { (bytes.vtable.into_vec)(&bytes.data, bytes.ptr, bytes.len) } let data = bytes.data_mut();
unsafe { (bytes.vtable.into_vec)(data, bytes.ptr, bytes.len) }
} }
} }
@@ -1057,12 +1069,12 @@ unsafe fn static_clone(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
Bytes::from_static(slice) Bytes::from_static(slice)
} }
unsafe fn static_to_vec(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { unsafe fn static_to_vec(_: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
let slice = slice::from_raw_parts(ptr, len); let slice = slice::from_raw_parts(ptr, len);
slice.to_vec() slice.to_vec()
} }
unsafe fn static_to_mut(_: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { unsafe fn static_to_mut(_: *mut (), ptr: *const u8, len: usize) -> BytesMut {
let slice = slice::from_raw_parts(ptr, len); let slice = slice::from_raw_parts(ptr, len);
BytesMut::from(slice) BytesMut::from(slice)
} }
@@ -1071,7 +1083,7 @@ fn static_is_unique(_: &AtomicPtr<()>) -> bool {
false false
} }
unsafe fn static_drop(_: &mut AtomicPtr<()>, _: *const u8, _: usize) { unsafe fn static_drop(_: *mut (), _: *const u8, _: usize) {
// nothing to drop for &'static [u8] // nothing to drop for &'static [u8]
} }
@@ -1108,15 +1120,15 @@ unsafe fn owned_clone<T>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> By
} }
} }
unsafe fn owned_to_vec<T>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { unsafe fn owned_to_vec<T>(owned: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
let slice = slice::from_raw_parts(ptr, len); let slice = slice::from_raw_parts(ptr, len);
let vec = slice.to_vec(); let vec = slice.to_vec();
owned_drop_impl::<T>(data.load(Ordering::Relaxed)); owned_drop_impl::<T>(owned);
vec vec
} }
unsafe fn owned_to_mut<T>(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { unsafe fn owned_to_mut<T>(owned: *mut (), ptr: *const u8, len: usize) -> BytesMut {
BytesMut::from_vec(owned_to_vec::<T>(data, ptr, len)) BytesMut::from_vec(owned_to_vec::<T>(owned, ptr, len))
} }
unsafe fn owned_is_unique(_data: &AtomicPtr<()>) -> bool { unsafe fn owned_is_unique(_data: &AtomicPtr<()>) -> bool {
@@ -1141,9 +1153,8 @@ unsafe fn owned_drop_impl<T>(owned: *mut ()) {
drop(Box::<Owned<T>>::from_raw(owned.cast())); drop(Box::<Owned<T>>::from_raw(owned.cast()));
} }
unsafe fn owned_drop<T>(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) { unsafe fn owned_drop<T>(data: *mut (), _ptr: *const u8, _len: usize) {
let owned = data.load(Ordering::Relaxed); owned_drop_impl::<T>(data);
owned_drop_impl::<T>(owned);
} }
// ===== impl PromotableVtable ===== // ===== impl PromotableVtable =====
@@ -1178,12 +1189,11 @@ unsafe fn promotable_even_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize
} }
unsafe fn promotable_to_vec( unsafe fn promotable_to_vec(
data: &AtomicPtr<()>, shared: *mut (),
ptr: *const u8, ptr: *const u8,
len: usize, len: usize,
f: fn(*mut ()) -> *mut u8, f: fn(*mut ()) -> *mut u8,
) -> Vec<u8> { ) -> Vec<u8> {
let shared = data.load(Ordering::Acquire);
let kind = shared as usize & KIND_MASK; let kind = shared as usize & KIND_MASK;
if kind == KIND_ARC { if kind == KIND_ARC {
@@ -1204,12 +1214,11 @@ unsafe fn promotable_to_vec(
} }
unsafe fn promotable_to_mut( unsafe fn promotable_to_mut(
data: &AtomicPtr<()>, shared: *mut (),
ptr: *const u8, ptr: *const u8,
len: usize, len: usize,
f: fn(*mut ()) -> *mut u8, f: fn(*mut ()) -> *mut u8,
) -> BytesMut { ) -> BytesMut {
let shared = data.load(Ordering::Acquire);
let kind = shared as usize & KIND_MASK; let kind = shared as usize & KIND_MASK;
if kind == KIND_ARC { if kind == KIND_ARC {
@@ -1232,31 +1241,28 @@ unsafe fn promotable_to_mut(
} }
} }
unsafe fn promotable_even_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { unsafe fn promotable_even_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
promotable_to_vec(data, ptr, len, |shared| { promotable_to_vec(shared, ptr, len, |shared| {
ptr_map(shared.cast(), |addr| addr & !KIND_MASK) ptr_map(shared.cast(), |addr| addr & !KIND_MASK)
}) })
} }
unsafe fn promotable_even_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { unsafe fn promotable_even_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
promotable_to_mut(data, ptr, len, |shared| { promotable_to_mut(shared, ptr, len, |shared| {
ptr_map(shared.cast(), |addr| addr & !KIND_MASK) ptr_map(shared.cast(), |addr| addr & !KIND_MASK)
}) })
} }
unsafe fn promotable_even_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) { unsafe fn promotable_even_drop(shared: *mut (), ptr: *const u8, len: usize) {
data.with_mut(|shared| { let kind = shared as usize & KIND_MASK;
let shared = *shared;
let kind = shared as usize & KIND_MASK;
if kind == KIND_ARC { if kind == KIND_ARC {
release_shared(shared.cast()); release_shared(shared.cast());
} else { } else {
debug_assert_eq!(kind, KIND_VEC); debug_assert_eq!(kind, KIND_VEC);
let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK); let buf = ptr_map(shared.cast(), |addr| addr & !KIND_MASK);
free_boxed_slice(buf, ptr, len); free_boxed_slice(buf, ptr, len);
} }
});
} }
unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes { unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Bytes {
@@ -1271,27 +1277,24 @@ unsafe fn promotable_odd_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize)
} }
} }
unsafe fn promotable_odd_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { unsafe fn promotable_odd_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
promotable_to_vec(data, ptr, len, |shared| shared.cast()) promotable_to_vec(shared, ptr, len, |shared| shared.cast())
} }
unsafe fn promotable_odd_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { unsafe fn promotable_odd_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
promotable_to_mut(data, ptr, len, |shared| shared.cast()) promotable_to_mut(shared, ptr, len, |shared| shared.cast())
} }
unsafe fn promotable_odd_drop(data: &mut AtomicPtr<()>, ptr: *const u8, len: usize) { unsafe fn promotable_odd_drop(shared: *mut (), ptr: *const u8, len: usize) {
data.with_mut(|shared| { let kind = shared as usize & KIND_MASK;
let shared = *shared;
let kind = shared as usize & KIND_MASK;
if kind == KIND_ARC { if kind == KIND_ARC {
release_shared(shared.cast()); release_shared(shared.cast());
} else { } else {
debug_assert_eq!(kind, KIND_VEC); debug_assert_eq!(kind, KIND_VEC);
free_boxed_slice(shared.cast(), ptr, len); free_boxed_slice(shared.cast(), ptr, len);
} }
});
} }
unsafe fn promotable_is_unique(data: &AtomicPtr<()>) -> bool { unsafe fn promotable_is_unique(data: &AtomicPtr<()>) -> bool {
@@ -1378,8 +1381,8 @@ unsafe fn shared_to_vec_impl(shared: *mut Shared, ptr: *const u8, len: usize) ->
} }
} }
unsafe fn shared_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { unsafe fn shared_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
shared_to_vec_impl(data.load(Ordering::Relaxed).cast(), ptr, len) shared_to_vec_impl(shared.cast(), ptr, len)
} }
unsafe fn shared_to_mut_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> BytesMut { unsafe fn shared_to_mut_impl(shared: *mut Shared, ptr: *const u8, len: usize) -> BytesMut {
@@ -1417,8 +1420,8 @@ unsafe fn shared_to_mut_impl(shared: *mut Shared, ptr: *const u8, len: usize) ->
} }
} }
unsafe fn shared_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { unsafe fn shared_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
shared_to_mut_impl(data.load(Ordering::Relaxed).cast(), ptr, len) shared_to_mut_impl(shared.cast(), ptr, len)
} }
pub(crate) unsafe fn shared_is_unique(data: &AtomicPtr<()>) -> bool { pub(crate) unsafe fn shared_is_unique(data: &AtomicPtr<()>) -> bool {
@@ -1427,10 +1430,8 @@ pub(crate) unsafe fn shared_is_unique(data: &AtomicPtr<()>) -> bool {
ref_cnt == 1 ref_cnt == 1
} }
unsafe fn shared_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) { unsafe fn shared_drop(shared: *mut (), _ptr: *const u8, _len: usize) {
data.with_mut(|shared| { release_shared(shared.cast());
release_shared(shared.cast());
});
} }
unsafe fn shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes { unsafe fn shallow_clone_arc(shared: *mut Shared, ptr: *const u8, len: usize) -> Bytes {
src/bytes_mut.rs
+6 -8
View File
@@ -1878,8 +1878,8 @@ unsafe fn shared_v_clone(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> By
Bytes::with_vtable(ptr, len, data, &SHARED_VTABLE) Bytes::with_vtable(ptr, len, data, &SHARED_VTABLE)
} }
unsafe fn shared_v_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> Vec<u8> { unsafe fn shared_v_to_vec(shared: *mut (), ptr: *const u8, len: usize) -> Vec<u8> {
let shared: *mut Shared = data.load(Ordering::Relaxed).cast(); let shared: *mut Shared = shared.cast();
if (*shared).is_unique() { if (*shared).is_unique() {
let shared = &mut *shared; let shared = &mut *shared;
@@ -1900,8 +1900,8 @@ unsafe fn shared_v_to_vec(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> V
} }
} }
unsafe fn shared_v_to_mut(data: &AtomicPtr<()>, ptr: *const u8, len: usize) -> BytesMut { unsafe fn shared_v_to_mut(shared: *mut (), ptr: *const u8, len: usize) -> BytesMut {
let shared: *mut Shared = data.load(Ordering::Relaxed).cast(); let shared: *mut Shared = shared.cast();
if (*shared).is_unique() { if (*shared).is_unique() {
let shared = &mut *shared; let shared = &mut *shared;
@@ -1935,10 +1935,8 @@ unsafe fn shared_v_is_unique(data: &AtomicPtr<()>) -> bool {
ref_count == 1 ref_count == 1
} }
unsafe fn shared_v_drop(data: &mut AtomicPtr<()>, _ptr: *const u8, _len: usize) { unsafe fn shared_v_drop(shared: *mut (), _ptr: *const u8, _len: usize) {
data.with_mut(|shared| { release_shared(shared.cast());
release_shared(*shared as *mut Shared);
});
} }
// compile-fails // compile-fails