diff --git a/vm/src/function/argument.rs b/vm/src/function/argument.rs index 18a9a81b7..aa01450fe 100644 --- a/vm/src/function/argument.rs +++ b/vm/src/function/argument.rs @@ -14,19 +14,20 @@ pub struct ArgCallable { } impl ArgCallable { - #[inline] pub fn invoke(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult { vm.invoke(&self.obj, args) } } impl Borrow for ArgCallable { + #[inline(always)] fn borrow(&self) -> &PyObject { &self.obj } } impl AsRef for ArgCallable { + #[inline(always)] fn as_ref(&self) -> &PyObject { &self.obj } @@ -116,24 +117,28 @@ impl ArgMapping { } impl Borrow for ArgMapping { + #[inline(always)] fn borrow(&self) -> &PyObject { &self.obj } } impl AsRef for ArgMapping { + #[inline(always)] fn as_ref(&self) -> &PyObject { &self.obj } } impl PyObjectWrap for ArgMapping { + #[inline(always)] fn into_object(self) -> PyObjectRef { self.obj } } impl IntoPyObject for ArgMapping { + #[inline(always)] fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { self.obj } diff --git a/vm/src/protocol/iter.rs b/vm/src/protocol/iter.rs index 1ccfce00f..c651ddf67 100644 --- a/vm/src/protocol/iter.rs +++ b/vm/src/protocol/iter.rs @@ -80,6 +80,7 @@ impl Borrow for PyIter where O: Borrow, { + #[inline(always)] fn borrow(&self) -> &PyObject { self.0.borrow() } @@ -89,6 +90,7 @@ impl AsRef for PyIter where O: Borrow, { + #[inline(always)] fn as_ref(&self) -> &PyObject { self.0.borrow() } @@ -99,12 +101,14 @@ where O: Borrow, { type Target = PyObject; + #[inline(always)] fn deref(&self) -> &Self::Target { self.0.borrow() } } impl IntoPyObject for PyIter { + #[inline(always)] fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { self.into() } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 0054c190a..f3f40ddbd 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -134,14 +134,17 @@ impl PyContext { context } + #[inline(always)] pub fn none(&self) -> PyObjectRef { self.none.clone().into() } + #[inline(always)] pub fn ellipsis(&self) -> PyObjectRef { self.ellipsis.clone().into() } + #[inline(always)] pub fn not_implemented(&self) -> PyObjectRef { self.not_implemented.clone().into() } @@ -170,19 +173,22 @@ impl PyContext { PyRef::new_ref(PyInt::from(i.clone()), self.types.int_type.clone(), None) } + #[inline] pub fn new_float(&self, value: f64) -> PyRef { PyRef::new_ref(PyFloat::from(value), self.types.float_type.clone(), None) } + #[inline] pub fn new_str(&self, s: impl Into) -> PyRef { pystr::PyStr::new_ref(s, self) } + #[inline] pub fn new_bytes(&self, data: Vec) -> PyRef { bytes::PyBytes::new_ref(data, self) } - #[inline] + #[inline(always)] pub fn new_bool(&self, b: bool) -> PyIntRef { let value = if b { &self.true_value @@ -192,14 +198,17 @@ impl PyContext { value.clone() } + #[inline(always)] pub fn new_tuple(&self, elements: Vec) -> PyTupleRef { PyTuple::new_ref(elements, self) } + #[inline(always)] pub fn new_list(&self, elements: Vec) -> PyListRef { PyList::new_ref(elements, self) } + #[inline(always)] pub fn new_dict(&self) -> PyDictRef { PyDict::new_ref(self) } @@ -376,12 +385,13 @@ impl TryFromObject for PyRefExact { } impl Deref for PyRefExact { type Target = PyRef; + #[inline(always)] fn deref(&self) -> &PyRef { &self.obj } } impl IntoPyObject for PyRefExact { - #[inline] + #[inline(always)] fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { self.obj.into() } @@ -418,6 +428,7 @@ impl PyObject { } // impl Borrow for PyRc { +// #[inline(always)] // fn borrow(&self) -> &PyObject { // unsafe { &*(&**self as *const T as *const PyObject) } // } @@ -432,12 +443,14 @@ pub struct PyLease<'a, T: PyObjectPayload> { impl<'a, T: PyObjectPayload + PyValue> PyLease<'a, T> { // Associated function on purpose, because of deref #[allow(clippy::wrong_self_convention)] + #[inline(always)] pub fn into_pyref(zelf: Self) -> PyRef { zelf.inner.clone() } } impl<'a, T: PyObjectPayload + PyValue> Borrow for PyLease<'a, T> { + #[inline(always)] fn borrow(&self) -> &PyObject { self.inner.as_ref() } @@ -445,7 +458,7 @@ impl<'a, T: PyObjectPayload + PyValue> Borrow for PyLease<'a, T> { impl<'a, T: PyObjectPayload + PyValue> Deref for PyLease<'a, T> { type Target = PyRef; - + #[inline(always)] fn deref(&self) -> &Self::Target { &self.inner } @@ -521,27 +534,28 @@ impl IntoPyRef

for T where P: PyValue + IntoPyObject + From, { + #[inline(always)] fn into_pyref(self, vm: &VirtualMachine) -> PyRef

{ P::from(self).into_ref(vm) } } impl IntoPyObject for PyRef { - #[inline] + #[inline(always)] fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { self.into() } } impl IntoPyObject for PyObjectRef { - #[inline] + #[inline(always)] fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { self } } impl IntoPyObject for &PyObject { - #[inline] + #[inline(always)] fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef { self.to_owned() } @@ -553,7 +567,7 @@ impl IntoPyObject for T where T: PyValue + Sized, { - #[inline] + #[inline(always)] fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { PyValue::into_object(self, vm) } @@ -563,6 +577,7 @@ impl IntoPyResult for T where T: IntoPyObject, { + #[inline(always)] fn into_pyresult(self, vm: &VirtualMachine) -> PyResult { Ok(self.into_pyobject(vm)) } @@ -572,6 +587,7 @@ impl IntoPyResult for PyResult where T: IntoPyObject, { + #[inline(always)] fn into_pyresult(self, vm: &VirtualMachine) -> PyResult { self.map(|res| T::into_pyobject(res, vm)) } @@ -600,6 +616,7 @@ pub trait PyValue: fmt::Debug + PyThreadingConstraint + Sized + 'static { None } + #[inline] fn _into_ref(self, cls: PyTypeRef, vm: &VirtualMachine) -> PyRef { let dict = if cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT) { Some(vm.ctx.new_dict()) @@ -609,24 +626,36 @@ pub trait PyValue: fmt::Debug + PyThreadingConstraint + Sized + 'static { PyRef::new_ref(self, cls, dict) } + #[inline] fn into_ref(self, vm: &VirtualMachine) -> PyRef { let cls = Self::class(vm); self._into_ref(cls.clone(), vm) } + #[cold] + fn _into_ref_with_type_error( + vm: &VirtualMachine, + cls: &PyTypeRef, + exact_class: &PyTypeRef, + ) -> PyBaseExceptionRef { + vm.new_type_error(format!( + "'{}' is not a subtype of '{}'", + &cls.name(), + exact_class.name() + )) + } + + #[inline] fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyTypeRef) -> PyResult> { let exact_class = Self::class(vm); if cls.issubclass(exact_class) { Ok(self._into_ref(cls, vm)) } else { - Err(vm.new_type_error(format!( - "'{}' is not a subtype of '{}'", - &cls.name(), - exact_class.name() - ))) + Err(Self::_into_ref_with_type_error(vm, &cls, exact_class)) } } + #[inline] fn into_pyresult_with_type(self, vm: &VirtualMachine, cls: PyTypeRef) -> PyResult { self.into_ref_with_type(vm, cls).into_pyresult(vm) } @@ -710,6 +739,7 @@ impl From for PyObjectRef where T: PyObjectWrap, { + #[inline(always)] fn from(py_ref: T) -> Self { py_ref.into_object() } diff --git a/vm/src/pyobjectrc.rs b/vm/src/pyobjectrc.rs index 0e1630c68..ec1841d1a 100644 --- a/vm/src/pyobjectrc.rs +++ b/vm/src/pyobjectrc.rs @@ -283,14 +283,17 @@ unsafe impl Link for WeakLink { type Target = PyObjectView; + #[inline(always)] fn as_raw(handle: &PyRef) -> NonNull { NonNull::from(&**handle) } + #[inline(always)] unsafe fn from_raw(ptr: NonNull) -> Self::Handle { PyRef::from_raw(ptr.as_ptr()) } + #[inline(always)] unsafe fn pointers(target: NonNull) -> NonNull> { NonNull::new_unchecked(ptr::addr_of_mut!((*target.as_ptr()).0.payload.pointers)) } @@ -331,7 +334,6 @@ impl PyWeak { guard.obj.is_none() } - #[inline(always)] fn drop_inner(&self) { let dealloc = { let mut guard = unsafe { self.parent.as_ref().lock() }; @@ -354,6 +356,7 @@ impl PyWeak { } impl Drop for PyWeak { + #[inline(always)] fn drop(&mut self) { // we do NOT have actual exclusive access! // no clue if doing this actually reduces chance of UB @@ -368,7 +371,7 @@ struct InstanceDict { } impl From for InstanceDict { - #[inline] + #[inline(always)] fn from(d: PyDictRef) -> Self { Self::new(d) } @@ -447,6 +450,7 @@ pub struct PyObject(PyInner); impl Deref for PyObjectRef { type Target = PyObject; + #[inline(always)] fn deref(&self) -> &PyObject { unsafe { self.ptr.as_ref() } } @@ -465,6 +469,7 @@ impl ToOwned for PyObject { } impl PyObjectRef { + #[inline(always)] pub fn into_raw(self) -> *const PyObject { let ptr = self.as_raw(); std::mem::forget(self); @@ -476,6 +481,7 @@ impl PyObjectRef { /// [`PyObjectRef::into_raw`]. The user is responsible for ensuring that the inner data is not /// dropped more than once due to mishandling the reference count by calling this function /// too many times. + #[inline(always)] pub unsafe fn from_raw(ptr: *const PyObject) -> Self { Self { ptr: NonNull::new_unchecked(ptr as *mut PyObject), @@ -486,6 +492,7 @@ impl PyObjectRef { /// /// If the downcast fails, the original ref is returned in as `Err` so /// another downcast can be attempted without unnecessary cloning. + #[inline(always)] pub fn downcast(self) -> Result, Self> { if self.payload_is::() { Ok(unsafe { PyRef::from_obj_unchecked(self) }) @@ -494,6 +501,7 @@ impl PyObjectRef { } } + #[inline(always)] pub fn downcast_ref(&self) -> Option<&PyObjectView> { if self.payload_is::() { // SAFETY: just checked that the payload is T, and PyRef is repr(transparent) over @@ -506,12 +514,14 @@ impl PyObjectRef { /// # Safety /// T must be the exact payload type + #[inline(always)] pub unsafe fn downcast_unchecked(self) -> PyRef { PyRef::from_obj_unchecked(self) } /// # Safety /// T must be the exact payload type + #[inline(always)] pub unsafe fn downcast_unchecked_ref(&self) -> &crate::PyObjectView { debug_assert!(self.payload_is::()); &*(self as *const PyObjectRef as *const PyRef) @@ -523,6 +533,7 @@ impl PyObjectRef { /// /// If the downcast fails, the original ref is returned in as `Err` so /// another downcast can be attempted without unnecessary cloning. + #[inline] pub fn downcast_exact( self, vm: &VirtualMachine, @@ -542,7 +553,7 @@ impl PyObjectRef { } impl PyObject { - #[inline] + #[inline(always)] fn weak_ref_list(&self) -> Option<&WeakRefList> { Some(&self.0.weak_list) } @@ -595,11 +606,12 @@ impl PyObject { self.weak_ref_list().map(|wrl| wrl.get_weak_references()) } - #[inline] + #[inline(always)] pub fn payload_is(&self) -> bool { self.0.typeid == TypeId::of::() } + #[inline(always)] pub fn payload(&self) -> Option<&T> { if self.payload_is::() { // we cast to a PyInner first because we don't know T's exact offset because of @@ -611,11 +623,12 @@ impl PyObject { } } + #[inline(always)] pub(crate) fn class_lock(&self) -> &PyRwLock { &self.0.typ } - #[inline] + #[inline(always)] pub fn payload_if_exact( &self, vm: &VirtualMachine, @@ -627,14 +640,16 @@ impl PyObject { } } - #[inline] + #[inline(always)] fn instance_dict(&self) -> Option<&InstanceDict> { self.0.dict.as_ref() } + #[inline(always)] pub fn dict(&self) -> Option { self.instance_dict().map(|d| d.get()) } + /// Set the dict field. Returns `Err(dict)` if this object does not have a dict field /// in the first place. pub fn set_dict(&self, dict: PyDictRef) -> Result<(), PyDictRef> { @@ -647,7 +662,7 @@ impl PyObject { } } - #[inline] + #[inline(always)] pub fn payload_if_subclass(&self, vm: &VirtualMachine) -> Option<&T> { if self.class().issubclass(T::class(vm)) { self.payload() @@ -656,6 +671,7 @@ impl PyObject { } } + #[inline(always)] pub fn downcast_ref(&self) -> Option<&PyObjectView> { if self.payload_is::() { // SAFETY: just checked that the payload is T, and PyRef is repr(transparent) over @@ -666,6 +682,7 @@ impl PyObject { } } + #[inline(always)] pub fn downcast_ref_if_exact( &self, vm: &VirtualMachine, @@ -677,13 +694,13 @@ impl PyObject { /// # Safety /// T must be the exact payload type - #[inline] + #[inline(always)] pub unsafe fn downcast_unchecked_ref(&self) -> &PyObjectView { debug_assert!(self.payload_is::()); &*(self as *const PyObject as *const PyObjectView) } - #[inline] + #[inline(always)] pub fn strong_count(&self) -> usize { self.0.ref_count.get() } @@ -693,12 +710,12 @@ impl PyObject { self.weak_ref_list().map(|wrl| wrl.count()) } - #[inline] + #[inline(always)] pub fn as_raw(&self) -> *const PyObject { self } - #[inline] + #[inline(always)] // the outer function is never inlined fn drop_slow_inner(&self) -> Result<(), ()> { // CPython-compatible drop implementation if let Some(slot_del) = self.class().mro_find_map(|cls| cls.slots.del.load()) { @@ -741,50 +758,56 @@ impl PyObject { } impl Borrow for PyObjectRef { + #[inline(always)] fn borrow(&self) -> &PyObject { self } } impl AsRef for PyObjectRef { + #[inline(always)] fn as_ref(&self) -> &PyObject { self } } impl AsRef for PyObject { + #[inline(always)] fn as_ref(&self) -> &PyObject { self } } impl<'a, T: PyObjectPayload> From<&'a PyObjectView> for &'a PyObject { + #[inline(always)] fn from(py_ref: &'a PyObjectView) -> Self { py_ref.as_object() } } impl Borrow for PyObjectWeak { - #[inline] + #[inline(always)] fn borrow(&self) -> &PyObject { self.weak.as_object() } } impl PyObjectWrap for PyObjectWeak { + #[inline(always)] fn into_object(self) -> PyObjectRef { self.weak.into_object() } } impl PyObjectWeak { - #[inline] + #[inline(always)] pub fn upgrade(&self) -> Option { self.weak.upgrade() } } impl Drop for PyObjectRef { + #[inline] fn drop(&mut self) { if self.0.ref_count.dec() { unsafe { PyObject::drop_slow(self.ptr) } @@ -930,6 +953,7 @@ impl Clone for PyRef { } impl PyRef { + #[inline(always)] unsafe fn from_raw(raw: *const PyObjectView) -> Self { Self { ptr: NonNull::new_unchecked(raw as *mut _), @@ -937,7 +961,7 @@ impl PyRef { } /// Safety: payload type of `obj` must be `T` - #[inline] + #[inline(always)] unsafe fn from_obj_unchecked(obj: PyObjectRef) -> Self { debug_assert!(obj.payload_is::()); let obj = ManuallyDrop::new(obj); @@ -946,7 +970,7 @@ impl PyRef { } } - #[inline] + #[inline(always)] pub fn new_ref(payload: T, typ: crate::builtins::PyTypeRef, dict: Option) -> Self { let inner = Box::into_raw(PyInner::new(payload, typ, dict)); Self { @@ -959,6 +983,7 @@ impl Borrow for PyRef where T: PyObjectPayload, { + #[inline(always)] fn borrow(&self) -> &PyObject { (**self).as_object() } @@ -989,6 +1014,7 @@ impl Borrow> for PyRef where T: PyObjectPayload, { + #[inline(always)] fn borrow(&self) -> &PyObjectView { self } diff --git a/vm/src/utils.rs b/vm/src/utils.rs index dc990afba..5d1ec49ce 100644 --- a/vm/src/utils.rs +++ b/vm/src/utils.rs @@ -12,6 +12,7 @@ pub enum Either { } impl, B: Borrow> Borrow for Either { + #[inline(always)] fn borrow(&self) -> &PyObject { match self { Either::A(a) => a.borrow(), @@ -21,6 +22,7 @@ impl, B: Borrow> Borrow for Either } impl, B: AsRef> AsRef for Either { + #[inline(always)] fn as_ref(&self) -> &PyObject { match self { Either::A(a) => a.as_ref(), @@ -30,6 +32,7 @@ impl, B: AsRef> AsRef for Either { } impl PyObjectWrap for Either { + #[inline(always)] fn into_object(self) -> PyObjectRef { match self { Either::A(a) => a.into_object(), @@ -39,6 +42,7 @@ impl PyObjectWrap for Either { } impl IntoPyObject for Either { + #[inline(always)] fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef { match self { Self::A(a) => a.into_pyobject(vm),