inline for object pointer operations

vm/src/function/argument.rs

@@ -14,19 +14,20 @@ pub struct ArgCallable {
 }
 
 impl ArgCallable {
-    #[inline]
     pub fn invoke(&self, args: impl IntoFuncArgs, vm: &VirtualMachine) -> PyResult {
         vm.invoke(&self.obj, args)
     }
 }
 
 impl Borrow<PyObject> for ArgCallable {
+    #[inline(always)]
     fn borrow(&self) -> &PyObject {
         &self.obj
     }
 }
 
 impl AsRef<PyObject> for ArgCallable {
+    #[inline(always)]
     fn as_ref(&self) -> &PyObject {
         &self.obj
     }
@@ -116,24 +117,28 @@ impl ArgMapping {
 }
 
 impl Borrow<PyObject> for ArgMapping {
+    #[inline(always)]
     fn borrow(&self) -> &PyObject {
         &self.obj
     }
 }
 
 impl AsRef<PyObject> for ArgMapping {
+    #[inline(always)]
     fn as_ref(&self) -> &PyObject {
         &self.obj
     }
 }
 
 impl PyObjectWrap for ArgMapping {
+    #[inline(always)]
     fn into_object(self) -> PyObjectRef {
         self.obj
     }
 }
 
 impl IntoPyObject for ArgMapping {
+    #[inline(always)]
     fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef {
         self.obj
     }

vm/src/protocol/iter.rs

@@ -80,6 +80,7 @@ impl<O> Borrow<PyObject> for PyIter<O>
 where
     O: Borrow<PyObject>,
 {
+    #[inline(always)]
     fn borrow(&self) -> &PyObject {
         self.0.borrow()
     }
@@ -89,6 +90,7 @@ impl<O> AsRef<PyObject> for PyIter<O>
 where
     O: Borrow<PyObject>,
 {
+    #[inline(always)]
     fn as_ref(&self) -> &PyObject {
         self.0.borrow()
     }
@@ -99,12 +101,14 @@ where
     O: Borrow<PyObject>,
 {
     type Target = PyObject;
+    #[inline(always)]
     fn deref(&self) -> &Self::Target {
         self.0.borrow()
     }
 }
 
 impl IntoPyObject for PyIter<PyObjectRef> {
+    #[inline(always)]
     fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef {
         self.into()
     }

vm/src/pyobject.rs

@@ -134,14 +134,17 @@ impl PyContext {
         context
     }
 
+    #[inline(always)]
     pub fn none(&self) -> PyObjectRef {
         self.none.clone().into()
     }
 
+    #[inline(always)]
     pub fn ellipsis(&self) -> PyObjectRef {
         self.ellipsis.clone().into()
     }
 
+    #[inline(always)]
     pub fn not_implemented(&self) -> PyObjectRef {
         self.not_implemented.clone().into()
     }
@@ -170,19 +173,22 @@ impl PyContext {
         PyRef::new_ref(PyInt::from(i.clone()), self.types.int_type.clone(), None)
     }
 
+    #[inline]
     pub fn new_float(&self, value: f64) -> PyRef<PyFloat> {
         PyRef::new_ref(PyFloat::from(value), self.types.float_type.clone(), None)
     }
 
+    #[inline]
     pub fn new_str(&self, s: impl Into<pystr::PyStr>) -> PyRef<PyStr> {
         pystr::PyStr::new_ref(s, self)
     }
 
+    #[inline]
     pub fn new_bytes(&self, data: Vec<u8>) -> PyRef<bytes::PyBytes> {
         bytes::PyBytes::new_ref(data, self)
     }
 
-    #[inline]
+    #[inline(always)]
     pub fn new_bool(&self, b: bool) -> PyIntRef {
         let value = if b {
             &self.true_value
@@ -192,14 +198,17 @@ impl PyContext {
         value.clone()
     }
 
+    #[inline(always)]
     pub fn new_tuple(&self, elements: Vec<PyObjectRef>) -> PyTupleRef {
         PyTuple::new_ref(elements, self)
     }
 
+    #[inline(always)]
     pub fn new_list(&self, elements: Vec<PyObjectRef>) -> PyListRef {
         PyList::new_ref(elements, self)
     }
 
+    #[inline(always)]
     pub fn new_dict(&self) -> PyDictRef {
         PyDict::new_ref(self)
     }
@@ -376,12 +385,13 @@ impl<T: PyValue> TryFromObject for PyRefExact<T> {
 }
 impl<T: PyValue> Deref for PyRefExact<T> {
     type Target = PyRef<T>;
+    #[inline(always)]
     fn deref(&self) -> &PyRef<T> {
         &self.obj
     }
 }
 impl<T: PyValue> IntoPyObject for PyRefExact<T> {
-    #[inline]
+    #[inline(always)]
     fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef {
         self.obj.into()
     }
@@ -418,6 +428,7 @@ impl PyObject {
 }
 
 // impl<T: ?Sized> Borrow<PyObject> for PyRc<T> {
+//     #[inline(always)]
 //     fn borrow(&self) -> &PyObject {
 //         unsafe { &*(&**self as *const T as *const PyObject) }
 //     }
@@ -432,12 +443,14 @@ pub struct PyLease<'a, T: PyObjectPayload> {
 impl<'a, T: PyObjectPayload + PyValue> PyLease<'a, T> {
     // Associated function on purpose, because of deref
     #[allow(clippy::wrong_self_convention)]
+    #[inline(always)]
     pub fn into_pyref(zelf: Self) -> PyRef<T> {
         zelf.inner.clone()
     }
 }
 
 impl<'a, T: PyObjectPayload + PyValue> Borrow<PyObject> for PyLease<'a, T> {
+    #[inline(always)]
     fn borrow(&self) -> &PyObject {
         self.inner.as_ref()
     }
@@ -445,7 +458,7 @@ impl<'a, T: PyObjectPayload + PyValue> Borrow<PyObject> for PyLease<'a, T> {
 
 impl<'a, T: PyObjectPayload + PyValue> Deref for PyLease<'a, T> {
     type Target = PyRef<T>;
-
+    #[inline(always)]
     fn deref(&self) -> &Self::Target {
         &self.inner
     }
@@ -521,27 +534,28 @@ impl<T, P> IntoPyRef<P> for T
 where
     P: PyValue + IntoPyObject + From<T>,
 {
+    #[inline(always)]
     fn into_pyref(self, vm: &VirtualMachine) -> PyRef<P> {
         P::from(self).into_ref(vm)
     }
 }
 
 impl<T: PyObjectPayload> IntoPyObject for PyRef<T> {
-    #[inline]
+    #[inline(always)]
     fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef {
         self.into()
     }
 }
 
 impl IntoPyObject for PyObjectRef {
-    #[inline]
+    #[inline(always)]
     fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef {
         self
     }
 }
 
 impl IntoPyObject for &PyObject {
-    #[inline]
+    #[inline(always)]
     fn into_pyobject(self, _vm: &VirtualMachine) -> PyObjectRef {
         self.to_owned()
     }
@@ -553,7 +567,7 @@ impl<T> IntoPyObject for T
 where
     T: PyValue + Sized,
 {
-    #[inline]
+    #[inline(always)]
     fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef {
         PyValue::into_object(self, vm)
     }
@@ -563,6 +577,7 @@ impl<T> IntoPyResult for T
 where
     T: IntoPyObject,
 {
+    #[inline(always)]
     fn into_pyresult(self, vm: &VirtualMachine) -> PyResult {
         Ok(self.into_pyobject(vm))
     }
@@ -572,6 +587,7 @@ impl<T> IntoPyResult for PyResult<T>
 where
     T: IntoPyObject,
 {
+    #[inline(always)]
     fn into_pyresult(self, vm: &VirtualMachine) -> PyResult {
         self.map(|res| T::into_pyobject(res, vm))
     }
@@ -600,6 +616,7 @@ pub trait PyValue: fmt::Debug + PyThreadingConstraint + Sized + 'static {
         None
     }
 
+    #[inline]
     fn _into_ref(self, cls: PyTypeRef, vm: &VirtualMachine) -> PyRef<Self> {
         let dict = if cls.slots.flags.has_feature(PyTypeFlags::HAS_DICT) {
             Some(vm.ctx.new_dict())
@@ -609,24 +626,36 @@ pub trait PyValue: fmt::Debug + PyThreadingConstraint + Sized + 'static {
         PyRef::new_ref(self, cls, dict)
     }
 
+    #[inline]
     fn into_ref(self, vm: &VirtualMachine) -> PyRef<Self> {
         let cls = Self::class(vm);
         self._into_ref(cls.clone(), vm)
     }
 
+    #[cold]
+    fn _into_ref_with_type_error(
+        vm: &VirtualMachine,
+        cls: &PyTypeRef,
+        exact_class: &PyTypeRef,
+    ) -> PyBaseExceptionRef {
+        vm.new_type_error(format!(
+            "'{}' is not a subtype of '{}'",
+            &cls.name(),
+            exact_class.name()
+        ))
+    }
+
+    #[inline]
     fn into_ref_with_type(self, vm: &VirtualMachine, cls: PyTypeRef) -> PyResult<PyRef<Self>> {
         let exact_class = Self::class(vm);
         if cls.issubclass(exact_class) {
             Ok(self._into_ref(cls, vm))
         } else {
-            Err(vm.new_type_error(format!(
-                "'{}' is not a subtype of '{}'",
-                &cls.name(),
-                exact_class.name()
-            )))
+            Err(Self::_into_ref_with_type_error(vm, &cls, exact_class))
         }
     }
 
+    #[inline]
     fn into_pyresult_with_type(self, vm: &VirtualMachine, cls: PyTypeRef) -> PyResult {
         self.into_ref_with_type(vm, cls).into_pyresult(vm)
     }
@@ -710,6 +739,7 @@ impl<T> From<T> for PyObjectRef
 where
     T: PyObjectWrap,
 {
+    #[inline(always)]
     fn from(py_ref: T) -> Self {
         py_ref.into_object()
     }

vm/src/pyobjectrc.rs

@@ -283,14 +283,17 @@ unsafe impl Link for WeakLink {
 
     type Target = PyObjectView<PyWeak>;
 
+    #[inline(always)]
     fn as_raw(handle: &PyRef<PyWeak>) -> NonNull<Self::Target> {
         NonNull::from(&**handle)
     }
 
+    #[inline(always)]
     unsafe fn from_raw(ptr: NonNull<Self::Target>) -> Self::Handle {
         PyRef::from_raw(ptr.as_ptr())
     }
 
+    #[inline(always)]
     unsafe fn pointers(target: NonNull<Self::Target>) -> NonNull<Pointers<Self::Target>> {
         NonNull::new_unchecked(ptr::addr_of_mut!((*target.as_ptr()).0.payload.pointers))
     }
@@ -331,7 +334,6 @@ impl PyWeak {
         guard.obj.is_none()
     }
 
-    #[inline(always)]
     fn drop_inner(&self) {
         let dealloc = {
             let mut guard = unsafe { self.parent.as_ref().lock() };
@@ -354,6 +356,7 @@ impl PyWeak {
 }
 
 impl Drop for PyWeak {
+    #[inline(always)]
     fn drop(&mut self) {
         // we do NOT have actual exclusive access!
         // no clue if doing this actually reduces chance of UB
@@ -368,7 +371,7 @@ struct InstanceDict {
 }
 
 impl From<PyDictRef> for InstanceDict {
-    #[inline]
+    #[inline(always)]
     fn from(d: PyDictRef) -> Self {
         Self::new(d)
     }
@@ -447,6 +450,7 @@ pub struct PyObject(PyInner<Erased>);
 
 impl Deref for PyObjectRef {
     type Target = PyObject;
+    #[inline(always)]
     fn deref(&self) -> &PyObject {
         unsafe { self.ptr.as_ref() }
     }
@@ -465,6 +469,7 @@ impl ToOwned for PyObject {
 }
 
 impl PyObjectRef {
+    #[inline(always)]
     pub fn into_raw(self) -> *const PyObject {
         let ptr = self.as_raw();
         std::mem::forget(self);
@@ -476,6 +481,7 @@ impl PyObjectRef {
     /// [`PyObjectRef::into_raw`]. The user is responsible for ensuring that the inner data is not
     /// dropped more than once due to mishandling the reference count by calling this function
     /// too many times.
+    #[inline(always)]
     pub unsafe fn from_raw(ptr: *const PyObject) -> Self {
         Self {
             ptr: NonNull::new_unchecked(ptr as *mut PyObject),
@@ -486,6 +492,7 @@ impl PyObjectRef {
     ///
     /// If the downcast fails, the original ref is returned in as `Err` so
     /// another downcast can be attempted without unnecessary cloning.
+    #[inline(always)]
     pub fn downcast<T: PyObjectPayload>(self) -> Result<PyRef<T>, Self> {
         if self.payload_is::<T>() {
             Ok(unsafe { PyRef::from_obj_unchecked(self) })
@@ -494,6 +501,7 @@ impl PyObjectRef {
         }
     }
 
+    #[inline(always)]
     pub fn downcast_ref<T: PyObjectPayload>(&self) -> Option<&PyObjectView<T>> {
         if self.payload_is::<T>() {
             // SAFETY: just checked that the payload is T, and PyRef is repr(transparent) over
@@ -506,12 +514,14 @@ impl PyObjectRef {
 
     /// # Safety
     /// T must be the exact payload type
+    #[inline(always)]
     pub unsafe fn downcast_unchecked<T: PyObjectPayload>(self) -> PyRef<T> {
         PyRef::from_obj_unchecked(self)
     }
 
     /// # Safety
     /// T must be the exact payload type
+    #[inline(always)]
     pub unsafe fn downcast_unchecked_ref<T: PyObjectPayload>(&self) -> &crate::PyObjectView<T> {
         debug_assert!(self.payload_is::<T>());
         &*(self as *const PyObjectRef as *const PyRef<T>)
@@ -523,6 +533,7 @@ impl PyObjectRef {
     ///
     /// If the downcast fails, the original ref is returned in as `Err` so
     /// another downcast can be attempted without unnecessary cloning.
+    #[inline]
     pub fn downcast_exact<T: PyObjectPayload + crate::PyValue>(
         self,
         vm: &VirtualMachine,
@@ -542,7 +553,7 @@ impl PyObjectRef {
 }
 
 impl PyObject {
-    #[inline]
+    #[inline(always)]
     fn weak_ref_list(&self) -> Option<&WeakRefList> {
         Some(&self.0.weak_list)
     }
@@ -595,11 +606,12 @@ impl PyObject {
         self.weak_ref_list().map(|wrl| wrl.get_weak_references())
     }
 
-    #[inline]
+    #[inline(always)]
     pub fn payload_is<T: PyObjectPayload>(&self) -> bool {
         self.0.typeid == TypeId::of::<T>()
     }
 
+    #[inline(always)]
     pub fn payload<T: PyObjectPayload>(&self) -> Option<&T> {
         if self.payload_is::<T>() {
             // we cast to a PyInner<T> first because we don't know T's exact offset because of
@@ -611,11 +623,12 @@ impl PyObject {
         }
     }
 
+    #[inline(always)]
     pub(crate) fn class_lock(&self) -> &PyRwLock<PyTypeRef> {
         &self.0.typ
     }
 
-    #[inline]
+    #[inline(always)]
     pub fn payload_if_exact<T: PyObjectPayload + crate::PyValue>(
         &self,
         vm: &VirtualMachine,
@@ -627,14 +640,16 @@ impl PyObject {
         }
     }
 
-    #[inline]
+    #[inline(always)]
     fn instance_dict(&self) -> Option<&InstanceDict> {
         self.0.dict.as_ref()
     }
 
+    #[inline(always)]
     pub fn dict(&self) -> Option<PyDictRef> {
         self.instance_dict().map(|d| d.get())
     }
+
     /// Set the dict field. Returns `Err(dict)` if this object does not have a dict field
     /// in the first place.
     pub fn set_dict(&self, dict: PyDictRef) -> Result<(), PyDictRef> {
@@ -647,7 +662,7 @@ impl PyObject {
         }
     }
 
-    #[inline]
+    #[inline(always)]
     pub fn payload_if_subclass<T: crate::PyValue>(&self, vm: &VirtualMachine) -> Option<&T> {
         if self.class().issubclass(T::class(vm)) {
             self.payload()
@@ -656,6 +671,7 @@ impl PyObject {
         }
     }
 
+    #[inline(always)]
     pub fn downcast_ref<T: PyObjectPayload>(&self) -> Option<&PyObjectView<T>> {
         if self.payload_is::<T>() {
             // SAFETY: just checked that the payload is T, and PyRef is repr(transparent) over
@@ -666,6 +682,7 @@ impl PyObject {
         }
     }
 
+    #[inline(always)]
     pub fn downcast_ref_if_exact<T: PyObjectPayload + crate::PyValue>(
         &self,
         vm: &VirtualMachine,
@@ -677,13 +694,13 @@ impl PyObject {
 
     /// # Safety
     /// T must be the exact payload type
-    #[inline]
+    #[inline(always)]
     pub unsafe fn downcast_unchecked_ref<T: PyObjectPayload>(&self) -> &PyObjectView<T> {
         debug_assert!(self.payload_is::<T>());
         &*(self as *const PyObject as *const PyObjectView<T>)
     }
 
-    #[inline]
+    #[inline(always)]
     pub fn strong_count(&self) -> usize {
         self.0.ref_count.get()
     }
@@ -693,12 +710,12 @@ impl PyObject {
         self.weak_ref_list().map(|wrl| wrl.count())
     }
 
-    #[inline]
+    #[inline(always)]
     pub fn as_raw(&self) -> *const PyObject {
         self
     }
 
-    #[inline]
+    #[inline(always)] // the outer function is never inlined
     fn drop_slow_inner(&self) -> Result<(), ()> {
         // CPython-compatible drop implementation
         if let Some(slot_del) = self.class().mro_find_map(|cls| cls.slots.del.load()) {
@@ -741,50 +758,56 @@ impl PyObject {
 }
 
 impl Borrow<PyObject> for PyObjectRef {
+    #[inline(always)]
     fn borrow(&self) -> &PyObject {
         self
     }
 }
 
 impl AsRef<PyObject> for PyObjectRef {
+    #[inline(always)]
     fn as_ref(&self) -> &PyObject {
         self
     }
 }
 
 impl AsRef<PyObject> for PyObject {
+    #[inline(always)]
     fn as_ref(&self) -> &PyObject {
         self
     }
 }
 
 impl<'a, T: PyObjectPayload> From<&'a PyObjectView<T>> for &'a PyObject {
+    #[inline(always)]
     fn from(py_ref: &'a PyObjectView<T>) -> Self {
         py_ref.as_object()
     }
 }
 
 impl Borrow<PyObject> for PyObjectWeak {
-    #[inline]
+    #[inline(always)]
     fn borrow(&self) -> &PyObject {
         self.weak.as_object()
     }
 }
 
 impl PyObjectWrap for PyObjectWeak {
+    #[inline(always)]
     fn into_object(self) -> PyObjectRef {
         self.weak.into_object()
     }
 }
 
 impl PyObjectWeak {
-    #[inline]
+    #[inline(always)]
     pub fn upgrade(&self) -> Option<PyObjectRef> {
         self.weak.upgrade()
     }
 }
 
 impl Drop for PyObjectRef {
+    #[inline]
     fn drop(&mut self) {
         if self.0.ref_count.dec() {
             unsafe { PyObject::drop_slow(self.ptr) }
@@ -930,6 +953,7 @@ impl<T: PyObjectPayload> Clone for PyRef<T> {
 }
 
 impl<T: PyObjectPayload> PyRef<T> {
+    #[inline(always)]
     unsafe fn from_raw(raw: *const PyObjectView<T>) -> Self {
         Self {
             ptr: NonNull::new_unchecked(raw as *mut _),
@@ -937,7 +961,7 @@ impl<T: PyObjectPayload> PyRef<T> {
     }
 
     /// Safety: payload type of `obj` must be `T`
-    #[inline]
+    #[inline(always)]
     unsafe fn from_obj_unchecked(obj: PyObjectRef) -> Self {
         debug_assert!(obj.payload_is::<T>());
         let obj = ManuallyDrop::new(obj);
@@ -946,7 +970,7 @@ impl<T: PyObjectPayload> PyRef<T> {
         }
     }
 
-    #[inline]
+    #[inline(always)]
     pub fn new_ref(payload: T, typ: crate::builtins::PyTypeRef, dict: Option<PyDictRef>) -> Self {
         let inner = Box::into_raw(PyInner::new(payload, typ, dict));
         Self {
@@ -959,6 +983,7 @@ impl<T> Borrow<PyObject> for PyRef<T>
 where
     T: PyObjectPayload,
 {
+    #[inline(always)]
     fn borrow(&self) -> &PyObject {
         (**self).as_object()
     }
@@ -989,6 +1014,7 @@ impl<T> Borrow<PyObjectView<T>> for PyRef<T>
 where
     T: PyObjectPayload,
 {
+    #[inline(always)]
     fn borrow(&self) -> &PyObjectView<T> {
         self
     }

vm/src/utils.rs

@@ -12,6 +12,7 @@ pub enum Either<A, B> {
 }
 
 impl<A: Borrow<PyObject>, B: Borrow<PyObject>> Borrow<PyObject> for Either<A, B> {
+    #[inline(always)]
     fn borrow(&self) -> &PyObject {
         match self {
             Either::A(a) => a.borrow(),
@@ -21,6 +22,7 @@ impl<A: Borrow<PyObject>, B: Borrow<PyObject>> Borrow<PyObject> for Either<A, B>
 }
 
 impl<A: AsRef<PyObject>, B: AsRef<PyObject>> AsRef<PyObject> for Either<A, B> {
+    #[inline(always)]
     fn as_ref(&self) -> &PyObject {
         match self {
             Either::A(a) => a.as_ref(),
@@ -30,6 +32,7 @@ impl<A: AsRef<PyObject>, B: AsRef<PyObject>> AsRef<PyObject> for Either<A, B> {
 }
 
 impl<A: PyObjectWrap, B: PyObjectWrap> PyObjectWrap for Either<A, B> {
+    #[inline(always)]
     fn into_object(self) -> PyObjectRef {
         match self {
             Either::A(a) => a.into_object(),
@@ -39,6 +42,7 @@ impl<A: PyObjectWrap, B: PyObjectWrap> PyObjectWrap for Either<A, B> {
 }
 
 impl<A: IntoPyObject, B: IntoPyObject> IntoPyObject for Either<A, B> {
+    #[inline(always)]
     fn into_pyobject(self, vm: &VirtualMachine) -> PyObjectRef {
         match self {
             Self::A(a) => a.into_pyobject(vm),