From 888b3822b7c915cd8a1934baf2eefbae194690b0 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 12 Aug 2021 03:18:28 +0900 Subject: [PATCH] Give a sole module for buffer protocol --- vm/src/buffer.rs | 167 +++++++++++++++++++++++++++++++++++ vm/src/builtins/bytearray.rs | 2 +- vm/src/builtins/bytes.rs | 14 ++- vm/src/builtins/memory.rs | 165 ++-------------------------------- vm/src/byteslike.rs | 2 +- vm/src/cformat.rs | 2 +- vm/src/lib.rs | 1 + vm/src/slots.rs | 5 +- vm/src/stdlib/array.rs | 2 +- vm/src/stdlib/io.rs | 5 +- vm/src/stdlib/sre.rs | 11 ++- 11 files changed, 193 insertions(+), 183 deletions(-) create mode 100644 vm/src/buffer.rs diff --git a/vm/src/buffer.rs b/vm/src/buffer.rs new file mode 100644 index 000000000..fbe791377 --- /dev/null +++ b/vm/src/buffer.rs @@ -0,0 +1,167 @@ +//! Buffer protocol + +use crate::common::borrow::{BorrowedValue, BorrowedValueMut}; +use crate::common::rc::PyRc; +use crate::PyThreadingConstraint; +use crate::{PyObjectRef, PyResult, TypeProtocol}; +use crate::{TryFromBorrowedObject, VirtualMachine}; +use std::{borrow::Cow, fmt::Debug, ops::Deref}; + +pub trait PyBuffer: Debug + PyThreadingConstraint { + fn get_options(&self) -> &BufferOptions; + /// Get the full inner buffer of this memory. You probably want [`as_contiguous()`], as + /// `obj_bytes` doesn't take into account the range a memoryview might operate on, among other + /// footguns. + fn obj_bytes(&self) -> BorrowedValue<[u8]>; + /// Get the full inner buffer of this memory, mutably. You probably want + /// [`as_contiguous_mut()`], as `obj_bytes` doesn't take into account the range a memoryview + /// might operate on, among other footguns. + fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]>; + fn release(&self); + + fn as_contiguous(&self) -> Option> { + if !self.get_options().contiguous { + return None; + } + Some(self.obj_bytes()) + } + + fn as_contiguous_mut(&self) -> Option> { + if !self.get_options().contiguous { + return None; + } + Some(self.obj_bytes_mut()) + } + + fn to_contiguous(&self) -> Vec { + self.obj_bytes().to_vec() + } +} + +#[derive(Debug, Clone)] +pub struct BufferOptions { + pub readonly: bool, + pub len: usize, + pub itemsize: usize, + pub contiguous: bool, + pub format: Cow<'static, str>, + // TODO: support multiple dimension array + pub ndim: usize, + pub shape: Vec, + pub strides: Vec, +} + +impl BufferOptions { + pub const DEFAULT: Self = BufferOptions { + readonly: true, + len: 0, + itemsize: 1, + contiguous: true, + format: Cow::Borrowed("B"), + ndim: 1, + shape: Vec::new(), + strides: Vec::new(), + }; +} + +impl Default for BufferOptions { + fn default() -> Self { + Self::DEFAULT + } +} + +#[derive(Debug)] +pub struct PyBufferRef(Box); + +impl TryFromBorrowedObject for PyBufferRef { + fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { + let obj_cls = obj.class(); + for cls in obj_cls.iter_mro() { + if let Some(f) = cls.slots.as_buffer.as_ref() { + return f(obj, vm).map(|x| PyBufferRef(x)); + } + } + Err(vm.new_type_error(format!( + "a bytes-like object is required, not '{}'", + obj_cls.name + ))) + } +} + +impl Drop for PyBufferRef { + fn drop(&mut self) { + self.0.release(); + } +} + +impl Deref for PyBufferRef { + type Target = dyn PyBuffer; + + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl PyBufferRef { + pub fn new(buffer: impl PyBuffer + 'static) -> Self { + Self(Box::new(buffer)) + } + + pub fn into_rcbuf(self) -> RcBuffer { + // move self.0 out of self; PyBufferRef impls Drop so it's tricky + let this = std::mem::ManuallyDrop::new(self); + let buf_box = unsafe { std::ptr::read(&this.0) }; + RcBuffer(buf_box.into()) + } +} + +impl From> for PyBufferRef { + fn from(buffer: Box) -> Self { + PyBufferRef(buffer) + } +} + +#[derive(Debug, Clone)] +pub struct RcBuffer(PyRc); +impl Deref for RcBuffer { + type Target = dyn PyBuffer; + fn deref(&self) -> &Self::Target { + self.0.deref() + } +} + +impl Drop for RcBuffer { + fn drop(&mut self) { + // check if this is the last rc before the inner buffer gets dropped + if let Some(buf) = PyRc::get_mut(&mut self.0) { + buf.release() + } + } +} + +impl PyBuffer for RcBuffer { + fn get_options(&self) -> &BufferOptions { + self.0.get_options() + } + fn obj_bytes(&self) -> BorrowedValue<[u8]> { + self.0.obj_bytes() + } + fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { + self.0.obj_bytes_mut() + } + fn release(&self) {} + fn as_contiguous(&self) -> Option> { + self.0.as_contiguous() + } + fn as_contiguous_mut(&self) -> Option> { + self.0.as_contiguous_mut() + } + fn to_contiguous(&self) -> Vec { + self.0.to_contiguous() + } +} + +pub(crate) trait ResizeGuard<'a> { + type Resizable: 'a; + fn try_resizable(&'a self, vm: &VirtualMachine) -> PyResult; +} diff --git a/vm/src/builtins/bytearray.rs b/vm/src/builtins/bytearray.rs index f6c821085..b0316e20c 100644 --- a/vm/src/builtins/bytearray.rs +++ b/vm/src/builtins/bytearray.rs @@ -2,11 +2,11 @@ use super::bytes::{PyBytes, PyBytesRef}; use super::dict::PyDictRef; use super::int::PyIntRef; -use super::memory::{BufferOptions, PyBuffer, ResizeGuard}; use super::pystr::PyStrRef; use super::pytype::PyTypeRef; use super::tuple::PyTupleRef; use crate::anystr::{self, AnyStr}; +use crate::buffer::{BufferOptions, PyBuffer, ResizeGuard}; use crate::bytesinner::{ bytes_decode, bytes_from_object, value_from_object, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs, diff --git a/vm/src/builtins/bytes.rs b/vm/src/builtins/bytes.rs index 1bd61fed3..73384e4ac 100644 --- a/vm/src/builtins/bytes.rs +++ b/vm/src/builtins/bytes.rs @@ -1,14 +1,9 @@ -use bstr::ByteSlice; -use crossbeam_utils::atomic::AtomicCell; -use rustpython_common::borrow::{BorrowedValue, BorrowedValueMut}; -use std::mem::size_of; -use std::ops::Deref; - use super::dict::PyDictRef; use super::int::PyIntRef; use super::pystr::PyStrRef; use super::pytype::PyTypeRef; use crate::anystr::{self, AnyStr}; +use crate::buffer::{BufferOptions, PyBuffer}; use crate::builtins::tuple::PyTupleRef; use crate::bytesinner::{ bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions, @@ -24,8 +19,11 @@ use crate::{ IdProtocol, IntoPyObject, PyClassImpl, PyComparisonValue, PyContext, PyIterable, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, }; - -use crate::builtins::memory::{BufferOptions, PyBuffer}; +use bstr::ByteSlice; +use crossbeam_utils::atomic::AtomicCell; +use rustpython_common::borrow::{BorrowedValue, BorrowedValueMut}; +use std::mem::size_of; +use std::ops::Deref; /// "bytes(iterable_of_ints) -> bytes\n\ /// bytes(string, encoding[, errors]) -> bytes\n\ diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index 6da565255..0973a7e28 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -1,5 +1,4 @@ -use std::{borrow::Cow, fmt::Debug, ops::Deref}; - +use crate::buffer::{BufferOptions, PyBuffer, PyBufferRef}; use crate::builtins::bytes::{PyBytes, PyBytesRef}; use crate::builtins::list::{PyList, PyListRef}; use crate::builtins::pystr::{PyStr, PyStrRef}; @@ -9,7 +8,6 @@ use crate::bytesinner::bytes_to_hex; use crate::common::borrow::{BorrowedValue, BorrowedValueMut}; use crate::common::hash::PyHash; use crate::common::lock::OnceCell; -use crate::common::rc::PyRc; use crate::function::{FuncArgs, OptionalArg}; use crate::sliceable::{convert_slice, saturate_range, wrap_index, SequenceIndex}; use crate::slots::{BufferProtocol, Comparable, Hashable, PyComparisonOp}; @@ -17,165 +15,14 @@ use crate::stdlib::pystruct::_struct::FormatSpec; use crate::utils::Either; use crate::{ IdProtocol, IntoPyObject, PyClassImpl, PyComparisonValue, PyContext, PyObjectRef, PyRef, - PyResult, PyThreadingConstraint, PyValue, TypeProtocol, + PyResult, PyValue, TypeProtocol, }; use crate::{TryFromBorrowedObject, TryFromObject, VirtualMachine}; use crossbeam_utils::atomic::AtomicCell; use itertools::Itertools; use num_bigint::BigInt; use num_traits::{One, Signed, ToPrimitive, Zero}; - -#[derive(Debug)] -pub struct PyBufferRef(Box); - -impl TryFromBorrowedObject for PyBufferRef { - fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult { - let obj_cls = obj.class(); - for cls in obj_cls.iter_mro() { - if let Some(f) = cls.slots.as_buffer.as_ref() { - return f(&obj, vm).map(|x| PyBufferRef(x)); - } - } - Err(vm.new_type_error(format!( - "a bytes-like object is required, not '{}'", - obj_cls.name - ))) - } -} - -impl Drop for PyBufferRef { - fn drop(&mut self) { - self.0.release(); - } -} -impl Deref for PyBufferRef { - type Target = dyn PyBuffer; - - fn deref(&self) -> &Self::Target { - self.0.deref() - } -} -impl PyBufferRef { - pub fn new(buffer: impl PyBuffer + 'static) -> Self { - Self(Box::new(buffer)) - } - pub fn into_rcbuf(self) -> RcBuffer { - // move self.0 out of self; PyBufferRef impls Drop so it's tricky - let this = std::mem::ManuallyDrop::new(self); - let buf_box = unsafe { std::ptr::read(&this.0) }; - RcBuffer(buf_box.into()) - } -} -impl From> for PyBufferRef { - fn from(buffer: Box) -> Self { - PyBufferRef(buffer) - } -} -#[derive(Debug, Clone)] -pub struct RcBuffer(PyRc); -impl Deref for RcBuffer { - type Target = dyn PyBuffer; - fn deref(&self) -> &Self::Target { - self.0.deref() - } -} -impl Drop for RcBuffer { - fn drop(&mut self) { - // check if this is the last rc before the inner buffer gets dropped - if let Some(buf) = PyRc::get_mut(&mut self.0) { - buf.release() - } - } -} -impl PyBuffer for RcBuffer { - fn get_options(&self) -> &BufferOptions { - self.0.get_options() - } - fn obj_bytes(&self) -> BorrowedValue<[u8]> { - self.0.obj_bytes() - } - fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> { - self.0.obj_bytes_mut() - } - fn release(&self) {} - fn as_contiguous(&self) -> Option> { - self.0.as_contiguous() - } - fn as_contiguous_mut(&self) -> Option> { - self.0.as_contiguous_mut() - } - fn to_contiguous(&self) -> Vec { - self.0.to_contiguous() - } -} - -pub trait PyBuffer: Debug + PyThreadingConstraint { - fn get_options(&self) -> &BufferOptions; - /// Get the full inner buffer of this memory. You probably want [`as_contiguous()`], as - /// `obj_bytes` doesn't take into account the range a memoryview might operate on, among other - /// footguns. - fn obj_bytes(&self) -> BorrowedValue<[u8]>; - /// Get the full inner buffer of this memory, mutably. You probably want - /// [`as_contiguous_mut()`], as `obj_bytes` doesn't take into account the range a memoryview - /// might operate on, among other footguns. - fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]>; - fn release(&self); - - fn as_contiguous(&self) -> Option> { - if !self.get_options().contiguous { - return None; - } - Some(self.obj_bytes()) - } - - fn as_contiguous_mut(&self) -> Option> { - if !self.get_options().contiguous { - return None; - } - Some(self.obj_bytes_mut()) - } - - fn to_contiguous(&self) -> Vec { - self.obj_bytes().to_vec() - } -} - -#[derive(Debug, Clone)] -pub struct BufferOptions { - pub readonly: bool, - pub len: usize, - pub itemsize: usize, - pub contiguous: bool, - pub format: Cow<'static, str>, - // TODO: support multiple dimension array - pub ndim: usize, - pub shape: Vec, - pub strides: Vec, -} - -impl BufferOptions { - pub const DEFAULT: Self = BufferOptions { - readonly: true, - len: 0, - itemsize: 1, - contiguous: true, - format: Cow::Borrowed("B"), - ndim: 1, - shape: Vec::new(), - strides: Vec::new(), - }; -} - -impl Default for BufferOptions { - fn default() -> Self { - Self::DEFAULT - } -} - -pub(crate) trait ResizeGuard<'a> { - type Resizable: 'a; - fn try_resizable(&'a self, vm: &VirtualMachine) -> PyResult; -} +use std::fmt::Debug; #[derive(FromArgs)] struct PyMemoryViewNewArgs { @@ -393,7 +240,7 @@ impl PyMemoryView { let itemsize = zelf.options.itemsize; let obj = zelf.obj.clone(); - let buffer = PyBufferRef(Box::new(zelf.clone())); + let buffer = PyBufferRef::new(zelf.clone()); zelf.exports.fetch_add(1); let options = zelf.options.clone(); let format_spec = zelf.format_spec.clone(); @@ -680,7 +527,7 @@ impl PyMemoryView { #[pymethod] fn toreadonly(zelf: PyRef, vm: &VirtualMachine) -> PyResult> { zelf.try_not_released(vm)?; - let buffer = PyBufferRef(Box::new(zelf.clone())); + let buffer = PyBufferRef::new(zelf.clone()); zelf.exports.fetch_add(1); Ok(PyMemoryView { obj: zelf.obj.clone(), @@ -751,7 +598,7 @@ impl PyMemoryView { ); } - let buffer = PyBufferRef(Box::new(zelf.clone())); + let buffer = PyBufferRef::new(zelf.clone()); zelf.exports.fetch_add(1); Ok(PyMemoryView { diff --git a/vm/src/byteslike.rs b/vm/src/byteslike.rs index 751f58e2f..adf7d1ca5 100644 --- a/vm/src/byteslike.rs +++ b/vm/src/byteslike.rs @@ -1,4 +1,4 @@ -use crate::builtins::memory::PyBufferRef; +use crate::buffer::PyBufferRef; use crate::builtins::PyStrRef; use crate::common::borrow::{BorrowedValue, BorrowedValueMut}; use crate::vm::VirtualMachine; diff --git a/vm/src/cformat.rs b/vm/src/cformat.rs index 0b4d0ae3b..12d9cdc59 100644 --- a/vm/src/cformat.rs +++ b/vm/src/cformat.rs @@ -1,8 +1,8 @@ +use crate::buffer::PyBufferRef; /// Implementation of Printf-Style string formatting /// [https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting] use crate::builtins::float::{try_bigint, IntoPyFloat, PyFloat}; use crate::builtins::int::{self, PyInt}; -use crate::builtins::memory::PyBufferRef; use crate::builtins::pystr::PyStr; use crate::builtins::{tuple, PyBytes}; use crate::common::float_ops; diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 21dc4d90d..8807d7c2f 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -43,6 +43,7 @@ pub use rustpython_derive::*; pub mod macros; mod anystr; +mod buffer; pub mod builtins; mod bytesinner; pub mod byteslike; diff --git a/vm/src/slots.rs b/vm/src/slots.rs index b8b67a0d7..d56aa1eed 100644 --- a/vm/src/slots.rs +++ b/vm/src/slots.rs @@ -1,6 +1,4 @@ -use std::cmp::Ordering; - -use crate::builtins::memory::PyBuffer; +use crate::buffer::PyBuffer; use crate::builtins::pystr::PyStrRef; use crate::common::hash::PyHash; use crate::common::lock::PyRwLock; @@ -12,6 +10,7 @@ use crate::{ TypeProtocol, }; use crossbeam_utils::atomic::AtomicCell; +use std::cmp::Ordering; bitflags! { pub struct PyTpFlags: u64 { diff --git a/vm/src/stdlib/array.rs b/vm/src/stdlib/array.rs index 709bb5b1e..fc4fc6822 100644 --- a/vm/src/stdlib/array.rs +++ b/vm/src/stdlib/array.rs @@ -1,6 +1,6 @@ +use crate::buffer::{BufferOptions, PyBuffer, ResizeGuard}; use crate::builtins::float::IntoPyFloat; use crate::builtins::list::{PyList, PyListRef}; -use crate::builtins::memory::{BufferOptions, PyBuffer, ResizeGuard}; use crate::builtins::pystr::PyStrRef; use crate::builtins::pytype::PyTypeRef; use crate::builtins::slice::PySliceRef; diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 280fc2c4d..3c63ce7e2 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -77,9 +77,8 @@ mod _io { use std::io::{self, prelude::*, Cursor, SeekFrom}; use std::ops::Range; - use crate::builtins::memory::{ - BufferOptions, PyBuffer, PyBufferRef, PyMemoryView, ResizeGuard, - }; + use crate::buffer::{BufferOptions, PyBuffer, PyBufferRef, ResizeGuard}; + use crate::builtins::memory::PyMemoryView; use crate::builtins::{ bytes::{PyBytes, PyBytesRef}, pybool, pytype, PyByteArray, PyStr, PyStrRef, PyTypeRef, diff --git a/vm/src/stdlib/sre.rs b/vm/src/stdlib/sre.rs index df62d90e7..a62042409 100644 --- a/vm/src/stdlib/sre.rs +++ b/vm/src/stdlib/sre.rs @@ -2,13 +2,8 @@ pub(crate) use _sre::make_module; #[pymodule] mod _sre { - use crossbeam_utils::atomic::AtomicCell; - use itertools::Itertools; - use num_traits::ToPrimitive; - use rustpython_common::hash::PyHash; - + use crate::buffer::PyBufferRef; use crate::builtins::list::PyListRef; - use crate::builtins::memory::PyBufferRef; use crate::builtins::tuple::PyTupleRef; use crate::builtins::{ PyCallableIterator, PyDictRef, PyInt, PyList, PyStr, PyStrRef, PyTypeRef, @@ -21,6 +16,10 @@ mod _sre { PyValue, StaticType, TryFromBorrowedObject, TryFromObject, }; use core::str; + use crossbeam_utils::atomic::AtomicCell; + use itertools::Itertools; + use num_traits::ToPrimitive; + use rustpython_common::hash::PyHash; use sre_engine::constants::SreFlag; use sre_engine::engine::{lower_ascii, lower_unicode, upper_unicode, State, StrDrive};