Rust-related/RustPython
1
0
Fork 1
mirror of https://github.com/RustPython/RustPython.git synced 2026-06-02 19:39:49 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3029 from youknowone/pybuffer

Browse Source 
Refactor PyBuffer
This commit is contained in:
Jeong YunWon
2021-09-15 15:11:19 +09:00
committed by GitHub
parent 76bb68a2d0 f2ecab46ef
commit 00a86ba65a
14 changed files with 337 additions and 419 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Lib/test/test_httplib.py vendored
View File

@@ -839,8 +839,6 @@ class BasicTest(TestCase):
with self.assertRaises(TypeError):
conn.request('POST', 'test', conn)
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_chunked(self):
expected = chunked_expected
sock = FakeSocket(chunked_start + last_chunk + chunked_end)
@@ -873,8 +871,6 @@ class BasicTest(TestCase):
finally:
resp.close()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_readinto_chunked(self):
expected = chunked_expected

12
Lib/test/test_io.py vendored
View File

@@ -1039,8 +1039,6 @@ class CIOTest(IOTest):
super().test_destructor(self)
class PyIOTest(IOTest):
# TODO: RUSTPYTHON, can't resize b/c of existing exports
@unittest.expectedFailure
def test_optional_abilities(self):
super().test_optional_abilities()
@@ -2630,8 +2628,7 @@ class TextIOWrapperTest(unittest.TestCase):
with self.assertRaisesRegex(LookupError, "is not a text encoding"):
self.TextIOWrapper(b, encoding="hex")
# TODO: RUSTPYTHON
@unittest.expectedFailure
@unittest.skip('TODO: RUSTPYTHON')
def test_detach(self):
r = self.BytesIO()
b = self.BufferedWriter(r)
@@ -2693,8 +2690,7 @@ class TextIOWrapperTest(unittest.TestCase):
t.write("A\rB")
self.assertEqual(r.getvalue(), b"XY\nZA\rB")
# TODO: RUSTPYTHON
@unittest.expectedFailure
@unittest.skip('TODO: RUSTPYTHON')
def test_reconfigure_line_buffering(self):
r = self.BytesIO()
b = self.BufferedWriter(r, 1000)
@@ -3924,16 +3920,12 @@ class PyTextIOWrapperTest(TextIOWrapperTest):
def test_newlines(self):
super().test_newlines()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_line_buffering(self):
super().test_line_buffering()
def test_seeking_too(self):
super().test_seeking_too()
# TODO: RUSTPYTHON, can't resize b/c of existing exports
@unittest.expectedFailure
def test_bufio_write_through(self):
super().test_bufio_write_through()

152
vm/src/buffer.rs
View File

@@ -3,12 +3,10 @@
use crate::common::borrow::{BorrowedValue, BorrowedValueMut};
use crate::common::rc::PyRc;
use crate::PyThreadingConstraint;
use crate::{PyObjectRef, PyResult, TypeProtocol};
use crate::{TryFromBorrowedObject, VirtualMachine};
use std::{borrow::Cow, fmt::Debug, ops::Deref};
use crate::{PyObjectRef, PyResult, TryFromBorrowedObject, TypeProtocol, VirtualMachine};
use std::{borrow::Cow, fmt::Debug};
pub trait PyBuffer: Debug + PyThreadingConstraint {
fn get_options(&self) -> &BufferOptions;
pub trait PyBufferInternal: Debug + PyThreadingConstraint {
/// Get the full inner buffer of this memory. You probably want [`as_contiguous()`], as
/// `obj_bytes` doesn't take into account the range a memoryview might operate on, among other
/// footguns.
@@ -18,49 +16,84 @@ pub trait PyBuffer: Debug + PyThreadingConstraint {
/// might operate on, among other footguns.
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]>;
fn release(&self);
// not included in PyBuffer protocol itself
fn retain(&self);
}
fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> {
if !self.get_options().contiguous {
#[derive(Debug)]
pub struct PyBuffer {
pub obj: PyObjectRef,
pub options: BufferOptions,
pub(crate) internal: PyRc<dyn PyBufferInternal>,
}
impl PyBuffer {
pub fn new(
obj: PyObjectRef,
buffer: impl PyBufferInternal + 'static,
options: BufferOptions,
) -> Self {
buffer.retain();
Self {
obj,
options,
internal: PyRc::new(buffer),
}
}
pub fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> {
if !self.options.contiguous {
return None;
}
Some(self.obj_bytes())
Some(self.internal.obj_bytes())
}
fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> {
if !self.get_options().contiguous {
pub fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> {
if !self.options.contiguous {
return None;
}
Some(self.obj_bytes_mut())
Some(self.internal.obj_bytes_mut())
}
fn to_contiguous(&self) -> Vec<u8> {
self.obj_bytes().to_vec()
pub fn to_contiguous(&self) -> Vec<u8> {
self.internal.obj_bytes().to_vec()
}
pub fn clone_with_options(&self, options: BufferOptions) -> Self {
self.internal.retain();
Self {
obj: self.obj.clone(),
options,
internal: self.internal.clone(),
}
}
}
#[derive(Debug, Clone)]
pub struct BufferOptions {
pub readonly: bool,
// buf
pub len: usize,
pub readonly: bool,
pub itemsize: usize,
pub contiguous: bool,
pub format: Cow<'static, str>,
// TODO: support multiple dimension array
pub ndim: usize,
pub ndim: usize, // TODO: support multiple dimension array
pub shape: Vec<usize>,
pub strides: Vec<isize>,
// suboffsets
// RustPython fields
pub contiguous: bool,
}
impl BufferOptions {
pub const DEFAULT: Self = BufferOptions {
readonly: true,
len: 0,
readonly: true,
itemsize: 1,
contiguous: true,
format: Cow::Borrowed("B"),
ndim: 1,
shape: Vec::new(),
strides: Vec::new(),
contiguous: true,
};
}
@@ -70,15 +103,12 @@ impl Default for BufferOptions {
}
}
#[derive(Debug)]
pub struct PyBufferRef(Box<dyn PyBuffer>);
impl TryFromBorrowedObject for PyBufferRef {
impl TryFromBorrowedObject for PyBuffer {
fn try_from_borrowed_object(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult<Self> {
let obj_cls = obj.class();
for cls in obj_cls.iter_mro() {
if let Some(f) = cls.slots.as_buffer.as_ref() {
return f(obj, vm).map(|x| PyBufferRef(x));
return f(obj, vm);
}
}
Err(vm.new_type_error(format!(
@@ -88,76 +118,18 @@ impl TryFromBorrowedObject for PyBufferRef {
}
}
impl Drop for PyBufferRef {
// What we actually want to implement is:
// impl<T> Drop for T where T: PyBufferInternal
// but it is not supported by Rust
impl Drop for PyBuffer {
fn drop(&mut self) {
self.0.release();
self.internal.release();
}
}
impl Deref for PyBufferRef {
type Target = dyn PyBuffer;
fn deref(&self) -> &Self::Target {
self.0.deref()
}
}
impl PyBufferRef {
pub fn new(buffer: impl PyBuffer + 'static) -> Self {
Self(Box::new(buffer))
}
pub fn into_rcbuf(self) -> RcBuffer {
// move self.0 out of self; PyBufferRef impls Drop so it's tricky
let this = std::mem::ManuallyDrop::new(self);
let buf_box = unsafe { std::ptr::read(&this.0) };
RcBuffer(buf_box.into())
}
}
impl From<Box<dyn PyBuffer>> for PyBufferRef {
fn from(buffer: Box<dyn PyBuffer>) -> Self {
PyBufferRef(buffer)
}
}
#[derive(Debug, Clone)]
pub struct RcBuffer(PyRc<dyn PyBuffer>);
impl Deref for RcBuffer {
type Target = dyn PyBuffer;
fn deref(&self) -> &Self::Target {
self.0.deref()
}
}
impl Drop for RcBuffer {
fn drop(&mut self) {
// check if this is the last rc before the inner buffer gets dropped
if let Some(buf) = PyRc::get_mut(&mut self.0) {
buf.release()
}
}
}
impl PyBuffer for RcBuffer {
fn get_options(&self) -> &BufferOptions {
self.0.get_options()
}
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.0.obj_bytes()
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
self.0.obj_bytes_mut()
}
fn release(&self) {}
fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> {
self.0.as_contiguous()
}
fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> {
self.0.as_contiguous_mut()
}
fn to_contiguous(&self) -> Vec<u8> {
self.0.to_contiguous()
impl Clone for PyBuffer {
fn clone(&self) -> Self {
self.clone_with_options(self.options.clone())
}
}

34
vm/src/builtins/bytearray.rs
View File

@@ -6,7 +6,7 @@ use super::pystr::PyStrRef;
use super::pytype::PyTypeRef;
use super::tuple::PyTupleRef;
use crate::anystr::{self, AnyStr};
use crate::buffer::{BufferOptions, PyBuffer, ResizeGuard};
use crate::buffer::{BufferOptions, PyBuffer, PyBufferInternal, ResizeGuard};
use crate::bytesinner::{
bytes_decode, bytes_from_object, value_from_object, ByteInnerFindOptions, ByteInnerNewOptions,
ByteInnerPaddingOptions, ByteInnerSplitOptions, ByteInnerTranslateOptions, DecodeArgs,
@@ -670,41 +670,35 @@ impl Comparable for PyByteArray {
}
impl AsBuffer for PyByteArray {
fn get_buffer(zelf: &PyRef<Self>, _vm: &VirtualMachine) -> PyResult<Box<dyn PyBuffer>> {
zelf.exports.fetch_add(1);
let buf = ByteArrayBuffer {
bytearray: zelf.clone(),
options: BufferOptions {
fn get_buffer(zelf: &PyRef<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let buffer = PyBuffer::new(
zelf.as_object().clone(),
zelf.clone(),
BufferOptions {
readonly: false,
len: zelf.len(),
..Default::default()
},
};
Ok(Box::new(buf))
);
Ok(buffer)
}
}
#[derive(Debug)]
struct ByteArrayBuffer {
bytearray: PyByteArrayRef,
options: BufferOptions,
}
impl PyBuffer for ByteArrayBuffer {
impl PyBufferInternal for PyRef<PyByteArray> {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.bytearray.borrow_buf().into()
self.borrow_buf().into()
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
PyRwLockWriteGuard::map(self.bytearray.inner_mut(), |inner| &mut *inner.elements).into()
PyRwLockWriteGuard::map(self.inner_mut(), |inner| &mut *inner.elements).into()
}
fn release(&self) {
self.bytearray.exports.fetch_sub(1);
self.exports.fetch_sub(1);
}
fn get_options(&self) -> &BufferOptions {
&self.options
fn retain(&self) {
self.exports.fetch_add(1);
}
}

30
vm/src/builtins/bytes.rs
View File

@@ -3,7 +3,7 @@ use super::int::PyIntRef;
use super::pystr::PyStrRef;
use super::pytype::PyTypeRef;
use crate::anystr::{self, AnyStr};
use crate::buffer::{BufferOptions, PyBuffer};
use crate::buffer::{BufferOptions, PyBuffer, PyBufferInternal};
use crate::builtins::tuple::PyTupleRef;
use crate::bytesinner::{
bytes_decode, ByteInnerFindOptions, ByteInnerNewOptions, ByteInnerPaddingOptions,
@@ -519,27 +519,22 @@ impl PyBytes {
}
impl AsBuffer for PyBytes {
fn get_buffer(zelf: &PyRef<Self>, _vm: &VirtualMachine) -> PyResult<Box<dyn PyBuffer>> {
let buf = BytesBuffer {
bytes: zelf.clone(),
options: BufferOptions {
fn get_buffer(zelf: &PyRef<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let buf = PyBuffer::new(
zelf.as_object().clone(),
zelf.clone(),
BufferOptions {
len: zelf.len(),
..Default::default()
},
};
Ok(Box::new(buf))
);
Ok(buf)
}
}
#[derive(Debug)]
struct BytesBuffer {
bytes: PyBytesRef,
options: BufferOptions,
}
impl PyBuffer for BytesBuffer {
impl PyBufferInternal for PyRef<PyBytes> {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.bytes.as_bytes().into()
self.as_bytes().into()
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
@@ -547,10 +542,7 @@ impl PyBuffer for BytesBuffer {
}
fn release(&self) {}
fn get_options(&self) -> &BufferOptions {
&self.options
}
fn retain(&self) {}
}
impl Hashable for PyBytes {

317
vm/src/builtins/memory.rs
View File

@@ -1,13 +1,16 @@
use crate::buffer::{BufferOptions, PyBuffer, PyBufferRef};
use crate::buffer::{BufferOptions, PyBuffer, PyBufferInternal};
use crate::builtins::bytes::{PyBytes, PyBytesRef};
use crate::builtins::list::{PyList, PyListRef};
use crate::builtins::pystr::{PyStr, PyStrRef};
use crate::builtins::pytype::PyTypeRef;
use crate::builtins::slice::PySliceRef;
use crate::bytesinner::bytes_to_hex;
use crate::common::borrow::{BorrowedValue, BorrowedValueMut};
use crate::common::hash::PyHash;
use crate::common::lock::OnceCell;
use crate::common::{
borrow::{BorrowedValue, BorrowedValueMut},
hash::PyHash,
lock::OnceCell,
rc::PyRc,
};
use crate::function::{FuncArgs, OptionalArg};
use crate::sliceable::{convert_slice, saturate_range, wrap_index, SequenceIndex};
use crate::slots::{AsBuffer, Comparable, Hashable, PyComparisonOp, SlotConstructor};
@@ -15,14 +18,14 @@ use crate::stdlib::pystruct::_struct::FormatSpec;
use crate::utils::Either;
use crate::{
IdProtocol, IntoPyObject, PyClassImpl, PyComparisonValue, PyContext, PyObjectRef, PyRef,
PyResult, PyValue, TypeProtocol,
PyResult, PyValue, TryFromBorrowedObject, TryFromObject, TypeProtocol, VirtualMachine,
};
use crate::{TryFromBorrowedObject, TryFromObject, VirtualMachine};
use crossbeam_utils::atomic::AtomicCell;
use itertools::Itertools;
use num_bigint::BigInt;
use num_traits::{One, Signed, ToPrimitive, Zero};
use std::fmt::Debug;
use std::ops::Deref;
#[derive(FromArgs)]
pub struct PyMemoryViewNewArgs {
@@ -33,9 +36,7 @@ pub struct PyMemoryViewNewArgs {
#[pyclass(module = false, name = "memoryview")]
#[derive(Debug)]
pub struct PyMemoryView {
obj: PyObjectRef,
buffer: PyBufferRef,
options: BufferOptions,
buffer: PyBuffer,
pub(crate) released: AtomicCell<bool>,
// start should always less or equal to the stop
// start and stop pointing to the memory index not slice index
@@ -44,7 +45,6 @@ pub struct PyMemoryView {
stop: usize,
// step can be negative, means read the memory from stop-1 to start
step: isize,
exports: AtomicCell<usize>,
format_spec: FormatSpec,
hash: OnceCell<PyHash>,
}
@@ -53,84 +53,94 @@ impl SlotConstructor for PyMemoryView {
type Args = PyMemoryViewNewArgs;
fn py_new(cls: PyTypeRef, args: Self::Args, vm: &VirtualMachine) -> PyResult {
let buffer = PyBufferRef::try_from_borrowed_object(vm, &args.object)?;
let zelf = PyMemoryView::from_buffer(args.object, buffer, vm)?;
let buffer = PyBuffer::try_from_borrowed_object(vm, &args.object)?;
let zelf = PyMemoryView::from_buffer(buffer, vm)?;
zelf.into_pyresult_with_type(vm, cls)
}
}
type PyMemoryViewRef = PyRef<PyMemoryView>;
#[pyimpl(with(Hashable, Comparable, AsBuffer, SlotConstructor))]
impl PyMemoryView {
fn parse_format(format: &str, vm: &VirtualMachine) -> PyResult<FormatSpec> {
FormatSpec::parse(format, vm)
}
pub fn from_buffer(
obj: PyObjectRef,
buffer: PyBufferRef,
vm: &VirtualMachine,
) -> PyResult<Self> {
pub fn from_buffer(buffer: PyBuffer, vm: &VirtualMachine) -> PyResult<Self> {
// when we get a buffer means the buffered object is size locked
// so we can assume the buffer's options will never change as long
// as memoryview is still alive
let options = buffer.get_options().clone();
let len = options.len;
let itemsize = options.itemsize;
let options = &buffer.options;
let stop = options.len * options.itemsize;
let format_spec = Self::parse_format(&options.format, vm)?;
Ok(PyMemoryView {
obj,
buffer,
options,
released: AtomicCell::new(false),
start: 0,
stop: len * itemsize,
stop,
step: 1,
exports: AtomicCell::new(0),
format_spec,
hash: OnceCell::new(),
})
}
pub fn from_buffer_range(
obj: PyObjectRef,
buffer: PyBufferRef,
mut buffer: PyBuffer,
range: std::ops::Range<usize>,
vm: &VirtualMachine,
) -> PyResult<Self> {
let options = buffer.get_options().clone();
let itemsize = options.itemsize;
let format_spec = Self::parse_format(&options.format, vm)?;
let itemsize = buffer.options.itemsize;
let format_spec = Self::parse_format(&buffer.options.format, vm)?;
buffer.options.len = range.len();
Ok(PyMemoryView {
obj,
buffer,
options,
released: AtomicCell::new(false),
start: range.start * itemsize,
stop: range.end * itemsize,
step: 1,
exports: AtomicCell::new(0),
format_spec,
hash: OnceCell::new(),
})
}
fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> {
if !self.buffer.options.contiguous {
return None;
}
Some(self.obj_bytes())
}
fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> {
if !self.buffer.options.contiguous {
return None;
}
Some(self.obj_bytes_mut())
}
pub fn try_bytes<F, R>(&self, vm: &VirtualMachine, f: F) -> PyResult<R>
where
F: FnOnce(&[u8]) -> R,
{
self.try_not_released(vm)?;
self.buffer.as_contiguous().map(|x| f(&*x)).ok_or_else(|| {
self.as_contiguous().map(|x| f(&*x)).ok_or_else(|| {
vm.new_type_error("non-contiguous memoryview is not a bytes-like object".to_owned())
})
}
#[pymethod]
fn release(&self) {
self._release();
}
fn _release(&self) {
// avoid double release
if self.released.compare_exchange(false, true).is_ok() && self.exports.load() == 0 {
self.buffer.release();
if self.released.compare_exchange(false, true).is_ok() {
unsafe {
// SAFETY: this branch is only once accessible form _release and guarded by AtomicCell released
let buffer: &std::cell::UnsafeCell<PyBuffer> = std::mem::transmute(&self.buffer);
let buffer = &mut *buffer.get();
let internal = std::mem::replace(&mut buffer.internal, PyRc::new(Released));
internal.release();
}
}
}
@@ -144,35 +154,37 @@ impl PyMemoryView {
#[pyproperty]
fn obj(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> {
self.try_not_released(vm).map(|_| self.obj.clone())
self.try_not_released(vm).map(|_| self.buffer.obj.clone())
}
#[pyproperty]
fn nbytes(&self, vm: &VirtualMachine) -> PyResult<usize> {
self.try_not_released(vm)
.map(|_| self.options.len * self.options.itemsize)
.map(|_| self.buffer.options.len * self.buffer.options.itemsize)
}
#[pyproperty]
fn readonly(&self, vm: &VirtualMachine) -> PyResult<bool> {
self.try_not_released(vm).map(|_| self.options.readonly)
self.try_not_released(vm)
.map(|_| self.buffer.options.readonly)
}
#[pyproperty]
fn itemsize(&self, vm: &VirtualMachine) -> PyResult<usize> {
self.try_not_released(vm).map(|_| self.options.itemsize)
self.try_not_released(vm)
.map(|_| self.buffer.options.itemsize)
}
#[pyproperty]
fn ndim(&self, vm: &VirtualMachine) -> PyResult<usize> {
self.try_not_released(vm).map(|_| self.options.ndim)
self.try_not_released(vm).map(|_| self.buffer.options.ndim)
}
// TODO
#[pyproperty]
fn shape(&self, vm: &VirtualMachine) -> PyResult<PyObjectRef> {
self.try_not_released(vm)
.map(|_| (self.options.len,).into_pyobject(vm))
.map(|_| (self.buffer.options.len,).into_pyobject(vm))
}
// TODO
@@ -184,7 +196,7 @@ impl PyMemoryView {
#[pyproperty]
fn format(&self, vm: &VirtualMachine) -> PyResult<PyStr> {
self.try_not_released(vm)
.map(|_| PyStr::from(&self.options.format))
.map(|_| PyStr::from(&self.buffer.options.format))
}
#[pymethod(magic)]
@@ -194,18 +206,18 @@ impl PyMemoryView {
#[pymethod(magic)]
fn exit(&self, _args: FuncArgs) {
self.release();
self._release();
}
// translate the slice index to memory index
fn get_pos(&self, i: isize) -> Option<usize> {
let len = self.options.len;
let itemsize = self.options.itemsize;
let len = self.buffer.options.len;
let itemsize = self.buffer.options.itemsize;
let i = wrap_index(i, len)?;
let i = if self.step < 0 {
self.stop - itemsize - (-self.step as usize) * i * itemsize
(len - 1 + (self.step as usize) * i) * itemsize
} else {
self.start + self.step as usize * i * itemsize
self.step as usize * i * itemsize
};
Some(i)
}
@@ -214,15 +226,14 @@ impl PyMemoryView {
let i = zelf
.get_pos(i)
.ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?;
zelf.format_spec
.unpack(&zelf.obj_bytes()[i..i + zelf.options.itemsize], vm)
.map(|x| {
if x.len() == 1 {
x.fast_getitem(0)
} else {
x.into_object()
}
})
let bytes = &zelf.obj_bytes()[i..i + zelf.buffer.options.itemsize];
zelf.format_spec.unpack(bytes, vm).map(|x| {
if x.len() == 1 {
x.fast_getitem(0)
} else {
x.into_object()
}
})
}
fn getitem_by_slice(zelf: PyRef<Self>, slice: PySliceRef, vm: &VirtualMachine) -> PyResult {
@@ -234,13 +245,9 @@ impl PyMemoryView {
return Err(vm.new_value_error("slice step cannot be zero".to_owned()));
}
let newstep: BigInt = step.clone() * zelf.step;
let len = zelf.options.len;
let itemsize = zelf.options.itemsize;
let len = zelf.buffer.options.len;
let itemsize = zelf.buffer.options.itemsize;
let obj = zelf.obj.clone();
let buffer = PyBufferRef::new(zelf.clone());
zelf.exports.fetch_add(1);
let options = zelf.options.clone();
let format_spec = zelf.format_spec.clone();
if newstep == BigInt::one() {
@@ -254,18 +261,15 @@ impl PyMemoryView {
let start = zelf.start + range.start * itemsize;
let stop = zelf.start + range.end * itemsize;
return Ok(PyMemoryView {
obj,
buffer,
options: BufferOptions {
buffer: zelf.buffer.clone_with_options(BufferOptions {
len: newlen,
contiguous: true,
..options
},
..zelf.buffer.options.clone()
}),
released: AtomicCell::new(false),
start,
stop,
step: 1,
exports: AtomicCell::new(0),
format_spec,
hash: OnceCell::new(),
}
@@ -302,18 +306,15 @@ impl PyMemoryView {
+ 1
} else {
return Ok(PyMemoryView {
obj,
buffer,
options: BufferOptions {
buffer: zelf.buffer.clone_with_options(BufferOptions {
len: 0,
contiguous: true,
..options
},
..zelf.buffer.options.clone()
}),
released: AtomicCell::new(false),
start: range.end,
stop: range.end,
step: 1,
exports: AtomicCell::new(0),
format_spec,
hash: OnceCell::new(),
}
@@ -333,19 +334,17 @@ impl PyMemoryView {
(start, stop)
};
let options = BufferOptions {
len: newlen,
contiguous: false,
..zelf.buffer.options.clone()
};
Ok(PyMemoryView {
obj,
buffer,
options: BufferOptions {
len: newlen,
contiguous: false,
..options
},
buffer: zelf.buffer.clone_with_options(options),
released: AtomicCell::new(false),
start,
stop,
step: newstep,
exports: AtomicCell::new(0),
format_spec,
hash: OnceCell::new(),
}
@@ -370,7 +369,7 @@ impl PyMemoryView {
let i = zelf
.get_pos(i)
.ok_or_else(|| vm.new_index_error("index out of range".to_owned()))?;
let itemsize = zelf.options.itemsize;
let itemsize = zelf.buffer.options.itemsize;
let data = zelf.format_spec.pack(vec![value], vm)?;
zelf.obj_bytes_mut()[i..i + itemsize].copy_from_slice(&data);
Ok(())
@@ -382,15 +381,15 @@ impl PyMemoryView {
items: PyObjectRef,
vm: &VirtualMachine,
) -> PyResult<()> {
let items = PyBufferRef::try_from_object(vm, items)?;
let options = items.get_options();
let items = PyBuffer::try_from_object(vm, items)?;
let options = &items.options;
let len = options.len;
let itemsize = options.itemsize;
if itemsize != zelf.options.itemsize {
if itemsize != zelf.buffer.options.itemsize {
return Err(vm.new_type_error(format!(
"memoryview: invalid type for format '{}'",
zelf.options.format
zelf.buffer.options.format
)));
}
@@ -400,11 +399,11 @@ impl PyMemoryView {
))
};
if options.format != zelf.options.format {
if options.format != zelf.buffer.options.format {
return diff_err();
}
let (range, step, is_negative_step) = convert_slice(&slice, zelf.options.len, vm)?;
let (range, step, is_negative_step) = convert_slice(&slice, zelf.buffer.options.len, vm)?;
let bytes = items.to_contiguous();
assert_eq!(bytes.len(), len * itemsize);
@@ -475,7 +474,7 @@ impl PyMemoryView {
vm: &VirtualMachine,
) -> PyResult<()> {
zelf.try_not_released(vm)?;
if zelf.options.readonly {
if zelf.buffer.options.readonly {
return Err(vm.new_type_error("cannot modify read-only memory".to_owned()));
}
match needle {
@@ -486,7 +485,7 @@ impl PyMemoryView {
#[pymethod(magic)]
fn len(&self, vm: &VirtualMachine) -> PyResult<usize> {
self.try_not_released(vm).map(|_| self.options.len)
self.try_not_released(vm).map(|_| self.buffer.options.len)
}
fn to_bytes_vec(zelf: &PyRef<Self>) -> Vec<u8> {
@@ -494,8 +493,8 @@ impl PyMemoryView {
bytes.to_vec()
} else {
let bytes = &*zelf.obj_bytes();
let len = zelf.options.len;
let itemsize = zelf.options.itemsize;
let len = zelf.buffer.options.len;
let itemsize = zelf.buffer.options.itemsize;
(0..len)
.map(|i| zelf.get_pos(i as isize).unwrap())
.flat_map(|i| bytes[i..i + itemsize].to_vec())
@@ -514,9 +513,15 @@ impl PyMemoryView {
zelf.try_not_released(vm)?;
let bytes = &*zelf.obj_bytes();
let elements: Vec<PyObjectRef> = (0..zelf.options.len)
let elements: Vec<PyObjectRef> = (0..zelf.buffer.options.len)
.map(|i| zelf.get_pos(i as isize).unwrap())
.map(|i| format_unpack(&zelf.format_spec, &bytes[i..i + zelf.options.itemsize], vm))
.map(|i| {
format_unpack(
&zelf.format_spec,
&bytes[i..i + zelf.buffer.options.itemsize],
vm,
)
})
.try_collect()?;
Ok(PyList::from(elements).into_ref(vm))
@@ -525,17 +530,13 @@ impl PyMemoryView {
#[pymethod]
fn toreadonly(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
zelf.try_not_released(vm)?;
let buffer = PyBufferRef::new(zelf.clone());
zelf.exports.fetch_add(1);
let buffer = zelf.buffer.clone_with_options(BufferOptions {
readonly: true,
..zelf.buffer.options.clone()
});
Ok(PyMemoryView {
obj: zelf.obj.clone(),
buffer,
options: BufferOptions {
readonly: true,
..zelf.options.clone()
},
released: AtomicCell::new(false),
exports: AtomicCell::new(0),
format_spec: zelf.format_spec.clone(),
hash: OnceCell::new(),
..*zelf
@@ -568,7 +569,7 @@ impl PyMemoryView {
&*guard
}
None => {
vec = zelf.to_contiguous();
vec = Self::to_bytes_vec(&zelf);
vec.as_slice()
}
};
@@ -580,7 +581,7 @@ impl PyMemoryView {
#[pymethod]
fn cast(zelf: PyRef<Self>, format: PyStrRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
zelf.try_not_released(vm)?;
if !zelf.options.contiguous {
if !zelf.buffer.options.contiguous {
return Err(vm.new_type_error(
"memoryview: casts are restricted to C-contiguous views".to_owned(),
));
@@ -588,7 +589,7 @@ impl PyMemoryView {
let format_spec = Self::parse_format(format.as_str(), vm)?;
let itemsize = format_spec.size();
let bytelen = zelf.options.len * zelf.options.itemsize;
let bytelen = zelf.buffer.options.len * zelf.buffer.options.itemsize;
if bytelen % itemsize != 0 {
return Err(
@@ -596,21 +597,15 @@ impl PyMemoryView {
);
}
let buffer = PyBufferRef::new(zelf.clone());
zelf.exports.fetch_add(1);
let buffer = zelf.buffer.clone_with_options(BufferOptions {
itemsize,
len: bytelen / itemsize,
format: format.to_string().into(),
..zelf.buffer.options.clone()
});
Ok(PyMemoryView {
obj: zelf.obj.clone(),
buffer,
options: BufferOptions {
itemsize,
len: bytelen / itemsize,
format: format.to_string().into(),
..zelf.options.clone()
},
released: AtomicCell::new(false),
stop: zelf.stop + itemsize - zelf.options.itemsize,
exports: AtomicCell::new(0),
format_spec,
hash: OnceCell::new(),
..*zelf
@@ -626,13 +621,13 @@ impl PyMemoryView {
return Ok(false);
}
let other = match PyBufferRef::try_from_borrowed_object(vm, other) {
let other = match PyBuffer::try_from_borrowed_object(vm, other) {
Ok(buf) => buf,
Err(_) => return Ok(false),
};
let a_options = &zelf.options;
let b_options = other.get_options();
let a_options = &zelf.buffer.options;
let b_options = &other.options;
if a_options.len != b_options.len
|| a_options.ndim != b_options.ndim
@@ -649,7 +644,7 @@ impl PyMemoryView {
&*a_guard
}
None => {
a_vec = zelf.to_contiguous();
a_vec = Self::to_bytes_vec(zelf);
a_vec.as_slice()
}
};
@@ -687,62 +682,64 @@ impl PyMemoryView {
}
}
impl Drop for PyMemoryView {
fn drop(&mut self) {
self.release();
}
}
impl AsBuffer for PyMemoryView {
fn get_buffer(zelf: &PyRef<Self>, vm: &VirtualMachine) -> PyResult<Box<dyn PyBuffer>> {
fn get_buffer(zelf: &PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyBuffer> {
if zelf.released.load() {
Err(vm.new_value_error("operation forbidden on released memoryview object".to_owned()))
} else {
Ok(Box::new(zelf.clone()))
Ok(PyBuffer::new(
zelf.as_object().clone(),
zelf.clone(),
zelf.buffer.options.clone(),
))
}
}
}
impl PyBuffer for PyMemoryViewRef {
fn get_options(&self) -> &BufferOptions {
&self.options
}
#[derive(Debug)]
struct Released;
impl PyBufferInternal for Released {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.buffer.obj_bytes()
panic!();
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
self.buffer.obj_bytes_mut()
panic!();
}
fn release(&self) {
if self.exports.fetch_sub(1) == 1 && !self.released.load() {
self.buffer.release();
}
}
fn release(&self) {}
fn as_contiguous(&self) -> Option<BorrowedValue<[u8]>> {
if !self.options.contiguous {
return None;
}
Some(BorrowedValue::map(self.obj_bytes(), |x| {
fn retain(&self) {}
}
impl PyBufferInternal for PyMemoryView {
// NOTE: This impl maybe is anti-pattern. Only used for internal usage.
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
BorrowedValue::map(self.buffer.internal.obj_bytes(), |x| {
&x[self.start..self.stop]
}))
})
}
fn as_contiguous_mut(&self) -> Option<BorrowedValueMut<[u8]>> {
if !self.options.contiguous {
return None;
}
Some(BorrowedValueMut::map(self.obj_bytes_mut(), |x| {
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
BorrowedValueMut::map(self.buffer.internal.obj_bytes_mut(), |x| {
&mut x[self.start..self.stop]
}))
})
}
fn to_contiguous(&self) -> Vec<u8> {
PyMemoryView::to_bytes_vec(self)
fn release(&self) {}
fn retain(&self) {}
}
impl PyBufferInternal for PyRef<PyMemoryView> {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.deref().obj_bytes()
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
self.deref().obj_bytes_mut()
}
fn release(&self) {}
fn retain(&self) {}
}
impl Comparable for PyMemoryView {
@@ -772,7 +769,7 @@ impl Hashable for PyMemoryView {
zelf.hash
.get_or_try_init(|| {
zelf.try_not_released(vm)?;
if !zelf.options.readonly {
if !zelf.buffer.options.readonly {
return Err(
vm.new_value_error("cannot hash writable memoryview object".to_owned())
);
@@ -785,7 +782,7 @@ impl Hashable for PyMemoryView {
&*guard
}
None => {
vec = zelf.to_contiguous();
vec = Self::to_bytes_vec(zelf);
vec.as_slice()
}
};

25
vm/src/byteslike.rs
View File

@@ -1,4 +1,4 @@
use crate::buffer::PyBufferRef;
use crate::buffer::PyBuffer;
use crate::builtins::PyStrRef;
use crate::common::borrow::{BorrowedValue, BorrowedValueMut};
use crate::vm::VirtualMachine;
@@ -8,11 +8,11 @@ use crate::{PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject};
/// any bytes-like object. Like the `y*` format code for `PyArg_Parse` in CPython.
#[derive(Debug)]
pub struct ArgBytesLike(PyBufferRef);
pub struct ArgBytesLike(PyBuffer);
/// A memory buffer, read-write access. Like the `w*` format code for `PyArg_Parse` in CPython.
#[derive(Debug)]
pub struct ArgMemoryBuffer(PyBufferRef);
pub struct ArgMemoryBuffer(PyBuffer);
impl ArgBytesLike {
pub fn with_ref<F, R>(&self, f: F) -> R
@@ -54,15 +54,15 @@ impl ArgMemoryBuffer {
impl ArgBytesLike {
pub fn new(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult<Self> {
let buffer = PyBufferRef::try_from_borrowed_object(vm, obj)?;
if buffer.get_options().contiguous {
let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?;
if buffer.options.contiguous {
Ok(Self(buffer))
} else {
Err(vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned()))
}
}
pub fn into_buffer(self) -> PyBufferRef {
pub fn into_buffer(self) -> PyBuffer {
self.0
}
@@ -82,7 +82,7 @@ pub fn try_bytes_like<R>(
obj: &PyObjectRef,
f: impl FnOnce(&[u8]) -> R,
) -> PyResult<R> {
let buffer = PyBufferRef::try_from_borrowed_object(vm, obj)?;
let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?;
buffer.as_contiguous().map(|x| f(&*x)).ok_or_else(|| {
vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned())
})
@@ -93,7 +93,7 @@ pub fn try_rw_bytes_like<R>(
obj: &PyObjectRef,
f: impl FnOnce(&mut [u8]) -> R,
) -> PyResult<R> {
let buffer = PyBufferRef::try_from_borrowed_object(vm, obj)?;
let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?;
buffer
.as_contiguous_mut()
.map(|mut x| f(&mut *x))
@@ -102,18 +102,17 @@ pub fn try_rw_bytes_like<R>(
impl ArgMemoryBuffer {
pub fn new(vm: &VirtualMachine, obj: &PyObjectRef) -> PyResult<Self> {
let buffer = PyBufferRef::try_from_borrowed_object(vm, obj)?;
let options = buffer.get_options();
if !options.contiguous {
let buffer = PyBuffer::try_from_borrowed_object(vm, obj)?;
if !buffer.options.contiguous {
Err(vm.new_type_error("non-contiguous buffer is not a bytes-like object".to_owned()))
} else if options.readonly {
} else if buffer.options.readonly {
Err(vm.new_type_error("buffer is not a read-write bytes-like object".to_owned()))
} else {
Ok(Self(buffer))
}
}
pub fn into_buffer(self) -> PyBufferRef {
pub fn into_buffer(self) -> PyBuffer {
self.0
}

4
vm/src/cformat.rs
View File

@@ -1,4 +1,4 @@
use crate::buffer::PyBufferRef;
use crate::buffer::PyBuffer;
/// Implementation of Printf-Style string formatting
/// [https://docs.python.org/3/library/stdtypes.html#printf-style-string-formatting]
use crate::builtins::float::{try_bigint, IntoPyFloat, PyFloat};
@@ -366,7 +366,7 @@ impl CFormatSpec {
Ok(s.into_bytes())
}
CFormatPreconversor::Str | CFormatPreconversor::Bytes => {
if let Ok(buffer) = PyBufferRef::try_from_borrowed_object(vm, &obj) {
if let Ok(buffer) = PyBuffer::try_from_borrowed_object(vm, &obj) {
let guard;
let vec;
let bytes = match buffer.as_contiguous() {

16
vm/src/slots.rs
View File

@@ -61,7 +61,7 @@ pub(crate) type RichCompareFunc = fn(
pub(crate) type GetattroFunc = fn(PyObjectRef, PyStrRef, &VirtualMachine) -> PyResult;
pub(crate) type SetattroFunc =
fn(&PyObjectRef, PyStrRef, Option<PyObjectRef>, &VirtualMachine) -> PyResult<()>;
pub(crate) type BufferFunc = fn(&PyObjectRef, &VirtualMachine) -> PyResult<Box<dyn PyBuffer>>;
pub(crate) type BufferFunc = fn(&PyObjectRef, &VirtualMachine) -> PyResult<PyBuffer>;
pub(crate) type IterFunc = fn(PyObjectRef, &VirtualMachine) -> PyResult;
pub(crate) type IterNextFunc = fn(&PyObjectRef, &VirtualMachine) -> PyResult;
@@ -510,16 +510,16 @@ pub trait SlotSetattro: PyValue {
#[pyimpl]
pub trait AsBuffer: PyValue {
// TODO: `flags` parameter
#[pyslot]
fn tp_as_buffer(zelf: &PyObjectRef, vm: &VirtualMachine) -> PyResult<Box<dyn PyBuffer>> {
if let Some(zelf) = zelf.downcast_ref() {
Self::get_buffer(zelf, vm)
} else {
Err(vm.new_type_error("unexpected payload for get_buffer".to_owned()))
}
fn as_buffer(zelf: &PyObjectRef, vm: &VirtualMachine) -> PyResult<PyBuffer> {
let zelf = zelf
.downcast_ref()
.ok_or_else(|| vm.new_type_error("unexpected payload for get_buffer".to_owned()))?;
Self::get_buffer(zelf, vm)
}
fn get_buffer(zelf: &PyRef<Self>, vm: &VirtualMachine) -> PyResult<Box<dyn PyBuffer>>;
fn get_buffer(zelf: &PyRef<Self>, vm: &VirtualMachine) -> PyResult<PyBuffer>;
}
#[pyimpl]

34
vm/src/stdlib/array.rs
View File

@@ -9,7 +9,7 @@ pub type wchar_t = u32;
#[pymodule(name = "array")]
mod array {
use crate::buffer::{BufferOptions, PyBuffer, ResizeGuard};
use crate::buffer::{BufferOptions, PyBuffer, PyBufferInternal, ResizeGuard};
use crate::builtins::float::IntoPyFloat;
use crate::builtins::list::{PyList, PyListRef};
use crate::builtins::pystr::{PyStr, PyStrRef};
@@ -1088,44 +1088,38 @@ mod array {
}
impl AsBuffer for PyArray {
fn get_buffer(zelf: &PyRef<Self>, _vm: &VirtualMachine) -> PyResult<Box<dyn PyBuffer>> {
zelf.exports.fetch_add(1);
fn get_buffer(zelf: &PyRef<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let array = zelf.read();
let buf = ArrayBuffer {
array: zelf.clone(),
options: BufferOptions {
let buf = PyBuffer::new(
zelf.as_object().clone(),
zelf.clone(),
BufferOptions {
readonly: false,
len: array.len(),
itemsize: array.itemsize(),
format: array.typecode_str().into(),
..Default::default()
},
};
Ok(Box::new(buf))
);
Ok(buf)
}
}
#[derive(Debug)]
struct ArrayBuffer {
array: PyArrayRef,
options: BufferOptions,
}
impl PyBuffer for ArrayBuffer {
impl PyBufferInternal for PyRef<PyArray> {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.array.get_bytes().into()
self.get_bytes().into()
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
self.array.get_bytes_mut().into()
self.get_bytes_mut().into()
}
fn release(&self) {
self.array.exports.fetch_sub(1);
self.exports.fetch_sub(1);
}
fn get_options(&self) -> &BufferOptions {
&self.options
fn retain(&self) {
self.exports.fetch_add(1);
}
}

102
vm/src/stdlib/io.rs
View File

@@ -77,7 +77,7 @@ mod _io {
use std::io::{self, prelude::*, Cursor, SeekFrom};
use std::ops::Range;
use crate::buffer::{BufferOptions, PyBuffer, PyBufferRef, ResizeGuard};
use crate::buffer::{BufferOptions, PyBuffer, PyBufferInternal, ResizeGuard};
use crate::builtins::memory::PyMemoryView;
use crate::builtins::{
bytes::{PyBytes, PyBytesRef},
@@ -858,14 +858,13 @@ mod _io {
/// None means non-blocking failed
fn raw_write(
&mut self,
buf: Option<PyBufferRef>,
buf: Option<PyBuffer>,
buf_range: Range<usize>,
vm: &VirtualMachine,
) -> PyResult<Option<usize>> {
let len = buf_range.len();
let res = if let Some(buf) = buf {
let memobj = PyMemoryView::from_buffer_range(vm.ctx.none(), buf, buf_range, vm)?
.into_pyobject(vm);
let memobj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?.into_pyobject(vm);
// TODO: loop if write() raises an interrupt
vm.call_method(self.raw.as_ref().unwrap(), "write", (memobj,))?
@@ -879,17 +878,20 @@ mod _io {
let writebuf = PyRc::new(BufferedRawBuffer {
data: std::mem::take(&mut self.buffer).into(),
range: buf_range,
options,
});
let raw = self.raw.as_ref().unwrap();
let memobj = PyMemoryView::from_buffer(
vm.ctx.none(),
PyBufferRef::new(writebuf.clone()),
PyBuffer {
obj: raw.clone(),
options,
internal: writebuf.clone(),
},
vm,
)?
.into_ref(vm);
// TODO: loop if write() raises an interrupt
let res = vm.call_method(self.raw.as_ref().unwrap(), "write", (memobj.clone(),));
let res = vm.call_method(raw, "write", (memobj.clone(),));
memobj.released.store(true);
self.buffer = std::mem::take(&mut writebuf.data.lock());
@@ -947,10 +949,9 @@ mod _io {
let mut remaining = buf_len;
let mut written = 0;
let rcbuf = obj.into_buffer().into_rcbuf();
let buffer = obj.into_buffer();
while remaining > self.buffer.len() {
let res =
self.raw_write(Some(PyBufferRef::new(rcbuf.clone())), written..buf_len, vm)?;
let res = self.raw_write(Some(buffer.clone()), written..buf_len, vm)?;
match res {
Some(n) => {
written += n;
@@ -965,7 +966,7 @@ mod _io {
// raw file is non-blocking
if remaining > self.buffer.len() {
// can't buffer everything, buffer what we can and error
let buf = rcbuf.as_contiguous().unwrap();
let buf = buffer.as_contiguous().unwrap();
let buffer_len = self.buffer.len();
self.buffer.copy_from_slice(&buf[written..][..buffer_len]);
self.raw_pos = 0;
@@ -988,7 +989,7 @@ mod _io {
self.reset_read();
}
if remaining > 0 {
let buf = rcbuf.as_contiguous().unwrap();
let buf = buffer.as_contiguous().unwrap();
self.buffer[..remaining].copy_from_slice(&buf[written..][..remaining]);
written += remaining;
}
@@ -1098,7 +1099,7 @@ mod _io {
fn raw_read(
&mut self,
v: Either<Option<&mut Vec<u8>>, PyBufferRef>,
v: Either<Option<&mut Vec<u8>>, PyBuffer>,
buf_range: Range<usize>,
vm: &VirtualMachine,
) -> PyResult<Option<usize>> {
@@ -1116,11 +1117,13 @@ mod _io {
let readbuf = PyRc::new(BufferedRawBuffer {
data: std::mem::take(v).into(),
range: buf_range,
options,
});
let memobj = PyMemoryView::from_buffer(
vm.ctx.none(),
PyBufferRef::new(readbuf.clone()),
PyBuffer {
obj: vm.ctx.none(),
options,
internal: readbuf.clone(),
},
vm,
)?
.into_ref(vm);
@@ -1135,8 +1138,7 @@ mod _io {
res?
}
Either::B(buf) => {
let memobj =
PyMemoryView::from_buffer_range(vm.ctx.none(), buf, buf_range, vm)?;
let memobj = PyMemoryView::from_buffer_range(buf, buf_range, vm)?;
// TODO: loop if readinto() raises an interrupt
vm.call_method(self.raw.as_ref().unwrap(), "readinto", (memobj,))?
}
@@ -1244,7 +1246,7 @@ mod _io {
fn readinto_generic(
&mut self,
buf: PyBufferRef,
buf: PyBuffer,
readinto1: bool,
vm: &VirtualMachine,
) -> PyResult<Option<usize>> {
@@ -1272,17 +1274,15 @@ mod _io {
self.reset_read();
self.pos = 0;
let rcbuf = buf.into_rcbuf();
let mut remaining = buf_len - written;
while remaining > 0 {
let n = if remaining as usize > self.buffer.len() {
let buf = PyBufferRef::new(rcbuf.clone());
self.raw_read(Either::B(buf), written..written + remaining, vm)?
self.raw_read(Either::B(buf.clone()), written..written + remaining, vm)?
} else if !(readinto1 && written != 0) {
let n = self.fill_buffer(vm)?;
if let Some(n) = n.filter(|&n| n > 0) {
let n = std::cmp::min(n, remaining);
rcbuf.as_contiguous_mut().unwrap()[written..][..n]
buf.as_contiguous_mut().unwrap()[written..][..n]
.copy_from_slice(&self.buffer[self.pos as usize..][..n]);
self.pos += n as Offset;
written += n;
@@ -1319,13 +1319,8 @@ mod _io {
struct BufferedRawBuffer {
data: PyMutex<Vec<u8>>,
range: Range<usize>,
options: BufferOptions,
}
impl PyBuffer for PyRc<BufferedRawBuffer> {
fn get_options(&self) -> &BufferOptions {
&self.options
}
impl PyBufferInternal for BufferedRawBuffer {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
BorrowedValue::map(self.data.lock().into(), |data| &data[self.range.clone()])
}
@@ -1337,6 +1332,7 @@ mod _io {
}
fn release(&self) {}
fn retain(&self) {}
}
pub fn get_offset(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<Offset> {
@@ -3352,47 +3348,33 @@ mod _io {
#[pymethod]
fn getbuffer(self, vm: &VirtualMachine) -> PyResult<PyMemoryView> {
self.exports.fetch_add(1);
let buffer = PyBufferRef::new(BytesIOBuffer {
bytesio: self.clone(),
options: BufferOptions {
readonly: false,
len: self.buffer.read().cursor.get_ref().len(),
..Default::default()
},
});
let view = PyMemoryView::from_buffer(self.into_object(), buffer, vm)?;
let options = BufferOptions {
readonly: false,
len: self.buffer.read().cursor.get_ref().len(),
..Default::default()
};
let buffer = PyBuffer::new(self.as_object().clone(), self, options);
let view = PyMemoryView::from_buffer(buffer, vm)?;
Ok(view)
}
}
#[derive(Debug)]
struct BytesIOBuffer {
bytesio: BytesIORef,
options: BufferOptions,
}
impl PyBuffer for BytesIOBuffer {
fn get_options(&self) -> &BufferOptions {
&self.options
}
impl PyBufferInternal for PyRef<BytesIO> {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
PyRwLockReadGuard::map(self.bytesio.buffer.read(), |x| {
x.cursor.get_ref().as_slice()
})
.into()
PyRwLockReadGuard::map(self.buffer.read(), |x| x.cursor.get_ref().as_slice()).into()
}
fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
PyRwLockWriteGuard::map(self.bytesio.buffer.write(), |x| {
x.cursor.get_mut().as_mut_slice()
})
.into()
PyRwLockWriteGuard::map(self.buffer.write(), |x| x.cursor.get_mut().as_mut_slice())
.into()
}
fn release(&self) {
self.bytesio.exports.fetch_sub(1);
self.exports.fetch_sub(1);
}
fn retain(&self) {
self.exports.fetch_add(1);
}
}

16
vm/src/stdlib/marshal.rs
View File

@@ -22,14 +22,14 @@ mod decl {
#[pyfunction]
fn loads(code_bytes: ArgBytesLike, vm: &VirtualMachine) -> PyResult<PyCode> {
let code =
bytecode::CodeObject::from_bytes(&*code_bytes.borrow_buf()).map_err(|e| match e {
bytecode::CodeDeserializeError::Eof => vm.new_exception_msg(
vm.ctx.exceptions.eof_error.clone(),
"end of file while deserializing bytecode".to_owned(),
),
_ => vm.new_value_error("Couldn't deserialize python bytecode".to_owned()),
})?;
let buf = &*code_bytes.borrow_buf();
let code = bytecode::CodeObject::from_bytes(buf).map_err(|e| match e {
bytecode::CodeDeserializeError::Eof => vm.new_exception_msg(
vm.ctx.exceptions.eof_error.clone(),
"end of file while deserializing bytecode".to_owned(),
),
_ => vm.new_value_error("Couldn't deserialize python bytecode".to_owned()),
})?;
Ok(PyCode {
code: vm.map_codeobj(code),
})

6
vm/src/stdlib/os.rs
View File

@@ -14,7 +14,7 @@ use num_bigint::BigInt;
use strum_macros::EnumString;
use super::errno::errors;
use crate::buffer::PyBufferRef;
use crate::buffer::PyBuffer;
use crate::builtins::bytes::{PyBytes, PyBytesRef};
use crate::builtins::dict::PyDictRef;
use crate::builtins::int;
@@ -213,8 +213,8 @@ pub(crate) fn fspath(
impl TryFromObject for PyPathLike {
fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult<Self> {
// path_converter in CPython
let obj = match PyBufferRef::try_from_borrowed_object(vm, &obj) {
Ok(buffer) => PyBytes::from(Vec::from(&*buffer.obj_bytes())).into_pyobject(vm),
let obj = match PyBuffer::try_from_borrowed_object(vm, &obj) {
Ok(buffer) => PyBytes::from(buffer.internal.obj_bytes().to_vec()).into_pyobject(vm),
Err(_) => obj,
};
let path = fspath(obj, true, vm)?;

4
vm/src/stdlib/sre.rs
View File

@@ -2,7 +2,7 @@ pub(crate) use _sre::make_module;
#[pymodule]
mod _sre {
use crate::buffer::PyBufferRef;
use crate::buffer::PyBuffer;
use crate::builtins::list::PyListRef;
use crate::builtins::tuple::PyTupleRef;
use crate::builtins::{
@@ -152,7 +152,7 @@ mod _sre {
let vec;
let s;
let str_drive = if self.isbytes {
buffer = PyBufferRef::try_from_borrowed_object(vm, &string)?;
buffer = PyBuffer::try_from_borrowed_object(vm, &string)?;
let bytes = match buffer.as_contiguous() {
Some(bytes) => {
guard = bytes;