From e63073bfb8cceee9330779161a49ff6338b6b426 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 30 Nov 2019 15:15:30 -0600 Subject: [PATCH 1/3] Update io module with a bunch of methods --- Lib/io.py | 95 +++++++++ vm/src/stdlib/io.rs | 510 +++++++++++++++++++++++++++++++------------- 2 files changed, 461 insertions(+), 144 deletions(-) diff --git a/Lib/io.py b/Lib/io.py index 5536a308c3..b5d33d5750 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -1 +1,96 @@ +"""The io module provides the Python interfaces to stream handling. The +builtin open function is defined in this module. + +At the top of the I/O hierarchy is the abstract base class IOBase. It +defines the basic interface to a stream. Note, however, that there is no +separation between reading and writing to streams; implementations are +allowed to raise an OSError if they do not support a given operation. + +Extending IOBase is RawIOBase which deals simply with the reading and +writing of raw bytes to a stream. FileIO subclasses RawIOBase to provide +an interface to OS files. + +BufferedIOBase deals with buffering on a raw byte stream (RawIOBase). Its +subclasses, BufferedWriter, BufferedReader, and BufferedRWPair buffer +streams that are readable, writable, and both respectively. +BufferedRandom provides a buffered interface to random access +streams. BytesIO is a simple stream of in-memory bytes. + +Another IOBase subclass, TextIOBase, deals with the encoding and decoding +of streams into text. TextIOWrapper, which extends it, is a buffered text +interface to a buffered raw stream (`BufferedIOBase`). Finally, StringIO +is an in-memory stream for text. + +Argument names are not part of the specification, and only the arguments +of open() are intended to be used as keyword arguments. + +data: + +DEFAULT_BUFFER_SIZE + + An int containing the default buffer size used by the module's buffered + I/O classes. open() uses the file's blksize (as obtained by os.stat) if + possible. +""" +# New I/O library conforming to PEP 3116. + +__author__ = ("Guido van Rossum , " + "Mike Verdone , " + "Mark Russell , " + "Antoine Pitrou , " + "Amaury Forgeot d'Arc , " + "Benjamin Peterson ") + +__all__ = ["BlockingIOError", "open", "IOBase", "RawIOBase", "FileIO", + "BytesIO", "StringIO", "BufferedIOBase", + "BufferedReader", "BufferedWriter", "BufferedRWPair", + "BufferedRandom", "TextIOBase", "TextIOWrapper", + "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END"] + + +import _io +import abc + from _io import * + +OpenWrapper = _io.open # for compatibility with _pyio + +# Pretend this exception was created here. +#UnsupportedOperation.__module__ = "io" + +# for seek() +SEEK_SET = 0 +SEEK_CUR = 1 +SEEK_END = 2 + +# Declaring ABCs in C is tricky so we do it here. +# Method descriptions and default implementations are inherited from the C +# version however. +class IOBase(_io._IOBase, metaclass=abc.ABCMeta): + __doc__ = _io._IOBase.__doc__ + +class RawIOBase(_io._RawIOBase, IOBase): + __doc__ = _io._RawIOBase.__doc__ + +class BufferedIOBase(_io._BufferedIOBase, IOBase): + __doc__ = _io._BufferedIOBase.__doc__ + +class TextIOBase(_io._TextIOBase, IOBase): + __doc__ = _io._TextIOBase.__doc__ + +RawIOBase.register(FileIO) + +for klass in (BytesIO, BufferedReader, BufferedWriter):#, BufferedRandom, + #BufferedRWPair): + BufferedIOBase.register(klass) + +for klass in (StringIO, TextIOWrapper): + TextIOBase.register(klass) +del klass + +try: + from _io import _WindowsConsoleIO +except ImportError: + pass +else: + RawIOBase.register(_WindowsConsoleIO) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index fa3e1a9183..f11baeebc3 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1,7 +1,8 @@ /* * I/O core tools. */ -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; +use std::fs; use std::io::prelude::*; use std::io::Cursor; use std::io::SeekFrom; @@ -10,22 +11,25 @@ use num_traits::ToPrimitive; use super::os; use crate::function::{OptionalArg, OptionalOption, PyFuncArgs}; +use crate::obj::objbool; use crate::obj::objbytearray::PyByteArray; use crate::obj::objbyteinner::PyBytesLike; use crate::obj::objbytes; -use crate::obj::objbytes::PyBytes; -use crate::obj::objint::{self, PyIntRef}; +use crate::obj::objint; +use crate::obj::objiter; use crate::obj::objstr::{self, PyStringRef}; -use crate::obj::objtype; -use crate::obj::objtype::PyClassRef; -use crate::pyobject::TypeProtocol; -use crate::pyobject::{BufferProtocol, Either, PyObjectRef, PyRef, PyResult, PyValue}; +use crate::obj::objtype::{self, PyClassRef}; +use crate::pyobject::{ + BufferProtocol, Either, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, +}; use crate::vm::VirtualMachine; fn byte_count(bytes: OptionalOption) -> i64 { bytes.flat_option().unwrap_or(-1 as i64) } +const DEFAULT_BUFFER_SIZE: usize = 8 * 1024; + #[derive(Debug)] struct BufferedIO { cursor: Cursor>, @@ -100,6 +104,7 @@ impl BufferedIO { #[derive(Debug)] struct PyStringIO { buffer: RefCell, + closed: Cell, } type PyStringIORef = PyRef; @@ -167,14 +172,34 @@ impl PyStringIORef { } } - fn close(self, _vm: &VirtualMachine) { - // TODO: discard the text buffer on close + fn truncate(self, size: OptionalOption, _vm: &VirtualMachine) { + let mut buffer = self.buffer.borrow_mut(); + let size = size.flat_option().unwrap_or_else(|| buffer.tell() as usize); + buffer.cursor.get_mut().truncate(size); } + + fn closed(self, _vm: &VirtualMachine) -> bool { + self.closed.get() + } + + fn close(self, _vm: &VirtualMachine) { + self.buffer.borrow_mut().cursor.get_mut().clear(); + self.closed.set(true); + } +} + +#[derive(FromArgs)] +struct StringIOArgs { + #[pyarg(positional_or_keyword, default = "None")] + #[allow(dead_code)] + // TODO: use this + newline: Option, } fn string_io_new( cls: PyClassRef, object: OptionalArg>, + _args: StringIOArgs, vm: &VirtualMachine, ) -> PyResult { let raw_string = match object { @@ -184,6 +209,7 @@ fn string_io_new( PyStringIO { buffer: RefCell::new(BufferedIO::new(Cursor::new(raw_string.into_bytes()))), + closed: Cell::new(false), } .into_ref_with_type(vm, cls) } @@ -191,6 +217,7 @@ fn string_io_new( #[derive(Debug)] struct PyBytesIO { buffer: RefCell, + closed: Cell, } type PyBytesIORef = PyRef; @@ -247,6 +274,15 @@ impl PyBytesIORef { None => Err(vm.new_value_error("Error Performing Operation".to_string())), } } + + fn closed(self, _vm: &VirtualMachine) -> bool { + self.closed.get() + } + + fn close(self, _vm: &VirtualMachine) { + self.buffer.borrow_mut().cursor.get_mut().clear(); + self.closed.set(true); + } } fn bytes_io_new( @@ -261,6 +297,7 @@ fn bytes_io_new( PyBytesIO { buffer: RefCell::new(BufferedIO::new(Cursor::new(raw_bytes))), + closed: Cell::new(false), } .into_ref_with_type(vm, cls) } @@ -269,7 +306,10 @@ fn io_base_cm_enter(instance: PyObjectRef, _vm: &VirtualMachine) -> PyObjectRef instance.clone() } -fn io_base_cm_exit(_args: PyFuncArgs, _vm: &VirtualMachine) {} +fn io_base_cm_exit(instance: PyObjectRef, _args: PyFuncArgs, vm: &VirtualMachine) -> PyResult<()> { + vm.call_method(&instance, "close", vec![])?; + Ok(()) +} // TODO Check if closed, then if so raise ValueError fn io_base_flush(_self: PyObjectRef, _vm: &VirtualMachine) {} @@ -277,49 +317,189 @@ fn io_base_flush(_self: PyObjectRef, _vm: &VirtualMachine) {} fn io_base_seekable(_self: PyObjectRef, _vm: &VirtualMachine) -> bool { false } +fn io_base_readable(_self: PyObjectRef, _vm: &VirtualMachine) -> bool { + false +} +fn io_base_writable(_self: PyObjectRef, _vm: &VirtualMachine) -> bool { + false +} + +fn io_base_closed(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm.get_attribute(instance, "__closed") +} + +fn io_base_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let closed = objbool::boolval(vm, io_base_closed(instance.clone(), vm)?)?; + if !closed { + let res = vm.call_method(&instance, "flush", vec![]); + vm.set_attr(&instance, "__closed", vm.ctx.new_bool(true))?; + res?; + } + Ok(()) +} + +fn io_base_readline( + instance: PyObjectRef, + size: OptionalOption, + vm: &VirtualMachine, +) -> PyResult> { + let size = byte_count(size); + let mut res = Vec::::new(); + let read = vm.get_attribute(instance, "read")?; + while size < 0 || res.len() < size as usize { + let read_res = PyBytesLike::try_from_object(vm, vm.invoke(&read, vec![vm.new_int(1)])?)?; + let b = read_res.to_cow(); + if b.is_empty() { + break; + } + res.extend_from_slice(b.as_ref()); + if res.ends_with(b"\n") { + break; + } + } + Ok(res) +} + +fn io_base_checkclosed( + instance: PyObjectRef, + msg: OptionalOption, + vm: &VirtualMachine, +) -> PyResult<()> { + if objbool::boolval(vm, vm.get_attribute(instance, "closed")?)? { + let msg = msg + .flat_option() + .unwrap_or_else(|| vm.new_str("I/O operation on closed file.".to_string())); + Err(vm.new_exception_obj(vm.ctx.exceptions.value_error.clone(), vec![msg])?) + } else { + Ok(()) + } +} + +fn io_base_checkreadable( + instance: PyObjectRef, + msg: OptionalOption, + vm: &VirtualMachine, +) -> PyResult<()> { + if !objbool::boolval(vm, vm.call_method(&instance, "readable", vec![])?)? { + let msg = msg + .flat_option() + .unwrap_or_else(|| vm.new_str("File or stream is not readable.".to_string())); + Err(vm.new_exception_obj(vm.ctx.exceptions.value_error.clone(), vec![msg])?) + } else { + Ok(()) + } +} + +fn io_base_checkwritable( + instance: PyObjectRef, + msg: OptionalOption, + vm: &VirtualMachine, +) -> PyResult<()> { + if !objbool::boolval(vm, vm.call_method(&instance, "writable", vec![])?)? { + let msg = msg + .flat_option() + .unwrap_or_else(|| vm.new_str("File or stream is not writable.".to_string())); + Err(vm.new_exception_obj(vm.ctx.exceptions.value_error.clone(), vec![msg])?) + } else { + Ok(()) + } +} + +fn io_base_checkseekable( + instance: PyObjectRef, + msg: OptionalOption, + vm: &VirtualMachine, +) -> PyResult<()> { + if !objbool::boolval(vm, vm.call_method(&instance, "seekable", vec![])?)? { + let msg = msg + .flat_option() + .unwrap_or_else(|| vm.new_str("File or stream is not seekable.".to_string())); + Err(vm.new_exception_obj(vm.ctx.exceptions.value_error.clone(), vec![msg])?) + } else { + Ok(()) + } +} + +fn io_base_iter(instance: PyObjectRef, _vm: &VirtualMachine) -> PyObjectRef { + instance +} +fn io_base_next(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let line = vm.call_method(&instance, "readline", vec![])?; + if !objbool::boolval(vm, line.clone())? { + Err(objiter::new_stop_iteration(vm)) + } else { + Ok(line) + } +} +fn io_base_readlines(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { + Ok(vm.ctx.new_list(vm.extract_elements(&instance)?)) +} + +fn raw_io_base_read( + instance: PyObjectRef, + size: OptionalOption, + vm: &VirtualMachine, +) -> PyResult { + let size = byte_count(size); + if size < 0 { + return vm.call_method(&instance, "readall", vec![]); + } + let b = PyByteArray::new(vec![0; size as usize]).into_ref(vm); + let n = >::try_from_object( + vm, + vm.call_method(&instance, "readinto", vec![b.as_object().clone()])?, + )?; + if let Some(n) = n { + let bytes = &mut b.inner.borrow_mut().elements; + bytes.truncate(n); + Ok(vm.ctx.new_bytes(bytes.clone())) + } else { + Ok(vm.get_none()) + } +} fn buffered_io_base_init( instance: PyObjectRef, raw: PyObjectRef, + buffer_size: OptionalArg, vm: &VirtualMachine, ) -> PyResult<()> { vm.set_attr(&instance, "raw", raw.clone())?; + vm.set_attr( + &instance, + "buffer_size", + vm.new_int(buffer_size.unwrap_or(DEFAULT_BUFFER_SIZE)), + )?; Ok(()) } -fn buffered_reader_read(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult> { - let buff_size = 8 * 1024; - let buffer = vm.ctx.new_bytearray(vec![0; buff_size]); +fn buffered_io_base_fileno(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let raw = vm.get_attribute(instance, "raw")?; + vm.call_method(&raw, "fileno", vec![]) +} - //buffer method - let mut result = vec![]; - let mut length = buff_size; - - let raw = vm.get_attribute(instance.clone(), "raw").unwrap(); - - //Iterates through the raw class, invoking the readinto method - //to obtain buff_size many bytes. Exit when less than buff_size many - //bytes are returned (when the end of the file is reached). - while length == buff_size { - vm.call_method(&raw, "readinto", vec![buffer.clone()]) - .map_err(|_| vm.new_value_error("IO Error".to_string()))?; - - //Copy bytes from the buffer vector into the results vector - if let Some(bytes) = buffer.payload::() { - result.extend_from_slice(&bytes.inner.borrow().elements); - }; - - let py_len = vm.call_method(&buffer, "__len__", PyFuncArgs::default())?; - length = objint::get_value(&py_len).to_usize().unwrap(); - } - - Ok(result) +fn buffered_reader_read( + instance: PyObjectRef, + size: OptionalOption, + vm: &VirtualMachine, +) -> PyResult { + vm.call_method( + &vm.get_attribute(instance.clone(), "raw")?, + "read", + vec![vm.new_int(byte_count(size))], + ) } fn buffered_reader_seekable(_self: PyObjectRef, _vm: &VirtualMachine) -> bool { true } +fn buffered_reader_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let raw = vm.get_attribute(instance, "raw")?; + vm.invoke(&vm.get_attribute(raw, "close")?, vec![])?; + Ok(()) +} + fn compute_c_flag(mode: &str) -> u32 { let flag = match mode.chars().next() { Some(mode) => match mode { @@ -336,63 +516,72 @@ fn compute_c_flag(mode: &str) -> u32 { fn file_io_init( file_io: PyObjectRef, - name: Either, + name: Either, mode: OptionalArg, vm: &VirtualMachine, ) -> PyResult { - let file_no = match &name { + let (name, file_no) = match name { Either::A(name) => { let mode = match mode { OptionalArg::Present(mode) => compute_c_flag(mode.as_str()), OptionalArg::Missing => libc::O_RDONLY as _, }; - let fno = os::os_open( - name.clone(), - mode as _, - OptionalArg::Missing, - OptionalArg::Missing, - vm, - )?; - vm.new_int(fno) + ( + name.clone().into_object(), + os::os_open( + name, + mode as _, + OptionalArg::Missing, + OptionalArg::Missing, + vm, + )?, + ) } - Either::B(fno) => fno.clone().into_object(), + Either::B(fno) => (vm.new_int(fno), fno), }; - vm.set_attr(&file_io, "name", name.into_object())?; - vm.set_attr(&file_io, "fileno", file_no)?; + vm.set_attr(&file_io, "name", name)?; + vm.set_attr(&file_io, "__fileno", vm.new_int(file_no))?; vm.set_attr(&file_io, "closefd", vm.new_bool(false))?; vm.set_attr(&file_io, "closed", vm.new_bool(false))?; Ok(vm.get_none()) } +fn fio_get_fileno(instance: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + io_base_checkclosed(instance.clone(), OptionalArg::Missing, vm)?; + let fileno = i64::try_from_object(vm, vm.get_attribute(instance.clone(), "__fileno")?)?; + Ok(os::rust_file(fileno)) +} +fn fio_set_fileno(instance: &PyObjectRef, f: fs::File, vm: &VirtualMachine) -> PyResult<()> { + let updated = os::raw_file_number(f); + vm.set_attr(&instance, "__fileno", vm.ctx.new_int(updated))?; + Ok(()) +} + fn file_io_read( instance: PyObjectRef, - read_byte: OptionalArg, + read_byte: OptionalOption, vm: &VirtualMachine, ) -> PyResult> { - let file_no = vm.get_attribute(instance.clone(), "fileno")?; - let raw_fd = objint::get_value(&file_no).to_i64().unwrap(); + let read_byte = byte_count(read_byte); - let mut handle = os::rust_file(raw_fd); + let mut handle = fio_get_fileno(&instance, vm)?; - let bytes = match read_byte { - OptionalArg::Missing => { - let mut bytes = vec![]; - handle - .read_to_end(&mut bytes) - .map_err(|_| vm.new_value_error("Error reading from Buffer".to_string()))?; - bytes - } - OptionalArg::Present(read_byte) => { - let mut bytes = vec![0; read_byte]; - handle - .read_exact(&mut bytes) - .map_err(|_| vm.new_value_error("Error reading from Buffer".to_string()))?; - let updated = os::raw_file_number(handle); - vm.set_attr(&instance, "fileno", vm.ctx.new_int(updated))?; - bytes - } + let bytes = if read_byte < 0 { + let mut bytes = vec![]; + handle + .read_to_end(&mut bytes) + .map_err(|e| os::convert_io_error(vm, e))?; + bytes + } else { + let mut bytes = vec![0; read_byte as usize]; + let n = handle + .read(&mut bytes) + .map_err(|e| os::convert_io_error(vm, e))?; + bytes.truncate(n); + bytes }; + fio_set_fileno(&instance, handle, vm)?; Ok(bytes) } @@ -408,11 +597,7 @@ fn file_io_readinto(instance: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine let py_length = vm.call_method(&obj, "__len__", PyFuncArgs::default())?; let length = objint::get_value(&py_length).to_u64().unwrap(); - let file_no = vm.get_attribute(instance.clone(), "fileno")?; - let raw_fd = objint::get_value(&file_no).to_i64().unwrap(); - - //extract unix file descriptor. - let handle = os::rust_file(raw_fd); + let handle = fio_get_fileno(&instance, vm)?; let mut f = handle.take(length); if let Some(bytes) = obj.payload::() { @@ -426,51 +611,29 @@ fn file_io_readinto(instance: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine } }; - let updated = os::raw_file_number(f.into_inner()); - vm.set_attr(&instance, "fileno", vm.ctx.new_int(updated))?; + fio_set_fileno(&instance, f.into_inner(), vm)?; + Ok(()) } -fn file_io_write(instance: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let file_no = vm.get_attribute(instance.clone(), "fileno")?; - let raw_fd = objint::get_value(&file_no).to_i64().unwrap(); +fn file_io_write(instance: PyObjectRef, obj: PyBytesLike, vm: &VirtualMachine) -> PyResult { + let mut handle = fio_get_fileno(&instance, vm)?; - //unsafe block - creates file handle from the UNIX file descriptor - //raw_fd is supported on UNIX only. This will need to be extended - //to support windows - i.e. raw file_handles - let mut handle = os::rust_file(raw_fd); + let len = handle + .write(obj.to_cow().as_ref()) + .map_err(|e| os::convert_io_error(vm, e))?; - let bytes = match_class!(match obj.clone() { - i @ PyBytes => Ok(i.get_value().to_vec()), - j @ PyByteArray => Ok(j.inner.borrow().elements.to_vec()), - obj => Err(vm.new_type_error(format!( - "a bytes-like object is required, not {}", - obj.class() - ))), - }); + fio_set_fileno(&instance, handle, vm)?; - match handle.write(&bytes?) { - Ok(len) => { - //reset raw fd on the FileIO object - let updated = os::raw_file_number(handle); - vm.set_attr(&instance, "fileno", vm.ctx.new_int(updated))?; - - //return number of bytes written - Ok(len) - } - Err(_) => Err(vm.new_value_error("Error Writing Bytes to Handle".to_string())), - } + //return number of bytes written + Ok(len) } #[cfg(windows)] fn file_io_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - use std::os::windows::io::IntoRawHandle; - let file_no = vm.get_attribute(instance.clone(), "fileno")?; - let raw_fd = objint::get_value(&file_no).to_i64().unwrap(); - let handle = os::rust_file(raw_fd); - let raw_handle = handle.into_raw_handle(); + let raw_handle = i64::try_from_object(vm, vm.get_attribute(instance.clone(), "__fileno")?)?; unsafe { - winapi::um::handleapi::CloseHandle(raw_handle); + winapi::um::handleapi::CloseHandle(raw_handle as _); } vm.set_attr(&instance, "closefd", vm.new_bool(true))?; vm.set_attr(&instance, "closed", vm.new_bool(true))?; @@ -479,10 +642,9 @@ fn file_io_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { #[cfg(unix)] fn file_io_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let file_no = vm.get_attribute(instance.clone(), "fileno")?; - let raw_fd = objint::get_value(&file_no).to_i32().unwrap(); + let raw_fd = i64::try_from_object(vm, vm.get_attribute(instance.clone(), "__fileno")?)?; unsafe { - libc::close(raw_fd); + libc::close(raw_fd as _); } vm.set_attr(&instance, "closefd", vm.new_bool(true))?; vm.set_attr(&instance, "closed", vm.new_bool(true))?; @@ -493,6 +655,10 @@ fn file_io_seekable(_self: PyObjectRef, _vm: &VirtualMachine) -> bool { true } +fn file_io_fileno(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm.get_attribute(instance, "__fileno") +} + fn buffered_writer_write(instance: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { let raw = vm.get_attribute(instance, "raw").unwrap(); @@ -517,7 +683,11 @@ fn text_io_wrapper_seekable(_self: PyObjectRef, _vm: &VirtualMachine) -> bool { true } -fn text_io_base_read(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { +fn text_io_wrapper_read( + instance: PyObjectRef, + size: OptionalOption, + vm: &VirtualMachine, +) -> PyResult { let buffered_reader_class = vm.try_class("_io", "BufferedReader")?; let raw = vm.get_attribute(instance.clone(), "buffer").unwrap(); @@ -526,23 +696,23 @@ fn text_io_base_read(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult, + vm: &VirtualMachine, +) -> PyResult { + let buffered_reader_class = vm.try_class("_io", "BufferedReader")?; + let raw = vm.get_attribute(instance.clone(), "buffer").unwrap(); + + if !objtype::isinstance(&raw, &buffered_reader_class) { + // TODO: this should be io.UnsupportedOperation error which derives both from ValueError *and* OSError + return Err(vm.new_value_error("not readable".to_string())); + } + + let bytes = vm.call_method( + &raw, + "readline", + vec![size.flat_option().unwrap_or_else(|| vm.get_none())], + )?; + let bytes = PyBytesLike::try_from_object(vm, bytes)?; + //format bytes into string + let rust_string = String::from_utf8(bytes.to_cow().into_owned()).map_err(|e| { + vm.new_unicode_decode_error(format!( + "cannot decode byte at index: {}", + e.utf8_error().valid_up_to() + )) + })?; + Ok(rust_string) +} + fn split_mode_string(mode_string: String) -> Result<(String, String), String> { let mut mode: char = '\0'; let mut typ: char = '\0'; @@ -698,23 +897,35 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; //IOBase the abstract base class of the IO Module - let io_base = py_class!(ctx, "IOBase", ctx.object(), { + let io_base = py_class!(ctx, "_IOBase", ctx.object(), { "__enter__" => ctx.new_rustfunc(io_base_cm_enter), "__exit__" => ctx.new_rustfunc(io_base_cm_exit), "seekable" => ctx.new_rustfunc(io_base_seekable), - "flush" => ctx.new_rustfunc(io_base_flush) + "readable" => ctx.new_rustfunc(io_base_readable), + "writable" => ctx.new_rustfunc(io_base_writable), + "flush" => ctx.new_rustfunc(io_base_flush), + "closed" => ctx.new_property(io_base_closed), + "__closed" => ctx.new_bool(false), + "close" => ctx.new_rustfunc(io_base_close), + "readline" => ctx.new_rustfunc(io_base_readline), + "_checkClosed" => ctx.new_rustfunc(io_base_checkclosed), + "_checkReadable" => ctx.new_rustfunc(io_base_checkreadable), + "_checkWritable" => ctx.new_rustfunc(io_base_checkwritable), + "_checkSeekable" => ctx.new_rustfunc(io_base_checkseekable), + "__iter__" => ctx.new_rustfunc(io_base_iter), + "__next__" => ctx.new_rustfunc(io_base_next), + "readlines" => ctx.new_rustfunc(io_base_readlines), }); // IOBase Subclasses - let raw_io_base = py_class!(ctx, "RawIOBase", io_base.clone(), {}); + let raw_io_base = py_class!(ctx, "_RawIOBase", io_base.clone(), { + "read" => ctx.new_rustfunc(raw_io_base_read), + }); - let buffered_io_base = py_class!(ctx, "BufferedIOBase", io_base.clone(), {}); + let buffered_io_base = py_class!(ctx, "_BufferedIOBase", io_base.clone(), {}); //TextIO Base has no public constructor - let text_io_base = py_class!(ctx, "TextIOBase", io_base.clone(), { - "read" => ctx.new_rustfunc(text_io_base_read), - "write" => ctx.new_rustfunc(text_io_base_write) - }); + let text_io_base = py_class!(ctx, "_TextIOBase", io_base.clone(), {}); // RawBaseIO Subclasses // TODO Fix name? @@ -725,7 +936,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "readinto" => ctx.new_rustfunc(file_io_readinto), "write" => ctx.new_rustfunc(file_io_write), "close" => ctx.new_rustfunc(file_io_close), - "seekable" => ctx.new_rustfunc(file_io_seekable) + "seekable" => ctx.new_rustfunc(file_io_seekable), + "fileno" => ctx.new_rustfunc(file_io_fileno), }); // BufferedIOBase Subclasses @@ -735,7 +947,9 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { //For more info see: https://github.com/RustPython/RustPython/issues/547 "__init__" => ctx.new_rustfunc(buffered_io_base_init), "read" => ctx.new_rustfunc(buffered_reader_read), - "seekable" => ctx.new_rustfunc(buffered_reader_seekable) + "seekable" => ctx.new_rustfunc(buffered_reader_seekable), + "close" => ctx.new_rustfunc(buffered_reader_close), + "fileno" => ctx.new_rustfunc(buffered_io_base_fileno), }); let buffered_writer = py_class!(ctx, "BufferedWriter", buffered_io_base.clone(), { @@ -744,13 +958,17 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { //For more info see: https://github.com/RustPython/RustPython/issues/547 "__init__" => ctx.new_rustfunc(buffered_io_base_init), "write" => ctx.new_rustfunc(buffered_writer_write), - "seekable" => ctx.new_rustfunc(buffered_writer_seekable) + "seekable" => ctx.new_rustfunc(buffered_writer_seekable), + "fileno" => ctx.new_rustfunc(buffered_io_base_fileno), }); //TextIOBase Subclass let text_io_wrapper = py_class!(ctx, "TextIOWrapper", text_io_base.clone(), { "__init__" => ctx.new_rustfunc(text_io_wrapper_init), - "seekable" => ctx.new_rustfunc(text_io_wrapper_seekable) + "seekable" => ctx.new_rustfunc(text_io_wrapper_seekable), + "read" => ctx.new_rustfunc(text_io_wrapper_read), + "write" => ctx.new_rustfunc(text_io_wrapper_write), + "readline" => ctx.new_rustfunc(text_io_wrapper_readline), }); //StringIO: in-memory text @@ -763,6 +981,8 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "getvalue" => ctx.new_rustfunc(PyStringIORef::getvalue), "tell" => ctx.new_rustfunc(PyStringIORef::tell), "readline" => ctx.new_rustfunc(PyStringIORef::readline), + "truncate" => ctx.new_rustfunc(PyStringIORef::truncate), + "closed" => ctx.new_property(PyStringIORef::closed), "close" => ctx.new_rustfunc(PyStringIORef::close), }); @@ -777,14 +997,16 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "getvalue" => ctx.new_rustfunc(PyBytesIORef::getvalue), "tell" => ctx.new_rustfunc(PyBytesIORef::tell), "readline" => ctx.new_rustfunc(PyBytesIORef::readline), + "closed" => ctx.new_property(PyBytesIORef::closed), + "close" => ctx.new_rustfunc(PyBytesIORef::close), }); py_module!(vm, "_io", { "open" => ctx.new_rustfunc(io_open), - "IOBase" => io_base, - "RawIOBase" => raw_io_base, - "BufferedIOBase" => buffered_io_base, - "TextIOBase" => text_io_base, + "_IOBase" => io_base, + "_RawIOBase" => raw_io_base, + "_BufferedIOBase" => buffered_io_base, + "_TextIOBase" => text_io_base, "FileIO" => file_io, "BufferedReader" => buffered_reader, "BufferedWriter" => buffered_writer, From 4263ce48e3bdd599fb21760ac073d51844fe62e2 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 1 Dec 2019 16:57:27 -0600 Subject: [PATCH 2/3] Include _io on WASM, just not FileIO --- Lib/io.py | 5 +- vm/src/stdlib/io.rs | 339 +++++++++++++++++++++++-------------------- vm/src/stdlib/mod.rs | 5 +- 3 files changed, 188 insertions(+), 161 deletions(-) diff --git a/Lib/io.py b/Lib/io.py index b5d33d5750..5f11c8c016 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -78,7 +78,10 @@ class BufferedIOBase(_io._BufferedIOBase, IOBase): class TextIOBase(_io._TextIOBase, IOBase): __doc__ = _io._TextIOBase.__doc__ -RawIOBase.register(FileIO) +try: + RawIOBase.register(FileIO) +except NameError: + pass for klass in (BytesIO, BufferedReader, BufferedWriter):#, BufferedRandom, #BufferedRWPair): diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index f11baeebc3..5c9bc1b847 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -9,7 +9,6 @@ use std::io::SeekFrom; use num_traits::ToPrimitive; -use super::os; use crate::function::{OptionalArg, OptionalOption, PyFuncArgs}; use crate::obj::objbool; use crate::obj::objbytearray::PyByteArray; @@ -500,163 +499,191 @@ fn buffered_reader_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult Ok(()) } -fn compute_c_flag(mode: &str) -> u32 { - let flag = match mode.chars().next() { - Some(mode) => match mode { - 'w' => libc::O_WRONLY | libc::O_CREAT, - 'x' => libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL, - 'a' => libc::O_APPEND, - '+' => libc::O_RDWR, - _ => libc::O_RDONLY, - }, - None => libc::O_RDONLY, - }; - flag as u32 -} +// disable FileIO on WASM +#[cfg(not(target_arch = "wasm32"))] +mod fileio { + use super::super::os; + use super::*; -fn file_io_init( - file_io: PyObjectRef, - name: Either, - mode: OptionalArg, - vm: &VirtualMachine, -) -> PyResult { - let (name, file_no) = match name { - Either::A(name) => { - let mode = match mode { - OptionalArg::Present(mode) => compute_c_flag(mode.as_str()), - OptionalArg::Missing => libc::O_RDONLY as _, - }; - ( - name.clone().into_object(), - os::os_open( - name, - mode as _, - OptionalArg::Missing, - OptionalArg::Missing, - vm, - )?, - ) + fn compute_c_flag(mode: &str) -> u32 { + let flag = match mode.chars().next() { + Some(mode) => match mode { + 'w' => libc::O_WRONLY | libc::O_CREAT, + 'x' => libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL, + 'a' => libc::O_APPEND, + '+' => libc::O_RDWR, + _ => libc::O_RDONLY, + }, + None => libc::O_RDONLY, + }; + flag as u32 + } + + fn file_io_init( + file_io: PyObjectRef, + name: Either, + mode: OptionalArg, + vm: &VirtualMachine, + ) -> PyResult { + let (name, file_no) = match name { + Either::A(name) => { + let mode = match mode { + OptionalArg::Present(mode) => compute_c_flag(mode.as_str()), + OptionalArg::Missing => libc::O_RDONLY as _, + }; + ( + name.clone().into_object(), + os::os_open( + name, + mode as _, + OptionalArg::Missing, + OptionalArg::Missing, + vm, + )?, + ) + } + Either::B(fno) => (vm.new_int(fno), fno), + }; + + vm.set_attr(&file_io, "name", name)?; + vm.set_attr(&file_io, "__fileno", vm.new_int(file_no))?; + vm.set_attr(&file_io, "closefd", vm.new_bool(false))?; + vm.set_attr(&file_io, "closed", vm.new_bool(false))?; + Ok(vm.get_none()) + } + + fn fio_get_fileno(instance: &PyObjectRef, vm: &VirtualMachine) -> PyResult { + io_base_checkclosed(instance.clone(), OptionalArg::Missing, vm)?; + let fileno = i64::try_from_object(vm, vm.get_attribute(instance.clone(), "__fileno")?)?; + Ok(os::rust_file(fileno)) + } + fn fio_set_fileno(instance: &PyObjectRef, f: fs::File, vm: &VirtualMachine) -> PyResult<()> { + let updated = os::raw_file_number(f); + vm.set_attr(&instance, "__fileno", vm.ctx.new_int(updated))?; + Ok(()) + } + + fn file_io_read( + instance: PyObjectRef, + read_byte: OptionalOption, + vm: &VirtualMachine, + ) -> PyResult> { + let read_byte = byte_count(read_byte); + + let mut handle = fio_get_fileno(&instance, vm)?; + + let bytes = if read_byte < 0 { + let mut bytes = vec![]; + handle + .read_to_end(&mut bytes) + .map_err(|e| os::convert_io_error(vm, e))?; + bytes + } else { + let mut bytes = vec![0; read_byte as usize]; + let n = handle + .read(&mut bytes) + .map_err(|e| os::convert_io_error(vm, e))?; + bytes.truncate(n); + bytes + }; + fio_set_fileno(&instance, handle, vm)?; + + Ok(bytes) + } + + fn file_io_readinto( + instance: PyObjectRef, + obj: PyObjectRef, + vm: &VirtualMachine, + ) -> PyResult<()> { + if !obj.readonly() { + return Err(vm.new_type_error( + "readinto() argument must be read-write bytes-like object".to_string(), + )); } - Either::B(fno) => (vm.new_int(fno), fno), - }; - vm.set_attr(&file_io, "name", name)?; - vm.set_attr(&file_io, "__fileno", vm.new_int(file_no))?; - vm.set_attr(&file_io, "closefd", vm.new_bool(false))?; - vm.set_attr(&file_io, "closed", vm.new_bool(false))?; - Ok(vm.get_none()) -} + //extract length of buffer + let py_length = vm.call_method(&obj, "__len__", PyFuncArgs::default())?; + let length = objint::get_value(&py_length).to_u64().unwrap(); -fn fio_get_fileno(instance: &PyObjectRef, vm: &VirtualMachine) -> PyResult { - io_base_checkclosed(instance.clone(), OptionalArg::Missing, vm)?; - let fileno = i64::try_from_object(vm, vm.get_attribute(instance.clone(), "__fileno")?)?; - Ok(os::rust_file(fileno)) -} -fn fio_set_fileno(instance: &PyObjectRef, f: fs::File, vm: &VirtualMachine) -> PyResult<()> { - let updated = os::raw_file_number(f); - vm.set_attr(&instance, "__fileno", vm.ctx.new_int(updated))?; - Ok(()) -} + let handle = fio_get_fileno(&instance, vm)?; -fn file_io_read( - instance: PyObjectRef, - read_byte: OptionalOption, - vm: &VirtualMachine, -) -> PyResult> { - let read_byte = byte_count(read_byte); + let mut f = handle.take(length); + if let Some(bytes) = obj.payload::() { + //TODO: Implement for MemoryView - let mut handle = fio_get_fileno(&instance, vm)?; + let value_mut = &mut bytes.inner.borrow_mut().elements; + value_mut.clear(); + match f.read_to_end(value_mut) { + Ok(_) => {} + Err(_) => return Err(vm.new_value_error("Error reading from Take".to_string())), + } + }; - let bytes = if read_byte < 0 { - let mut bytes = vec![]; - handle - .read_to_end(&mut bytes) - .map_err(|e| os::convert_io_error(vm, e))?; - bytes - } else { - let mut bytes = vec![0; read_byte as usize]; - let n = handle - .read(&mut bytes) - .map_err(|e| os::convert_io_error(vm, e))?; - bytes.truncate(n); - bytes - }; - fio_set_fileno(&instance, handle, vm)?; + fio_set_fileno(&instance, f.into_inner(), vm)?; - Ok(bytes) -} - -fn file_io_readinto(instance: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - if !obj.readonly() { - return Err(vm.new_type_error( - "readinto() argument must be read-write bytes-like object".to_string(), - )); + Ok(()) } - //extract length of buffer - let py_length = vm.call_method(&obj, "__len__", PyFuncArgs::default())?; - let length = objint::get_value(&py_length).to_u64().unwrap(); + fn file_io_write( + instance: PyObjectRef, + obj: PyBytesLike, + vm: &VirtualMachine, + ) -> PyResult { + let mut handle = fio_get_fileno(&instance, vm)?; - let handle = fio_get_fileno(&instance, vm)?; + let len = handle + .write(obj.to_cow().as_ref()) + .map_err(|e| os::convert_io_error(vm, e))?; - let mut f = handle.take(length); - if let Some(bytes) = obj.payload::() { - //TODO: Implement for MemoryView + fio_set_fileno(&instance, handle, vm)?; - let value_mut = &mut bytes.inner.borrow_mut().elements; - value_mut.clear(); - match f.read_to_end(value_mut) { - Ok(_) => {} - Err(_) => return Err(vm.new_value_error("Error reading from Take".to_string())), + //return number of bytes written + Ok(len) + } + + #[cfg(windows)] + fn file_io_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let raw_handle = i64::try_from_object(vm, vm.get_attribute(instance.clone(), "__fileno")?)?; + unsafe { + winapi::um::handleapi::CloseHandle(raw_handle as _); } - }; - - fio_set_fileno(&instance, f.into_inner(), vm)?; - - Ok(()) -} - -fn file_io_write(instance: PyObjectRef, obj: PyBytesLike, vm: &VirtualMachine) -> PyResult { - let mut handle = fio_get_fileno(&instance, vm)?; - - let len = handle - .write(obj.to_cow().as_ref()) - .map_err(|e| os::convert_io_error(vm, e))?; - - fio_set_fileno(&instance, handle, vm)?; - - //return number of bytes written - Ok(len) -} - -#[cfg(windows)] -fn file_io_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let raw_handle = i64::try_from_object(vm, vm.get_attribute(instance.clone(), "__fileno")?)?; - unsafe { - winapi::um::handleapi::CloseHandle(raw_handle as _); + vm.set_attr(&instance, "closefd", vm.new_bool(true))?; + vm.set_attr(&instance, "closed", vm.new_bool(true))?; + Ok(()) } - vm.set_attr(&instance, "closefd", vm.new_bool(true))?; - vm.set_attr(&instance, "closed", vm.new_bool(true))?; - Ok(()) -} -#[cfg(unix)] -fn file_io_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { - let raw_fd = i64::try_from_object(vm, vm.get_attribute(instance.clone(), "__fileno")?)?; - unsafe { - libc::close(raw_fd as _); + #[cfg(unix)] + fn file_io_close(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult<()> { + let raw_fd = i64::try_from_object(vm, vm.get_attribute(instance.clone(), "__fileno")?)?; + unsafe { + libc::close(raw_fd as _); + } + vm.set_attr(&instance, "closefd", vm.new_bool(true))?; + vm.set_attr(&instance, "closed", vm.new_bool(true))?; + Ok(()) } - vm.set_attr(&instance, "closefd", vm.new_bool(true))?; - vm.set_attr(&instance, "closed", vm.new_bool(true))?; - Ok(()) -} -fn file_io_seekable(_self: PyObjectRef, _vm: &VirtualMachine) -> bool { - true -} + fn file_io_seekable(_self: PyObjectRef, _vm: &VirtualMachine) -> bool { + true + } -fn file_io_fileno(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { - vm.get_attribute(instance, "__fileno") + fn file_io_fileno(instance: PyObjectRef, vm: &VirtualMachine) -> PyResult { + vm.get_attribute(instance, "__fileno") + } + + pub fn make_fileio(ctx: &crate::pyobject::PyContext, raw_io_base: PyClassRef) -> PyClassRef { + py_class!(ctx, "FileIO", raw_io_base, { + "__init__" => ctx.new_rustfunc(file_io_init), + "name" => ctx.str_type(), + "read" => ctx.new_rustfunc(file_io_read), + "readinto" => ctx.new_rustfunc(file_io_readinto), + "write" => ctx.new_rustfunc(file_io_write), + "close" => ctx.new_rustfunc(file_io_close), + "seekable" => ctx.new_rustfunc(file_io_seekable), + "fileno" => ctx.new_rustfunc(file_io_fileno), + }) + } } fn buffered_writer_write(instance: PyObjectRef, obj: PyObjectRef, vm: &VirtualMachine) -> PyResult { @@ -850,7 +877,12 @@ pub fn io_open(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // Construct a FileIO (subclass of RawIOBase) // This is subsequently consumed by a Buffered Class. - let file_io_class = vm.get_attribute(io_module.clone(), "FileIO").unwrap(); + let file_io_class = vm.get_attribute(io_module.clone(), "FileIO").map_err(|_| { + // TODO: UnsupportedOperation here + vm.new_os_error( + "Couldn't get FileIO, io.open likely isn't supported on your platform".to_string(), + ) + })?; let file_io_obj = vm.invoke( &file_io_class, vec![file.clone(), vm.ctx.new_str(mode.clone())], @@ -927,19 +959,6 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { //TextIO Base has no public constructor let text_io_base = py_class!(ctx, "_TextIOBase", io_base.clone(), {}); - // RawBaseIO Subclasses - // TODO Fix name? - let file_io = py_class!(ctx, "FileIO", raw_io_base.clone(), { - "__init__" => ctx.new_rustfunc(file_io_init), - "name" => ctx.str_type(), - "read" => ctx.new_rustfunc(file_io_read), - "readinto" => ctx.new_rustfunc(file_io_readinto), - "write" => ctx.new_rustfunc(file_io_write), - "close" => ctx.new_rustfunc(file_io_close), - "seekable" => ctx.new_rustfunc(file_io_seekable), - "fileno" => ctx.new_rustfunc(file_io_fileno), - }); - // BufferedIOBase Subclasses let buffered_reader = py_class!(ctx, "BufferedReader", buffered_io_base.clone(), { //workaround till the buffered classes can be fixed up to be more @@ -1001,20 +1020,26 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "close" => ctx.new_rustfunc(PyBytesIORef::close), }); - py_module!(vm, "_io", { + let module = py_module!(vm, "_io", { "open" => ctx.new_rustfunc(io_open), "_IOBase" => io_base, - "_RawIOBase" => raw_io_base, + "_RawIOBase" => raw_io_base.clone(), "_BufferedIOBase" => buffered_io_base, "_TextIOBase" => text_io_base, - "FileIO" => file_io, "BufferedReader" => buffered_reader, "BufferedWriter" => buffered_writer, "TextIOWrapper" => text_io_wrapper, "StringIO" => string_io, "BytesIO" => bytes_io, "DEFAULT_BUFFER_SIZE" => ctx.new_int(8 * 1024), - }) + }); + + #[cfg(not(target_arch = "wasm32"))] + extend_module!(vm, module, { + "FileIO" => fileio::make_fileio(ctx, raw_io_base), + }); + + module } #[cfg(test)] diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 411baf1619..cf46062fd7 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -9,6 +9,7 @@ mod errno; mod functools; mod hashlib; mod imp; +pub mod io; mod itertools; mod json; #[cfg(feature = "rustpython-parser")] @@ -35,8 +36,6 @@ use std::collections::HashMap; use crate::vm::VirtualMachine; -#[cfg(not(target_arch = "wasm32"))] -pub mod io; #[cfg(not(target_arch = "wasm32"))] mod multiprocessing; #[cfg(not(target_arch = "wasm32"))] @@ -70,6 +69,7 @@ pub fn get_module_inits() -> HashMap { "errno".to_string() => Box::new(errno::make_module), "hashlib".to_string() => Box::new(hashlib::make_module), "itertools".to_string() => Box::new(itertools::make_module), + "_io".to_string() => Box::new(io::make_module), "json".to_string() => Box::new(json::make_module), "marshal".to_string() => Box::new(marshal::make_module), "math".to_string() => Box::new(math::make_module), @@ -106,7 +106,6 @@ pub fn get_module_inits() -> HashMap { // disable some modules on WASM #[cfg(not(target_arch = "wasm32"))] { - modules.insert("_io".to_string(), Box::new(io::make_module)); modules.insert("_os".to_string(), Box::new(os::make_module)); modules.insert("_socket".to_string(), Box::new(socket::make_module)); modules.insert( From 255dd5f4917715080b617f5730dd58a676262332 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Tue, 3 Dec 2019 11:37:46 -0600 Subject: [PATCH 3/3] Unify buffer and closed --- vm/src/stdlib/io.rs | 77 ++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 5c9bc1b847..8a70a94d42 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -1,7 +1,7 @@ /* * I/O core tools. */ -use std::cell::{Cell, RefCell}; +use std::cell::{RefCell, RefMut}; use std::fs; use std::io::prelude::*; use std::io::Cursor; @@ -102,8 +102,7 @@ impl BufferedIO { #[derive(Debug)] struct PyStringIO { - buffer: RefCell, - closed: Cell, + buffer: RefCell>, } type PyStringIORef = PyRef; @@ -115,11 +114,20 @@ impl PyValue for PyStringIO { } impl PyStringIORef { + fn buffer(&self, vm: &VirtualMachine) -> PyResult> { + let buffer = self.buffer.borrow_mut(); + if buffer.is_some() { + Ok(RefMut::map(buffer, |opt| opt.as_mut().unwrap())) + } else { + Err(vm.new_value_error("I/O operation on closed file.".to_string())) + } + } + //write string to underlying vector fn write(self, data: PyStringRef, vm: &VirtualMachine) -> PyResult { let bytes = data.as_str().as_bytes(); - match self.buffer.borrow_mut().write(bytes) { + match self.buffer(vm)?.write(bytes) { Some(value) => Ok(vm.ctx.new_int(value)), None => Err(vm.new_type_error("Error Writing String".to_string())), } @@ -127,7 +135,7 @@ impl PyStringIORef { //return the entire contents of the underlying fn getvalue(self, vm: &VirtualMachine) -> PyResult { - match String::from_utf8(self.buffer.borrow().getvalue()) { + match String::from_utf8(self.buffer(vm)?.getvalue()) { Ok(result) => Ok(vm.ctx.new_str(result)), Err(_) => Err(vm.new_value_error("Error Retrieving Value".to_string())), } @@ -135,7 +143,7 @@ impl PyStringIORef { //skip to the jth position fn seek(self, offset: u64, vm: &VirtualMachine) -> PyResult { - match self.buffer.borrow_mut().seek(offset) { + match self.buffer(vm)?.seek(offset) { Some(value) => Ok(vm.ctx.new_int(value)), None => Err(vm.new_value_error("Error Performing Operation".to_string())), } @@ -149,7 +157,7 @@ impl PyStringIORef { //If k is undefined || k == -1, then we read all bytes until the end of the file. //This also increments the stream position by the value of k fn read(self, bytes: OptionalOption, vm: &VirtualMachine) -> PyResult { - let data = match self.buffer.borrow_mut().read(byte_count(bytes)) { + let data = match self.buffer(vm)?.read(byte_count(bytes)) { Some(value) => value, None => Vec::new(), }; @@ -160,30 +168,30 @@ impl PyStringIORef { } } - fn tell(self, _vm: &VirtualMachine) -> u64 { - self.buffer.borrow().tell() + fn tell(self, vm: &VirtualMachine) -> PyResult { + Ok(self.buffer(vm)?.tell()) } fn readline(self, vm: &VirtualMachine) -> PyResult { - match self.buffer.borrow_mut().readline() { + match self.buffer(vm)?.readline() { Some(line) => Ok(line), None => Err(vm.new_value_error("Error Performing Operation".to_string())), } } - fn truncate(self, size: OptionalOption, _vm: &VirtualMachine) { - let mut buffer = self.buffer.borrow_mut(); + fn truncate(self, size: OptionalOption, vm: &VirtualMachine) -> PyResult<()> { + let mut buffer = self.buffer(vm)?; let size = size.flat_option().unwrap_or_else(|| buffer.tell() as usize); buffer.cursor.get_mut().truncate(size); + Ok(()) } fn closed(self, _vm: &VirtualMachine) -> bool { - self.closed.get() + self.buffer.borrow().is_none() } fn close(self, _vm: &VirtualMachine) { - self.buffer.borrow_mut().cursor.get_mut().clear(); - self.closed.set(true); + self.buffer.replace(None); } } @@ -207,16 +215,14 @@ fn string_io_new( }; PyStringIO { - buffer: RefCell::new(BufferedIO::new(Cursor::new(raw_string.into_bytes()))), - closed: Cell::new(false), + buffer: RefCell::new(Some(BufferedIO::new(Cursor::new(raw_string.into_bytes())))), } .into_ref_with_type(vm, cls) } #[derive(Debug)] struct PyBytesIO { - buffer: RefCell, - closed: Cell, + buffer: RefCell>, } type PyBytesIORef = PyRef; @@ -228,24 +234,31 @@ impl PyValue for PyBytesIO { } impl PyBytesIORef { - fn write(self, data: PyBytesLike, vm: &VirtualMachine) -> PyResult { - let bytes = data.to_cow(); + fn buffer(&self, vm: &VirtualMachine) -> PyResult> { + let buffer = self.buffer.borrow_mut(); + if buffer.is_some() { + Ok(RefMut::map(buffer, |opt| opt.as_mut().unwrap())) + } else { + Err(vm.new_value_error("I/O operation on closed file.".to_string())) + } + } - match self.buffer.borrow_mut().write(&bytes) { + fn write(self, data: PyBytesLike, vm: &VirtualMachine) -> PyResult { + match self.buffer(vm)?.write(data.to_cow().as_ref()) { Some(value) => Ok(value), None => Err(vm.new_type_error("Error Writing Bytes".to_string())), } } //Retrieves the entire bytes object value from the underlying buffer fn getvalue(self, vm: &VirtualMachine) -> PyResult { - Ok(vm.ctx.new_bytes(self.buffer.borrow().getvalue())) + Ok(vm.ctx.new_bytes(self.buffer(vm)?.getvalue())) } //Takes an integer k (bytes) and returns them from the underlying buffer //If k is undefined || k == -1, then we read all bytes until the end of the file. //This also increments the stream position by the value of k fn read(self, bytes: OptionalOption, vm: &VirtualMachine) -> PyResult { - match self.buffer.borrow_mut().read(byte_count(bytes)) { + match self.buffer(vm)?.read(byte_count(bytes)) { Some(value) => Ok(vm.ctx.new_bytes(value)), None => Err(vm.new_value_error("Error Retrieving Value".to_string())), } @@ -253,7 +266,7 @@ impl PyBytesIORef { //skip to the jth position fn seek(self, offset: u64, vm: &VirtualMachine) -> PyResult { - match self.buffer.borrow_mut().seek(offset) { + match self.buffer(vm)?.seek(offset) { Some(value) => Ok(vm.ctx.new_int(value)), None => Err(vm.new_value_error("Error Performing Operation".to_string())), } @@ -263,24 +276,23 @@ impl PyBytesIORef { true } - fn tell(self, _vm: &VirtualMachine) -> u64 { - self.buffer.borrow().tell() + fn tell(self, vm: &VirtualMachine) -> PyResult { + Ok(self.buffer(vm)?.tell()) } fn readline(self, vm: &VirtualMachine) -> PyResult> { - match self.buffer.borrow_mut().readline() { + match self.buffer(vm)?.readline() { Some(line) => Ok(line.as_bytes().to_vec()), None => Err(vm.new_value_error("Error Performing Operation".to_string())), } } fn closed(self, _vm: &VirtualMachine) -> bool { - self.closed.get() + self.buffer.borrow().is_none() } fn close(self, _vm: &VirtualMachine) { - self.buffer.borrow_mut().cursor.get_mut().clear(); - self.closed.set(true); + self.buffer.replace(None); } } @@ -295,8 +307,7 @@ fn bytes_io_new( }; PyBytesIO { - buffer: RefCell::new(BufferedIO::new(Cursor::new(raw_bytes))), - closed: Cell::new(false), + buffer: RefCell::new(Some(BufferedIO::new(Cursor::new(raw_bytes)))), } .into_ref_with_type(vm, cls) }