From c14a8302e82e5ae844fb9e89f6a14a2e197698bf Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Fri, 22 Feb 2019 20:14:10 -0800 Subject: [PATCH 01/13] Initial prototype of "extractor pattern" for native funcs --- vm/src/obj/objbool.rs | 8 ++- vm/src/obj/objint.rs | 20 +++++- vm/src/obj/objrange.rs | 31 +++++----- vm/src/pyobject.rs | 134 +++++++++++++++++++++++++++++++++++++++-- 4 files changed, 168 insertions(+), 25 deletions(-) diff --git a/vm/src/obj/objbool.rs b/vm/src/obj/objbool.rs index c5c9cb43b..41681a476 100644 --- a/vm/src/obj/objbool.rs +++ b/vm/src/obj/objbool.rs @@ -1,10 +1,16 @@ use super::objtype; use crate::pyobject::{ - PyContext, PyFuncArgs, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol, + IntoPyObject, PyContext, PyFuncArgs, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol, }; use crate::vm::VirtualMachine; use num_traits::Zero; +impl IntoPyObject for bool { + fn into_pyobject(self, ctx: &PyContext) -> PyResult { + Ok(ctx.new_bool(self)) + } +} + pub fn boolval(vm: &mut VirtualMachine, obj: PyObjectRef) -> Result { let result = match obj.borrow().payload { PyObjectPayload::Integer { ref value } => !value.is_zero(), diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 7e3da093e..5217ab012 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -3,8 +3,8 @@ use super::objstr; use super::objtype; use crate::format::FormatSpec; use crate::pyobject::{ - FromPyObjectRef, PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult, - TypeProtocol, + FromPyObject, FromPyObjectRef, IntoPyObject, PyContext, PyFuncArgs, PyObject, PyObjectPayload, + PyObjectRef, PyResult, TypeProtocol, }; use crate::vm::VirtualMachine; use num_bigint::{BigInt, ToBigInt}; @@ -15,6 +15,22 @@ use std::hash::{Hash, Hasher}; // This proxy allows for easy switching between types. type IntType = BigInt; +pub type PyInt = BigInt; + +impl IntoPyObject for PyInt { + fn into_pyobject(self, ctx: &PyContext) -> PyResult { + Ok(ctx.new_int(self)) + } +} + +// TODO: macro to impl for all primitive ints + +impl IntoPyObject for usize { + fn into_pyobject(self, ctx: &PyContext) -> PyResult { + Ok(ctx.new_int(self)) + } +} + fn int_repr(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!(vm, args, required = [(int, Some(vm.ctx.int_type()))]); let v = get_value(int); diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index c5075ccbd..db9b97a08 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -1,7 +1,8 @@ -use super::objint; +use super::objint::{self, PyInt}; use super::objtype; use crate::pyobject::{ - PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol, + FromPyObject, PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult, + TypeProtocol, }; use crate::vm::VirtualMachine; use num_bigint::{BigInt, Sign}; @@ -18,6 +19,14 @@ pub struct RangeType { pub step: BigInt, } +type PyRange = RangeType; + +impl FromPyObject for PyRange { + fn from_pyobject(obj: PyObjectRef) -> PyResult { + Ok(get_value(&obj)) + } +} + impl RangeType { #[inline] pub fn try_len(&self) -> Option { @@ -345,22 +354,12 @@ fn range_bool(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_bool(len > 0)) } -fn range_contains(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(zelf, Some(vm.ctx.range_type())), (needle, None)] - ); - - let range = get_value(zelf); - - let result = if objtype::isinstance(needle, &vm.ctx.int_type()) { - range.contains(&objint::get_value(needle)) +fn range_contains(vm: &mut VirtualMachine, zelf: PyRange, needle: PyObjectRef) -> bool { + if objtype::isinstance(&needle, &vm.ctx.int_type()) { + zelf.contains(&objint::get_value(&needle)) } else { false - }; - - Ok(vm.ctx.new_bool(result)) + } } fn range_index(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 0158028d0..2178cb04b 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -72,7 +72,7 @@ pub type PyObjectWeakRef = Weak>; /// Use this type for function which return a python object or and exception. /// Both the python object and the python exception are `PyObjectRef` types /// since exceptions are also python objects. -pub type PyResult = Result; // A valid value, or an exception +pub type PyResult = Result; // A valid value, or an exception /// For attributes we do not use a dict, but a hashmap. This is probably /// faster, unordered, and only supports strings as keys. @@ -553,13 +553,15 @@ impl PyContext { ) } - pub fn new_rustfunc PyResult>( - &self, - function: F, - ) -> PyObjectRef { + pub fn new_rustfunc(&self, factory: F) -> PyObjectRef + where + F: PyNativeFuncFactory, + T: FromPyFuncArgs, + R: IntoPyObject, + { PyObject::new( PyObjectPayload::RustFunction { - function: Box::new(function), + function: factory.create(), }, self.builtin_function_or_method_type(), ) @@ -922,6 +924,126 @@ impl PyFuncArgs { } } +pub trait FromPyObject: Sized { + fn from_pyobject(obj: PyObjectRef) -> PyResult; +} + +impl FromPyObject for PyObjectRef { + fn from_pyobject(obj: PyObjectRef) -> PyResult { + Ok(obj) + } +} + +pub trait IntoPyObject { + fn into_pyobject(self, ctx: &PyContext) -> PyResult; +} + +impl IntoPyObject for PyObjectRef { + fn into_pyobject(self, ctx: &PyContext) -> PyResult { + Ok(self) + } +} + +impl IntoPyObject for PyResult { + fn into_pyobject(self, ctx: &PyContext) -> PyResult { + self + } +} + +pub trait FromPyFuncArgs: Sized { + fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult; +} + +impl FromPyFuncArgs for Vec +where + T: FromPyFuncArgs, +{ + fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult { + let mut v = Vec::with_capacity(args.args.len()); + // TODO: This will loop infinitely if T::from_py_func_args doesn't + // consume any positional args. Check for this and panic. + while !args.args.is_empty() { + v.push(T::from_py_func_args(args)?); + } + Ok(v) + } +} + +macro_rules! tuple_from_py_func_args { + ($($T:ident),+) => { + impl<$($T),+> FromPyFuncArgs for ($($T,)+) + where + $($T: FromPyFuncArgs),+ + { + fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult { + Ok(($($T::from_py_func_args(args)?,)+)) + } + } + }; +} + +tuple_from_py_func_args!(A); +tuple_from_py_func_args!(A, B); +tuple_from_py_func_args!(A, B, C); +tuple_from_py_func_args!(A, B, C, D); +tuple_from_py_func_args!(A, B, C, D, E); + +impl FromPyFuncArgs for T +where + T: FromPyObject, +{ + fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult { + Self::from_pyobject(args.shift()) + } +} + +pub type PyNativeFunc = Box PyResult>; + +pub trait PyNativeFuncFactory { + fn create(self) -> PyNativeFunc; +} + +impl PyNativeFuncFactory<(A,), R> for F +where + F: Fn(&mut VirtualMachine, A) -> R + 'static, + A: FromPyFuncArgs, + R: IntoPyObject, +{ + fn create(self) -> PyNativeFunc { + Box::new(move |vm, mut args| { + // TODO: type-checking! + (self)(vm, A::from_py_func_args(&mut args)?).into_pyobject(&vm.ctx) + }) + } +} + +impl PyNativeFuncFactory<(A, B), R> for F +where + F: Fn(&mut VirtualMachine, A, B) -> R + 'static, + A: FromPyFuncArgs, + B: FromPyFuncArgs, + R: IntoPyObject, +{ + fn create(self) -> PyNativeFunc { + Box::new(move |vm, mut args| { + (self)( + vm, + A::from_py_func_args(&mut args)?, + B::from_py_func_args(&mut args)?, + ) + .into_pyobject(&vm.ctx) + }) + } +} + +impl FromPyFuncArgs for PyFuncArgs { + fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult { + // HACK HACK HACK + // TODO: get rid of this clone! + Ok(args.clone()) + } +} + /// Rather than determining the type of a python object, this enum is more /// a holder for the rust payload of a python object. It is more a carrier /// of rust data for a particular python object. Determine the python type From a74cbbbe192a0da4361b31622bae81e86ca52a36 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sat, 23 Feb 2019 08:23:15 -0800 Subject: [PATCH 02/13] Impl PyNativeFuncFactory for up to 5-tuples --- vm/src/pyobject.rs | 51 +++++++++++++++++++--------------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 2178cb04b..402e46741 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -1003,38 +1003,29 @@ pub trait PyNativeFuncFactory { fn create(self) -> PyNativeFunc; } -impl PyNativeFuncFactory<(A,), R> for F -where - F: Fn(&mut VirtualMachine, A) -> R + 'static, - A: FromPyFuncArgs, - R: IntoPyObject, -{ - fn create(self) -> PyNativeFunc { - Box::new(move |vm, mut args| { - // TODO: type-checking! - (self)(vm, A::from_py_func_args(&mut args)?).into_pyobject(&vm.ctx) - }) - } +macro_rules! tuple_py_native_func_factory { + ($($T:ident),+) => { + impl PyNativeFuncFactory<($($T,)+), R> for F + where + F: Fn(&mut VirtualMachine, $($T),+) -> R + 'static, + $($T: FromPyFuncArgs,)+ + R: IntoPyObject, + { + fn create(self) -> PyNativeFunc { + Box::new(move |vm, mut args| { + (self)(vm, $($T::from_py_func_args(&mut args)?,)+) + .into_pyobject(&vm.ctx) + }) + } + } + }; } -impl PyNativeFuncFactory<(A, B), R> for F -where - F: Fn(&mut VirtualMachine, A, B) -> R + 'static, - A: FromPyFuncArgs, - B: FromPyFuncArgs, - R: IntoPyObject, -{ - fn create(self) -> PyNativeFunc { - Box::new(move |vm, mut args| { - (self)( - vm, - A::from_py_func_args(&mut args)?, - B::from_py_func_args(&mut args)?, - ) - .into_pyobject(&vm.ctx) - }) - } -} +tuple_py_native_func_factory!(A); +tuple_py_native_func_factory!(A, B); +tuple_py_native_func_factory!(A, B, C); +tuple_py_native_func_factory!(A, B, C, D); +tuple_py_native_func_factory!(A, B, C, D, E); impl FromPyFuncArgs for PyFuncArgs { fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult { From fb0384d24d1774681c2443401cc085279eeb4d91 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sat, 23 Feb 2019 09:36:10 -0800 Subject: [PATCH 03/13] Some prerequisite data types for arg checking --- vm/src/obj/objrange.rs | 4 ++++ vm/src/pyobject.rs | 48 +++++++++++++++++++++++++++++------------- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index db9b97a08..1da01c890 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -22,6 +22,10 @@ pub struct RangeType { type PyRange = RangeType; impl FromPyObject for PyRange { + fn typ(ctx: &PyContext) -> Option { + Some(ctx.range_type()) + } + fn from_pyobject(obj: PyObjectRef) -> PyResult { Ok(get_value(&obj)) } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 402e46741..d08cb24be 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -925,10 +925,16 @@ impl PyFuncArgs { } pub trait FromPyObject: Sized { + fn typ(ctx: &PyContext) -> Option; + fn from_pyobject(obj: PyObjectRef) -> PyResult; } impl FromPyObject for PyObjectRef { + fn typ(_ctx: &PyContext) -> Option { + None + } + fn from_pyobject(obj: PyObjectRef) -> PyResult { Ok(obj) } @@ -951,22 +957,9 @@ impl IntoPyObject for PyResult { } pub trait FromPyFuncArgs: Sized { - fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult; -} + fn required_params(ctx: &PyContext) -> Vec; -impl FromPyFuncArgs for Vec -where - T: FromPyFuncArgs, -{ - fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult { - let mut v = Vec::with_capacity(args.args.len()); - // TODO: This will loop infinitely if T::from_py_func_args doesn't - // consume any positional args. Check for this and panic. - while !args.args.is_empty() { - v.push(T::from_py_func_args(args)?); - } - Ok(v) - } + fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult; } macro_rules! tuple_from_py_func_args { @@ -975,6 +968,10 @@ macro_rules! tuple_from_py_func_args { where $($T: FromPyFuncArgs),+ { + fn required_params(ctx: &PyContext) -> Vec { + vec![$($T::required_params(ctx),)+].into_iter().flatten().collect() + } + fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult { Ok(($($T::from_py_func_args(args)?,)+)) } @@ -992,6 +989,13 @@ impl FromPyFuncArgs for T where T: FromPyObject, { + fn required_params(ctx: &PyContext) -> Vec { + vec![Parameter { + kind: ParameterKind::PositionalOnly, + typ: T::typ(ctx), + }] + } + fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult { Self::from_pyobject(args.shift()) } @@ -1027,7 +1031,21 @@ tuple_py_native_func_factory!(A, B, C); tuple_py_native_func_factory!(A, B, C, D); tuple_py_native_func_factory!(A, B, C, D, E); +pub struct Parameter { + typ: Option, + kind: ParameterKind, +} + +pub enum ParameterKind { + PositionalOnly, + KeywordOnly { name: String }, +} + impl FromPyFuncArgs for PyFuncArgs { + fn required_params(_ctx: &PyContext) -> Vec { + vec![] + } + fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult { // HACK HACK HACK // TODO: get rid of this clone! From 2919d7f5208aee3a49e63c00d46224540572039f Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sat, 23 Feb 2019 10:04:29 -0800 Subject: [PATCH 04/13] Initial arg type checking --- vm/src/obj/objint.rs | 4 +- vm/src/obj/objrange.rs | 2 +- vm/src/obj/objstr.rs | 24 ++++++---- vm/src/pyobject.rs | 104 +++++++++++++++++++++++++++++++++-------- 4 files changed, 102 insertions(+), 32 deletions(-) diff --git a/vm/src/obj/objint.rs b/vm/src/obj/objint.rs index 5217ab012..eea031397 100644 --- a/vm/src/obj/objint.rs +++ b/vm/src/obj/objint.rs @@ -3,8 +3,8 @@ use super::objstr; use super::objtype; use crate::format::FormatSpec; use crate::pyobject::{ - FromPyObject, FromPyObjectRef, IntoPyObject, PyContext, PyFuncArgs, PyObject, PyObjectPayload, - PyObjectRef, PyResult, TypeProtocol, + FromPyObjectRef, IntoPyObject, PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, + PyResult, TypeProtocol, }; use crate::vm::VirtualMachine; use num_bigint::{BigInt, ToBigInt}; diff --git a/vm/src/obj/objrange.rs b/vm/src/obj/objrange.rs index 1da01c890..e2ac5de45 100644 --- a/vm/src/obj/objrange.rs +++ b/vm/src/obj/objrange.rs @@ -1,4 +1,4 @@ -use super::objint::{self, PyInt}; +use super::objint; use super::objtype; use crate::pyobject::{ FromPyObject, PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult, diff --git a/vm/src/obj/objstr.rs b/vm/src/obj/objstr.rs index 9c2051e6f..04394f7c5 100644 --- a/vm/src/obj/objstr.rs +++ b/vm/src/obj/objstr.rs @@ -3,7 +3,8 @@ use super::objsequence::PySliceableSequence; use super::objtype; use crate::format::{FormatParseError, FormatPart, FormatString}; use crate::pyobject::{ - PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol, + FromPyObject, PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult, + TypeProtocol, }; use crate::vm::VirtualMachine; use num_traits::ToPrimitive; @@ -17,6 +18,16 @@ extern crate unicode_segmentation; use self::unicode_segmentation::UnicodeSegmentation; +impl FromPyObject for String { + fn typ(ctx: &PyContext) -> Option { + Some(ctx.str_type()) + } + + fn from_pyobject(obj: PyObjectRef) -> PyResult { + Ok(get_value(&obj)) + } +} + pub fn init(context: &PyContext) { let str_type = &context.str_type; context.set_attr(&str_type, "__add__", context.new_rustfunc(str_add)); @@ -474,15 +485,8 @@ fn str_rstrip(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { Ok(vm.ctx.new_str(value)) } -fn str_endswith(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { - arg_check!( - vm, - args, - required = [(s, Some(vm.ctx.str_type())), (pat, Some(vm.ctx.str_type()))] - ); - let value = get_value(&s); - let pat = get_value(&pat); - Ok(vm.ctx.new_bool(value.ends_with(pat.as_str()))) +fn str_endswith(_vm: &mut VirtualMachine, zelf: String, suffix: String) -> bool { + zelf.ends_with(&suffix) } fn str_isidentifier(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index d08cb24be..d733238e3 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -556,12 +556,10 @@ impl PyContext { pub fn new_rustfunc(&self, factory: F) -> PyObjectRef where F: PyNativeFuncFactory, - T: FromPyFuncArgs, - R: IntoPyObject, { PyObject::new( PyObjectPayload::RustFunction { - function: factory.create(), + function: factory.create(self), }, self.builtin_function_or_method_type(), ) @@ -945,13 +943,13 @@ pub trait IntoPyObject { } impl IntoPyObject for PyObjectRef { - fn into_pyobject(self, ctx: &PyContext) -> PyResult { + fn into_pyobject(self, _ctx: &PyContext) -> PyResult { Ok(self) } } impl IntoPyObject for PyResult { - fn into_pyobject(self, ctx: &PyContext) -> PyResult { + fn into_pyobject(self, _ctx: &PyContext) -> PyResult { self } } @@ -991,7 +989,7 @@ where { fn required_params(ctx: &PyContext) -> Vec { vec![Parameter { - kind: ParameterKind::PositionalOnly, + kind: PositionalOnly, typ: T::typ(ctx), }] } @@ -1004,7 +1002,16 @@ where pub type PyNativeFunc = Box PyResult>; pub trait PyNativeFuncFactory { - fn create(self) -> PyNativeFunc; + fn create(self, ctx: &PyContext) -> PyNativeFunc; +} + +impl PyNativeFuncFactory for F +where + F: Fn(&mut VirtualMachine, PyFuncArgs) -> PyResult + 'static, +{ + fn create(self, _ctx: &PyContext) -> PyNativeFunc { + Box::new(self) + } } macro_rules! tuple_py_native_func_factory { @@ -1015,8 +1022,16 @@ macro_rules! tuple_py_native_func_factory { $($T: FromPyFuncArgs,)+ R: IntoPyObject, { - fn create(self) -> PyNativeFunc { + fn create(self, ctx: &PyContext) -> PyNativeFunc { + let parameters = vec![$($T::required_params(ctx)),+] + .into_iter() + .flatten() + .collect(); + let signature = Signature::new(parameters); + Box::new(move |vm, mut args| { + signature.check(vm, &mut args)?; + (self)(vm, $($T::from_py_func_args(&mut args)?,)+) .into_pyobject(&vm.ctx) }) @@ -1031,27 +1046,78 @@ tuple_py_native_func_factory!(A, B, C); tuple_py_native_func_factory!(A, B, C, D); tuple_py_native_func_factory!(A, B, C, D, E); +#[derive(Debug)] +pub struct Signature { + positional_params: Vec, + keyword_params: HashMap, +} + +impl Signature { + fn new(params: Vec) -> Self { + let mut positional_params = Vec::new(); + let mut keyword_params = HashMap::new(); + for param in params { + match param.kind { + PositionalOnly => { + positional_params.push(param); + } + KeywordOnly { ref name } => { + keyword_params.insert(name.clone(), param); + } + } + } + + Self { + positional_params, + keyword_params, + } + } + + fn arg_type(&self, pos: usize) -> Option<&PyObjectRef> { + self.positional_params[pos].typ.as_ref() + } + + #[allow(unused)] + fn kwarg_type(&self, name: &str) -> Option<&PyObjectRef> { + self.keyword_params[name].typ.as_ref() + } + + fn check(&self, vm: &mut VirtualMachine, args: &PyFuncArgs) -> PyResult<()> { + // TODO: check arity + + for (pos, arg) in args.args.iter().enumerate() { + if let Some(expected_type) = self.arg_type(pos) { + if !objtype::isinstance(arg, expected_type) { + let arg_typ = arg.typ(); + let expected_type_name = vm.to_pystr(&expected_type)?; + let actual_type = vm.to_pystr(&arg_typ)?; + return Err(vm.new_type_error(format!( + "argument of type {} is required for parameter {} (got: {})", + expected_type_name, + pos + 1, + actual_type + ))); + } + } + } + + Ok(()) + } +} + +#[derive(Debug)] pub struct Parameter { typ: Option, kind: ParameterKind, } +#[derive(Debug)] pub enum ParameterKind { PositionalOnly, KeywordOnly { name: String }, } -impl FromPyFuncArgs for PyFuncArgs { - fn required_params(_ctx: &PyContext) -> Vec { - vec![] - } - - fn from_py_func_args(args: &mut PyFuncArgs) -> PyResult { - // HACK HACK HACK - // TODO: get rid of this clone! - Ok(args.clone()) - } -} +use self::ParameterKind::*; /// Rather than determining the type of a python object, this enum is more /// a holder for the rust payload of a python object. It is more a carrier From 8186c77eb2c62c01589b03fad5183a2bd9c1b5f4 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 23 Feb 2019 12:38:29 +0200 Subject: [PATCH 05/13] Initial socket module --- vm/src/pyobject.rs | 5 + vm/src/stdlib/mod.rs | 2 + vm/src/stdlib/socket.rs | 238 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 245 insertions(+) create mode 100644 vm/src/stdlib/socket.rs diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index dd1f839ff..9e7a31f0f 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -29,6 +29,7 @@ use crate::obj::objsuper; use crate::obj::objtuple; use crate::obj::objtype; use crate::obj::objzip; +use crate::stdlib::socket::Socket; use crate::vm::VirtualMachine; use num_bigint::BigInt; use num_bigint::ToBigInt; @@ -1045,6 +1046,9 @@ pub enum PyObjectPayload { RustFunction { function: Box PyResult>, }, + Socket { + socket: Socket, + }, } impl fmt::Debug for PyObjectPayload { @@ -1082,6 +1086,7 @@ impl fmt::Debug for PyObjectPayload { PyObjectPayload::Instance { .. } => write!(f, "instance"), PyObjectPayload::RustFunction { .. } => write!(f, "rust function"), PyObjectPayload::Frame { .. } => write!(f, "frame"), + PyObjectPayload::Socket { .. } => write!(f, "socket"), } } } diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index a0dcd9bff..0a0d378be 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -6,6 +6,7 @@ mod math; mod pystruct; mod random; mod re; +pub mod socket; mod string; mod time_module; mod tokenize; @@ -46,6 +47,7 @@ pub fn get_module_inits() -> HashMap { { modules.insert("io".to_string(), io::mk_module as StdlibInitFunc); modules.insert("os".to_string(), os::mk_module as StdlibInitFunc); + modules.insert("socket".to_string(), socket::mk_module as StdlibInitFunc); } modules diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs new file mode 100644 index 000000000..b94b12434 --- /dev/null +++ b/vm/src/stdlib/socket.rs @@ -0,0 +1,238 @@ +use crate::obj::objbytes; +use crate::obj::objint; +use crate::obj::objstr; + +use crate::pyobject::{ + PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol, +}; +use crate::vm::VirtualMachine; + +use num_traits::ToPrimitive; +use std::fmt; +use std::io::Read; +use std::io::Write; +use std::net::{TcpListener, TcpStream, UdpSocket}; + +#[derive(Debug)] +enum AddressFamily { + AF_UNIX = 1, + AF_INET = 2, + AF_INET6 = 3, +} + +impl AddressFamily { + fn from_i32(value: i32) -> AddressFamily { + match value { + 1 => AddressFamily::AF_UNIX, + 2 => AddressFamily::AF_INET, + 3 => AddressFamily::AF_INET6, + _ => panic!("Unknown value: {}", value), + } + } +} + +#[derive(Debug)] +enum SocketKind { + SOCK_STREAM = 1, + SOCK_DGRAM = 2, +} + +impl SocketKind { + fn from_i32(value: i32) -> SocketKind { + match value { + 1 => SocketKind::SOCK_STREAM, + 2 => SocketKind::SOCK_DGRAM, + _ => panic!("Unknown value: {}", value), + } + } +} + +impl fmt::Display for AddressFamily { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +impl fmt::Display for SocketKind { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub struct Socket { + af: AddressFamily, + sk: SocketKind, + tcp_listener: Option, + tcp_stream: Option, + udp_socket: Option, +} + +impl Socket { + fn new(af: AddressFamily, sk: SocketKind) -> Socket { + Socket { + af: af, + sk: sk, + tcp_listener: None, + tcp_stream: None, + udp_socket: None, + } + } + + fn get_tcp_stream(&self) -> Option<&TcpStream> { + match &self.tcp_stream { + Some(v) => Some(v), + None => None, + } + } +} + +fn socket_new(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [ + (cls, None), + (family_int, Some(vm.ctx.int_type())), + (kind_int, Some(vm.ctx.int_type())) + ] + ); + + let family = AddressFamily::from_i32(objint::get_value(family_int).to_i32().unwrap()); + let kind = SocketKind::from_i32(objint::get_value(kind_int).to_i32().unwrap()); + + let socket = Socket::new(family, kind); + + Ok(PyObject::new( + PyObjectPayload::Socket { socket }, + cls.clone(), + )) +} + +fn socket_connect(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(zelf, None), (address, Some(vm.ctx.str_type()))] + ); + + let mut mut_obj = zelf.borrow_mut(); + + match mut_obj.payload { + PyObjectPayload::Socket { ref mut socket } => { + if let Ok(stream) = TcpStream::connect(objstr::get_value(&address)) { + socket.tcp_stream = Some(stream); + Ok(vm.get_none()) + } else { + // TODO: Socket error + Err(vm.new_type_error("socket failed".to_string())) + } + } + _ => Err(vm.new_type_error("".to_string())), + } +} + +fn socket_recv(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(zelf, None), (bufsize, Some(vm.ctx.int_type()))] + ); + let mut mut_obj = zelf.borrow_mut(); + + match mut_obj.payload { + PyObjectPayload::Socket { ref mut socket } => match socket.af { + AddressFamily::AF_INET => match socket.sk { + SocketKind::SOCK_STREAM => { + let mut buffer = Vec::new(); + socket + .get_tcp_stream() + .unwrap() + .read_to_end(&mut buffer) + .unwrap(); + Ok(PyObject::new( + PyObjectPayload::Bytes { value: buffer }, + vm.ctx.bytes_type(), + )) + } + _ => Err(vm.new_type_error("".to_string())), + }, + _ => Err(vm.new_type_error("".to_string())), + }, + _ => Err(vm.new_type_error("".to_string())), + } +} + +fn socket_send(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(zelf, None), (bytes, Some(vm.ctx.bytes_type()))] + ); + let mut mut_obj = zelf.borrow_mut(); + + match mut_obj.payload { + PyObjectPayload::Socket { ref mut socket } => match socket.af { + AddressFamily::AF_INET => match socket.sk { + SocketKind::SOCK_STREAM => { + socket + .get_tcp_stream() + .unwrap() + .write(&objbytes::get_value(&bytes)) + .unwrap(); + Ok(vm.get_none()) + } + _ => Err(vm.new_type_error("".to_string())), + }, + _ => Err(vm.new_type_error("".to_string())), + }, + _ => Err(vm.new_type_error("".to_string())), + } +} + +fn socket_close(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, None)]); + let mut mut_obj = zelf.borrow_mut(); + + match mut_obj.payload { + PyObjectPayload::Socket { ref mut socket } => match socket.af { + AddressFamily::AF_INET => match socket.sk { + SocketKind::SOCK_STREAM => { + socket.tcp_stream = None; + Ok(vm.get_none()) + } + _ => Err(vm.new_type_error("".to_string())), + }, + _ => Err(vm.new_type_error("".to_string())), + }, + _ => Err(vm.new_type_error("".to_string())), + } +} + +pub fn mk_module(ctx: &PyContext) -> PyObjectRef { + let py_mod = ctx.new_module(&"socket".to_string(), ctx.new_scope(None)); + + ctx.set_attr( + &py_mod, + &AddressFamily::AF_INET.to_string(), + ctx.new_int(AddressFamily::AF_INET as i32), + ); + + ctx.set_attr( + &py_mod, + &SocketKind::SOCK_STREAM.to_string(), + ctx.new_int(SocketKind::SOCK_STREAM as i32), + ); + + let socket = { + let socket = ctx.new_class("socket", ctx.object()); + ctx.set_attr(&socket, "__new__", ctx.new_rustfunc(socket_new)); + ctx.set_attr(&socket, "connect", ctx.new_rustfunc(socket_connect)); + ctx.set_attr(&socket, "recv", ctx.new_rustfunc(socket_recv)); + ctx.set_attr(&socket, "send", ctx.new_rustfunc(socket_send)); + ctx.set_attr(&socket, "close", ctx.new_rustfunc(socket_close)); + socket + }; + ctx.set_attr(&py_mod, "socket", socket.clone()); + + py_mod +} From 49837f57a713522dfc2726d9eb72b44e79a20cab Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 24 Feb 2019 20:38:26 +0200 Subject: [PATCH 06/13] Use Connection enum --- vm/src/stdlib/socket.rs | 142 +++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 76 deletions(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index b94b12434..7b2b76ddb 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -8,24 +8,24 @@ use crate::pyobject::{ use crate::vm::VirtualMachine; use num_traits::ToPrimitive; -use std::fmt; +use std::io; use std::io::Read; use std::io::Write; use std::net::{TcpListener, TcpStream, UdpSocket}; #[derive(Debug)] enum AddressFamily { - AF_UNIX = 1, - AF_INET = 2, - AF_INET6 = 3, + AfUnix = 1, + AfInet = 2, + AfInet6 = 3, } impl AddressFamily { fn from_i32(value: i32) -> AddressFamily { match value { - 1 => AddressFamily::AF_UNIX, - 2 => AddressFamily::AF_INET, - 3 => AddressFamily::AF_INET6, + 1 => AddressFamily::AfUnix, + 2 => AddressFamily::AfInet, + 3 => AddressFamily::AfInet6, _ => panic!("Unknown value: {}", value), } } @@ -33,55 +33,59 @@ impl AddressFamily { #[derive(Debug)] enum SocketKind { - SOCK_STREAM = 1, - SOCK_DGRAM = 2, + SockStream = 1, + SockDgram = 2, } impl SocketKind { fn from_i32(value: i32) -> SocketKind { match value { - 1 => SocketKind::SOCK_STREAM, - 2 => SocketKind::SOCK_DGRAM, + 1 => SocketKind::SockStream, + 2 => SocketKind::SockDgram, _ => panic!("Unknown value: {}", value), } } } -impl fmt::Display for AddressFamily { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", self) +enum Connection { + TcpListener(TcpListener), + TcpStream(TcpStream), + UdpSocket(UdpSocket), +} + +impl Read for Connection { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Connection::TcpStream(con) => con.read(buf), + _ => Err(io::Error::new(io::ErrorKind::Other, "oh no!")), + } } } -impl fmt::Display for SocketKind { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{:?}", self) +impl Write for Connection { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + Connection::TcpStream(con) => con.write(buf), + _ => Err(io::Error::new(io::ErrorKind::Other, "oh no!")), + } + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) } } pub struct Socket { - af: AddressFamily, + address_family: AddressFamily, sk: SocketKind, - tcp_listener: Option, - tcp_stream: Option, - udp_socket: Option, + con: Option, } impl Socket { - fn new(af: AddressFamily, sk: SocketKind) -> Socket { + fn new(address_family: AddressFamily, sk: SocketKind) -> Socket { Socket { - af: af, + address_family, sk: sk, - tcp_listener: None, - tcp_stream: None, - udp_socket: None, - } - } - - fn get_tcp_stream(&self) -> Option<&TcpStream> { - match &self.tcp_stream { - Some(v) => Some(v), - None => None, + con: None, } } } @@ -97,10 +101,10 @@ fn socket_new(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { ] ); - let family = AddressFamily::from_i32(objint::get_value(family_int).to_i32().unwrap()); + let address_family = AddressFamily::from_i32(objint::get_value(family_int).to_i32().unwrap()); let kind = SocketKind::from_i32(objint::get_value(kind_int).to_i32().unwrap()); - let socket = Socket::new(family, kind); + let socket = Socket::new(address_family, kind); Ok(PyObject::new( PyObjectPayload::Socket { socket }, @@ -120,7 +124,7 @@ fn socket_connect(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { match mut_obj.payload { PyObjectPayload::Socket { ref mut socket } => { if let Ok(stream) = TcpStream::connect(objstr::get_value(&address)) { - socket.tcp_stream = Some(stream); + socket.con = Some(Connection::TcpStream(stream)); Ok(vm.get_none()) } else { // TODO: Socket error @@ -140,24 +144,17 @@ fn socket_recv(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { let mut mut_obj = zelf.borrow_mut(); match mut_obj.payload { - PyObjectPayload::Socket { ref mut socket } => match socket.af { - AddressFamily::AF_INET => match socket.sk { - SocketKind::SOCK_STREAM => { - let mut buffer = Vec::new(); - socket - .get_tcp_stream() - .unwrap() - .read_to_end(&mut buffer) - .unwrap(); - Ok(PyObject::new( - PyObjectPayload::Bytes { value: buffer }, - vm.ctx.bytes_type(), - )) - } - _ => Err(vm.new_type_error("".to_string())), - }, - _ => Err(vm.new_type_error("".to_string())), - }, + PyObjectPayload::Socket { ref mut socket } => { + let mut buffer = Vec::new(); + let _temp = match socket.con { + Some(ref mut v) => v.read_to_end(&mut buffer).unwrap(), + None => 0, + }; + Ok(PyObject::new( + PyObjectPayload::Bytes { value: buffer }, + vm.ctx.bytes_type(), + )) + } _ => Err(vm.new_type_error("".to_string())), } } @@ -171,20 +168,13 @@ fn socket_send(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { let mut mut_obj = zelf.borrow_mut(); match mut_obj.payload { - PyObjectPayload::Socket { ref mut socket } => match socket.af { - AddressFamily::AF_INET => match socket.sk { - SocketKind::SOCK_STREAM => { - socket - .get_tcp_stream() - .unwrap() - .write(&objbytes::get_value(&bytes)) - .unwrap(); - Ok(vm.get_none()) - } - _ => Err(vm.new_type_error("".to_string())), - }, - _ => Err(vm.new_type_error("".to_string())), - }, + PyObjectPayload::Socket { ref mut socket } => { + match socket.con { + Some(ref mut v) => v.write(&objbytes::get_value(&bytes)).unwrap(), + None => return Err(vm.new_type_error("".to_string())), + }; + Ok(vm.get_none()) + } _ => Err(vm.new_type_error("".to_string())), } } @@ -194,10 +184,10 @@ fn socket_close(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { let mut mut_obj = zelf.borrow_mut(); match mut_obj.payload { - PyObjectPayload::Socket { ref mut socket } => match socket.af { - AddressFamily::AF_INET => match socket.sk { - SocketKind::SOCK_STREAM => { - socket.tcp_stream = None; + PyObjectPayload::Socket { ref mut socket } => match socket.address_family { + AddressFamily::AfInet => match socket.sk { + SocketKind::SockStream => { + socket.con = None; Ok(vm.get_none()) } _ => Err(vm.new_type_error("".to_string())), @@ -213,14 +203,14 @@ pub fn mk_module(ctx: &PyContext) -> PyObjectRef { ctx.set_attr( &py_mod, - &AddressFamily::AF_INET.to_string(), - ctx.new_int(AddressFamily::AF_INET as i32), + "AF_INET", + ctx.new_int(AddressFamily::AfInet as i32), ); ctx.set_attr( &py_mod, - &SocketKind::SOCK_STREAM.to_string(), - ctx.new_int(SocketKind::SOCK_STREAM as i32), + "SOCK_STREAM", + ctx.new_int(SocketKind::SockStream as i32), ); let socket = { From c84d35126e4494532bf03b762a1de7986438350a Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 24 Feb 2019 21:35:47 +0200 Subject: [PATCH 07/13] Add socket.{bind,listen,accept} --- vm/src/stdlib/socket.rs | 77 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 7b2b76ddb..d592cf15e 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -11,9 +11,9 @@ use num_traits::ToPrimitive; use std::io; use std::io::Read; use std::io::Write; -use std::net::{TcpListener, TcpStream, UdpSocket}; +use std::net::{SocketAddr, TcpListener, TcpStream, UdpSocket}; -#[derive(Debug)] +#[derive(Copy, Clone)] enum AddressFamily { AfUnix = 1, AfInet = 2, @@ -31,7 +31,7 @@ impl AddressFamily { } } -#[derive(Debug)] +#[derive(Copy, Clone)] enum SocketKind { SockStream = 1, SockDgram = 2, @@ -53,6 +53,15 @@ enum Connection { UdpSocket(UdpSocket), } +impl Connection { + fn accept(&mut self) -> io::Result<(TcpStream, SocketAddr)> { + match self { + Connection::TcpListener(con) => con.accept(), + _ => Err(io::Error::new(io::ErrorKind::Other, "oh no!")), + } + } +} + impl Read for Connection { fn read(&mut self, buf: &mut [u8]) -> io::Result { match self { @@ -135,6 +144,65 @@ fn socket_connect(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { } } +fn socket_bind(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!( + vm, + args, + required = [(zelf, None), (address, Some(vm.ctx.str_type()))] + ); + + let mut mut_obj = zelf.borrow_mut(); + + match mut_obj.payload { + PyObjectPayload::Socket { ref mut socket } => { + if let Ok(stream) = TcpListener::bind(objstr::get_value(&address)) { + socket.con = Some(Connection::TcpListener(stream)); + Ok(vm.get_none()) + } else { + // TODO: Socket error + Err(vm.new_type_error("socket failed".to_string())) + } + } + _ => Err(vm.new_type_error("".to_string())), + } +} + +fn socket_listen(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + Ok(vm.get_none()) +} + +fn socket_accept(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { + arg_check!(vm, args, required = [(zelf, None)]); + + let mut mut_obj = zelf.borrow_mut(); + + match mut_obj.payload { + PyObjectPayload::Socket { ref mut socket } => { + let ret = match socket.con { + Some(ref mut v) => v.accept(), + None => return Err(vm.new_type_error("".to_string())), + }; + + let tcp_stream = match ret { + Ok((socket, _addr)) => socket, + _ => return Err(vm.new_type_error("".to_string())), + }; + + let socket = Socket { + address_family: socket.address_family.clone(), + sk: socket.sk.clone(), + con: Some(Connection::TcpStream(tcp_stream)), + }; + + Ok(PyObject::new( + PyObjectPayload::Socket { socket }, + mut_obj.typ(), + )) + } + _ => Err(vm.new_type_error("".to_string())), + } +} + fn socket_recv(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, @@ -219,6 +287,9 @@ pub fn mk_module(ctx: &PyContext) -> PyObjectRef { ctx.set_attr(&socket, "connect", ctx.new_rustfunc(socket_connect)); ctx.set_attr(&socket, "recv", ctx.new_rustfunc(socket_recv)); ctx.set_attr(&socket, "send", ctx.new_rustfunc(socket_send)); + ctx.set_attr(&socket, "bind", ctx.new_rustfunc(socket_bind)); + ctx.set_attr(&socket, "accept", ctx.new_rustfunc(socket_accept)); + ctx.set_attr(&socket, "listen", ctx.new_rustfunc(socket_listen)); ctx.set_attr(&socket, "close", ctx.new_rustfunc(socket_close)); socket }; From 1322c8d6db34a8ea25b29971e9bf150bd92e67ca Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 24 Feb 2019 22:20:07 +0200 Subject: [PATCH 08/13] Change socket parameters to match CPython --- vm/src/stdlib/socket.rs | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index d592cf15e..f1634d645 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -1,7 +1,7 @@ use crate::obj::objbytes; use crate::obj::objint; +use crate::obj::objsequence::get_elements; use crate::obj::objstr; - use crate::pyobject::{ PyContext, PyFuncArgs, PyObject, PyObjectPayload, PyObjectRef, PyResult, TypeProtocol, }; @@ -125,14 +125,20 @@ fn socket_connect(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, args, - required = [(zelf, None), (address, Some(vm.ctx.str_type()))] + required = [(zelf, None), (address, Some(vm.ctx.tuple_type()))] ); + let elements = get_elements(address); + let host = objstr::get_value(&elements[0]); + let port = objint::get_value(&elements[1]); + + let address_string = format!("{}:{}", host, port.to_string()); + let mut mut_obj = zelf.borrow_mut(); match mut_obj.payload { PyObjectPayload::Socket { ref mut socket } => { - if let Ok(stream) = TcpStream::connect(objstr::get_value(&address)) { + if let Ok(stream) = TcpStream::connect(address_string) { socket.con = Some(Connection::TcpStream(stream)); Ok(vm.get_none()) } else { @@ -148,14 +154,20 @@ fn socket_bind(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, args, - required = [(zelf, None), (address, Some(vm.ctx.str_type()))] + required = [(zelf, None), (address, Some(vm.ctx.tuple_type()))] ); + let elements = get_elements(address); + let host = objstr::get_value(&elements[0]); + let port = objint::get_value(&elements[1]); + + let address_string = format!("{}:{}", host, port.to_string()); + let mut mut_obj = zelf.borrow_mut(); match mut_obj.payload { PyObjectPayload::Socket { ref mut socket } => { - if let Ok(stream) = TcpListener::bind(objstr::get_value(&address)) { + if let Ok(stream) = TcpListener::bind(address_string) { socket.con = Some(Connection::TcpListener(stream)); Ok(vm.get_none()) } else { @@ -194,9 +206,13 @@ fn socket_accept(vm: &mut VirtualMachine, args: PyFuncArgs) -> PyResult { con: Some(Connection::TcpStream(tcp_stream)), }; + let sock_obj = PyObject::new(PyObjectPayload::Socket { socket }, mut_obj.typ()); + + let elements = vec![sock_obj, vm.get_none()]; + Ok(PyObject::new( - PyObjectPayload::Socket { socket }, - mut_obj.typ(), + PyObjectPayload::Sequence { elements }, + vm.ctx.tuple_type(), )) } _ => Err(vm.new_type_error("".to_string())), From c87cf9004390fc598646eff3be86daa0aa5735ad Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sun, 24 Feb 2019 22:20:20 +0200 Subject: [PATCH 09/13] Add socket test --- tests/snippets/stdlib_socket.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 tests/snippets/stdlib_socket.py diff --git a/tests/snippets/stdlib_socket.py b/tests/snippets/stdlib_socket.py new file mode 100644 index 000000000..5419d802f --- /dev/null +++ b/tests/snippets/stdlib_socket.py @@ -0,0 +1,20 @@ +import socket + +listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +listener.bind(("127.0.0.1", 8080)) +listener.listen(1) + +connector = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +connector.connect(("127.0.0.1", 8080)) +connection = listener.accept()[0] + +message_a = b'aaaa' +message_b = b'bbbbb' + +connector.send(message_a) +connector.close() +recv_a = connection.recv(10) + +connection.close() +listener.close() + From f2079807bd36e89ae1cb4e98998c2f283572c0be Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sun, 24 Feb 2019 14:22:18 -0600 Subject: [PATCH 10/13] Fix snippet selector's default selection --- wasm/demo/src/index.ejs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wasm/demo/src/index.ejs b/wasm/demo/src/index.ejs index 2d06c3200..0cb2def1d 100644 --- a/wasm/demo/src/index.ejs +++ b/wasm/demo/src/index.ejs @@ -20,7 +20,7 @@ From d6e317b185ba7b2f39e5a37d4f3b13854df3e478 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 24 Feb 2019 10:38:19 -0800 Subject: [PATCH 11/13] Move f-string parser into own module and clean up a bit --- parser/src/fstring.rs | 200 ++++++++++++++++++++++++++++++++++ parser/src/lib.rs | 1 + parser/src/parser.rs | 219 -------------------------------------- parser/src/python.lalrpop | 10 +- 4 files changed, 207 insertions(+), 223 deletions(-) create mode 100644 parser/src/fstring.rs diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs new file mode 100644 index 000000000..d5881cdfa --- /dev/null +++ b/parser/src/fstring.rs @@ -0,0 +1,200 @@ +use std::iter; +use std::mem; +use std::str; + +use lalrpop_util::ParseError as LalrpopError; + +use crate::ast::StringGroup; +use crate::lexer::{LexicalError, Location, Tok}; +use crate::parser::parse_expression; + +use self::StringGroup::*; + +// TODO: consolidate these with ParseError +#[derive(Debug, PartialEq)] +pub enum FStringError { + UnclosedLbrace, + UnopenedRbrace, + InvalidExpression, +} + +impl From for LalrpopError { + fn from(_err: FStringError) -> Self { + lalrpop_util::ParseError::User { + error: LexicalError::StringError, + } + } +} + +struct FStringParser<'a> { + chars: iter::Peekable>, +} + +impl<'a> FStringParser<'a> { + fn new(source: &'a str) -> Self { + Self { + chars: source.chars().peekable(), + } + } + + fn parse_formatted_value(&mut self) -> Result { + let mut expression = String::new(); + let mut spec = String::new(); + let mut depth = 0; + + while let Some(ch) = self.chars.next() { + match ch { + ':' if depth == 0 => { + while let Some(&next) = self.chars.peek() { + if next != '}' { + spec.push(next); + self.chars.next(); + } else { + break; + } + } + } + '{' => { + if let Some('{') = self.chars.peek() { + expression.push_str("{{"); + self.chars.next(); + } else { + expression.push('{'); + depth += 1; + } + } + '}' => { + if let Some('}') = self.chars.peek() { + expression.push_str("}}"); + self.chars.next(); + } else if depth > 0 { + expression.push('}'); + depth -= 1; + } else { + return Ok(FormattedValue { + value: Box::new( + parse_expression(expression.trim()) + .map_err(|_| FStringError::InvalidExpression)?, + ), + spec, + }); + } + } + _ => { + expression.push(ch); + } + } + } + + return Err(FStringError::UnclosedLbrace); + } + + fn parse(mut self) -> Result { + let mut content = String::new(); + let mut values = vec![]; + + while let Some(ch) = self.chars.next() { + match ch { + '{' => { + if let Some('{') = self.chars.peek() { + self.chars.next(); + content.push('{'); + } else { + if !content.is_empty() { + values.push(Constant { + value: mem::replace(&mut content, String::new()), + }); + } + + values.push(self.parse_formatted_value()?); + } + } + '}' => { + if let Some('}') = self.chars.peek() { + self.chars.next(); + content.push('}'); + } else { + return Err(FStringError::UnopenedRbrace); + } + } + _ => { + content.push(ch); + } + } + } + + if !content.is_empty() { + values.push(Constant { value: content }) + } + + Ok(match values.len() { + 0 => Constant { + value: String::new(), + }, + 1 => values.into_iter().next().unwrap(), + _ => Joined { values }, + }) + } +} + +pub fn parse_fstring(source: &str) -> Result { + FStringParser::new(source).parse() +} + +#[cfg(test)] +mod tests { + use crate::ast; + + use super::*; + + fn mk_ident(name: &str) -> ast::Expression { + ast::Expression::Identifier { + name: name.to_owned(), + } + } + + #[test] + fn test_parse_fstring() { + let source = String::from("{a}{ b }{{foo}}"); + let parse_ast = parse_fstring(&source).unwrap(); + + assert_eq!( + parse_ast, + Joined { + values: vec![ + FormattedValue { + value: Box::new(mk_ident("a")), + spec: String::new(), + }, + FormattedValue { + value: Box::new(mk_ident("b")), + spec: String::new(), + }, + Constant { + value: "{foo}".to_owned() + } + ] + } + ); + } + + #[test] + fn test_parse_empty_fstring() { + assert_eq!( + parse_fstring(""), + Ok(Constant { + value: String::new(), + }), + ); + } + + #[test] + fn test_parse_invalid_fstring() { + assert_eq!(parse_fstring("{"), Err(FStringError::UnclosedLbrace)); + assert_eq!(parse_fstring("}"), Err(FStringError::UnopenedRbrace)); + assert_eq!( + parse_fstring("{class}"), + Err(FStringError::InvalidExpression) + ); + } +} diff --git a/parser/src/lib.rs b/parser/src/lib.rs index f7f369968..b10c3d551 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -3,6 +3,7 @@ extern crate log; pub mod ast; pub mod error; +mod fstring; pub mod lexer; pub mod parser; #[cfg_attr(rustfmt, rustfmt_skip)] diff --git a/parser/src/parser.rs b/parser/src/parser.rs index fb8f2f0e5..2a39a41f8 100644 --- a/parser/src/parser.rs +++ b/parser/src/parser.rs @@ -65,180 +65,12 @@ pub fn parse_expression(source: &str) -> Result { do_lalr_parsing!(source, Expression, StartExpression) } -// TODO: consolidate these with ParseError -#[derive(Debug, PartialEq)] -pub enum FStringError { - UnclosedLbrace, - UnopenedRbrace, - InvalidExpression, -} - -impl From - for lalrpop_util::ParseError -{ - fn from(_err: FStringError) -> Self { - lalrpop_util::ParseError::User { - error: lexer::LexicalError::StringError, - } - } -} - -enum ParseState { - Text { - content: String, - }, - FormattedValue { - expression: String, - spec: Option, - depth: usize, - }, -} - -pub fn parse_fstring(source: &str) -> Result { - use self::ParseState::*; - - let mut values = vec![]; - let mut state = ParseState::Text { - content: String::new(), - }; - - let mut chars = source.chars().peekable(); - while let Some(ch) = chars.next() { - state = match state { - Text { mut content } => match ch { - '{' => { - if let Some('{') = chars.peek() { - chars.next(); - content.push('{'); - Text { content } - } else { - if !content.is_empty() { - values.push(ast::StringGroup::Constant { value: content }); - } - - FormattedValue { - expression: String::new(), - spec: None, - depth: 0, - } - } - } - '}' => { - if let Some('}') = chars.peek() { - chars.next(); - content.push('}'); - Text { content } - } else { - return Err(FStringError::UnopenedRbrace); - } - } - _ => { - content.push(ch); - Text { content } - } - }, - - FormattedValue { - mut expression, - mut spec, - depth, - } => match ch { - ':' if depth == 0 => FormattedValue { - expression, - spec: Some(String::new()), - depth, - }, - '{' => { - if let Some('{') = chars.peek() { - expression.push_str("{{"); - chars.next(); - FormattedValue { - expression, - spec, - depth, - } - } else { - expression.push('{'); - FormattedValue { - expression, - spec, - depth: depth + 1, - } - } - } - '}' => { - if let Some('}') = chars.peek() { - expression.push_str("}}"); - chars.next(); - FormattedValue { - expression, - spec, - depth, - } - } else if depth > 0 { - expression.push('}'); - FormattedValue { - expression, - spec, - depth: depth - 1, - } - } else { - values.push(ast::StringGroup::FormattedValue { - value: Box::new(match parse_expression(expression.trim()) { - Ok(expr) => expr, - Err(_) => return Err(FStringError::InvalidExpression), - }), - spec: spec.unwrap_or_default(), - }); - Text { - content: String::new(), - } - } - } - _ => { - if let Some(spec) = spec.as_mut() { - spec.push(ch) - } else { - expression.push(ch); - } - FormattedValue { - expression, - spec, - depth, - } - } - }, - }; - } - - match state { - Text { content } => { - if !content.is_empty() { - values.push(ast::StringGroup::Constant { value: content }) - } - } - FormattedValue { .. } => { - return Err(FStringError::UnclosedLbrace); - } - } - - Ok(match values.len() { - 0 => ast::StringGroup::Constant { - value: String::new(), - }, - 1 => values.into_iter().next().unwrap(), - _ => ast::StringGroup::Joined { values }, - }) -} - #[cfg(test)] mod tests { use super::ast; use super::parse_expression; - use super::parse_fstring; use super::parse_program; use super::parse_statement; - use super::FStringError; use num_bigint::BigInt; #[test] @@ -630,55 +462,4 @@ mod tests { } ); } - - fn mk_ident(name: &str) -> ast::Expression { - ast::Expression::Identifier { - name: name.to_owned(), - } - } - - #[test] - fn test_parse_fstring() { - let source = String::from("{a}{ b }{{foo}}"); - let parse_ast = parse_fstring(&source).unwrap(); - - assert_eq!( - parse_ast, - ast::StringGroup::Joined { - values: vec![ - ast::StringGroup::FormattedValue { - value: Box::new(mk_ident("a")), - spec: String::new(), - }, - ast::StringGroup::FormattedValue { - value: Box::new(mk_ident("b")), - spec: String::new(), - }, - ast::StringGroup::Constant { - value: "{foo}".to_owned() - } - ] - } - ); - } - - #[test] - fn test_parse_empty_fstring() { - assert_eq!( - parse_fstring(""), - Ok(ast::StringGroup::Constant { - value: String::new(), - }), - ); - } - - #[test] - fn test_parse_invalid_fstring() { - assert_eq!(parse_fstring("{"), Err(FStringError::UnclosedLbrace)); - assert_eq!(parse_fstring("}"), Err(FStringError::UnopenedRbrace)); - assert_eq!( - parse_fstring("{class}"), - Err(FStringError::InvalidExpression) - ); - } } diff --git a/parser/src/python.lalrpop b/parser/src/python.lalrpop index 8ee407f9a..2748877dd 100644 --- a/parser/src/python.lalrpop +++ b/parser/src/python.lalrpop @@ -4,10 +4,12 @@ // See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword #![allow(unknown_lints,clippy)] -use super::ast; -use super::lexer; -use super::parser; use std::iter::FromIterator; + +use crate::ast; +use crate::fstring::parse_fstring; +use crate::lexer; + use num_bigint::BigInt; grammar; @@ -1008,7 +1010,7 @@ StringGroup: ast::StringGroup = { let mut values = vec![]; for (value, is_fstring) in s { values.push(if is_fstring { - parser::parse_fstring(&value)? + parse_fstring(&value)? } else { ast::StringGroup::Constant { value } }) From ddc154a1dda4f1f7ba04b3a5a50e151c714d9e98 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 24 Feb 2019 13:46:50 -0800 Subject: [PATCH 12/13] f-strings: support conversion flags --- parser/src/ast.rs | 12 ++++++++++++ parser/src/fstring.rs | 36 ++++++++++++++++++++++++++---------- tests/snippets/fstrings.py | 19 +++++++++++++++++++ vm/src/bytecode.rs | 6 +++++- vm/src/compile.rs | 11 +++++++++-- vm/src/frame.rs | 11 +++++++++-- 6 files changed, 80 insertions(+), 15 deletions(-) diff --git a/parser/src/ast.rs b/parser/src/ast.rs index e607c4e4b..749003d9a 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -357,6 +357,17 @@ pub enum Number { Complex { real: f64, imag: f64 }, } +/// Transforms a value prior to formatting it. +#[derive(Copy, Clone, Debug, PartialEq)] +pub enum ConversionFlag { + /// Converts by calling `str()`. + Str, + /// Converts by calling `ascii()`. + Ascii, + /// Converts by calling `repr()`. + Repr, +} + #[derive(Debug, PartialEq)] pub enum StringGroup { Constant { @@ -364,6 +375,7 @@ pub enum StringGroup { }, FormattedValue { value: Box, + conversion: Option, spec: String, }, Joined { diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index d5881cdfa..34ac22d2d 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -4,10 +4,11 @@ use std::str; use lalrpop_util::ParseError as LalrpopError; -use crate::ast::StringGroup; +use crate::ast::{ConversionFlag, StringGroup}; use crate::lexer::{LexicalError, Location, Tok}; use crate::parser::parse_expression; +use self::FStringError::*; use self::StringGroup::*; // TODO: consolidate these with ParseError @@ -16,6 +17,7 @@ pub enum FStringError { UnclosedLbrace, UnopenedRbrace, InvalidExpression, + InvalidConversionFlag, } impl From for LalrpopError { @@ -41,9 +43,23 @@ impl<'a> FStringParser<'a> { let mut expression = String::new(); let mut spec = String::new(); let mut depth = 0; + let mut conversion = None; while let Some(ch) = self.chars.next() { match ch { + '!' if depth == 0 => { + conversion = Some(match self.chars.next() { + Some('s') => ConversionFlag::Str, + Some('a') => ConversionFlag::Ascii, + Some('r') => ConversionFlag::Repr, + Some(_) => { + return Err(InvalidConversionFlag); + } + None => { + break; + } + }) + } ':' if depth == 0 => { while let Some(&next) = self.chars.peek() { if next != '}' { @@ -74,8 +90,9 @@ impl<'a> FStringParser<'a> { return Ok(FormattedValue { value: Box::new( parse_expression(expression.trim()) - .map_err(|_| FStringError::InvalidExpression)?, + .map_err(|_| InvalidExpression)?, ), + conversion, spec, }); } @@ -86,7 +103,7 @@ impl<'a> FStringParser<'a> { } } - return Err(FStringError::UnclosedLbrace); + return Err(UnclosedLbrace); } fn parse(mut self) -> Result { @@ -114,7 +131,7 @@ impl<'a> FStringParser<'a> { self.chars.next(); content.push('}'); } else { - return Err(FStringError::UnopenedRbrace); + return Err(UnopenedRbrace); } } _ => { @@ -164,10 +181,12 @@ mod tests { values: vec![ FormattedValue { value: Box::new(mk_ident("a")), + conversion: None, spec: String::new(), }, FormattedValue { value: Box::new(mk_ident("b")), + conversion: None, spec: String::new(), }, Constant { @@ -190,11 +209,8 @@ mod tests { #[test] fn test_parse_invalid_fstring() { - assert_eq!(parse_fstring("{"), Err(FStringError::UnclosedLbrace)); - assert_eq!(parse_fstring("}"), Err(FStringError::UnopenedRbrace)); - assert_eq!( - parse_fstring("{class}"), - Err(FStringError::InvalidExpression) - ); + assert_eq!(parse_fstring("{"), Err(UnclosedLbrace)); + assert_eq!(parse_fstring("}"), Err(UnopenedRbrace)); + assert_eq!(parse_fstring("{class}"), Err(InvalidExpression)); } } diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index 2ee45742f..eb00c1cde 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -15,3 +15,22 @@ assert f'{foo}' f"{foo}" 'foo' == 'barbarfoo' #assert f"{1 != 2}" == 'True' assert fr'x={4*10}\n' == 'x=40\\n' assert f'{16:0>+#10x}' == '00000+0x10' + + +# conversion flags + +class Value: + def __format__(self, spec): + return "foo" + + def __repr__(self): + return "bar" + + def __str__(self): + return "baz" + +v = Value() + +assert f'{v}' == 'foo' +assert f'{v!r}' == 'bar' +assert f'{v!s}' == 'baz' diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 1e0b85f13..b7cefe0b3 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -169,6 +169,7 @@ pub enum Instruction { }, Unpack, FormatValue { + conversion: Option, spec: String, }, } @@ -361,7 +362,10 @@ impl Instruction { UnpackSequence { size } => w!(UnpackSequence, size), UnpackEx { before, after } => w!(UnpackEx, before, after), Unpack => w!(Unpack), - FormatValue { spec } => w!(FormatValue, spec), + FormatValue { + conversion: _, + spec, + } => w!(FormatValue, spec), // TODO: write conversion } } } diff --git a/vm/src/compile.rs b/vm/src/compile.rs index 866b0dad7..a562c1d3d 100644 --- a/vm/src/compile.rs +++ b/vm/src/compile.rs @@ -1352,9 +1352,16 @@ impl Compiler { }, }); } - ast::StringGroup::FormattedValue { value, spec } => { + ast::StringGroup::FormattedValue { + value, + conversion, + spec, + } => { self.compile_expression(value)?; - self.emit(Instruction::FormatValue { spec: spec.clone() }); + self.emit(Instruction::FormatValue { + conversion: *conversion, + spec: spec.clone(), + }); } } Ok(()) diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 5de7f6809..f97be6894 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -654,8 +654,15 @@ impl Frame { } Ok(None) } - bytecode::Instruction::FormatValue { spec } => { - let value = self.pop_value(); + bytecode::Instruction::FormatValue { conversion, spec } => { + use ast::ConversionFlag::*; + let value = match conversion { + Some(Str) => vm.to_str(&self.pop_value())?, + Some(Repr) => vm.to_repr(&self.pop_value())?, + Some(Ascii) => self.pop_value(), // TODO + None => self.pop_value(), + }; + let spec = vm.new_str(spec.clone()); let formatted = vm.call_method(&value, "__format__", vec![spec])?; self.push_value(formatted); From f050acba736af99190b8dec8b6908aa44a050940 Mon Sep 17 00:00:00 2001 From: Joey Hain Date: Sun, 24 Feb 2019 17:24:17 -0800 Subject: [PATCH 13/13] f-strings: allow ':' and '!' to be used in the expression --- parser/src/fstring.rs | 68 ++++++++++++++++++++++++-------------- tests/snippets/fstrings.py | 9 +++-- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 34ac22d2d..e6e5586b0 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -18,6 +18,8 @@ pub enum FStringError { UnopenedRbrace, InvalidExpression, InvalidConversionFlag, + EmptyExpression, + MismatchedDelimiter, } impl From for LalrpopError { @@ -42,12 +44,12 @@ impl<'a> FStringParser<'a> { fn parse_formatted_value(&mut self) -> Result { let mut expression = String::new(); let mut spec = String::new(); - let mut depth = 0; + let mut delims = Vec::new(); let mut conversion = None; while let Some(ch) = self.chars.next() { match ch { - '!' if depth == 0 => { + '!' if delims.is_empty() => { conversion = Some(match self.chars.next() { Some('s') => ConversionFlag::Str, Some('a') => ConversionFlag::Ascii, @@ -60,7 +62,7 @@ impl<'a> FStringParser<'a> { } }) } - ':' if depth == 0 => { + ':' if delims.is_empty() => { while let Some(&next) = self.chars.peek() { if next != '}' { spec.push(next); @@ -70,31 +72,47 @@ impl<'a> FStringParser<'a> { } } } - '{' => { - if let Some('{') = self.chars.peek() { - expression.push_str("{{"); - self.chars.next(); - } else { - expression.push('{'); - depth += 1; + '(' | '{' | '[' => { + expression.push(ch); + delims.push(ch); + } + ')' => { + if delims.pop() != Some('(') { + return Err(MismatchedDelimiter); } + expression.push(ch); + } + ']' => { + if delims.pop() != Some('[') { + return Err(MismatchedDelimiter); + } + expression.push(ch); + } + '}' if !delims.is_empty() => { + if delims.pop() != Some('{') { + return Err(MismatchedDelimiter); + } + expression.push(ch); } '}' => { - if let Some('}') = self.chars.peek() { - expression.push_str("}}"); - self.chars.next(); - } else if depth > 0 { - expression.push('}'); - depth -= 1; - } else { - return Ok(FormattedValue { - value: Box::new( - parse_expression(expression.trim()) - .map_err(|_| InvalidExpression)?, - ), - conversion, - spec, - }); + if expression.is_empty() { + return Err(EmptyExpression); + } + return Ok(FormattedValue { + value: Box::new( + parse_expression(expression.trim()).map_err(|_| InvalidExpression)?, + ), + conversion, + spec, + }); + } + '"' | '\'' => { + expression.push(ch); + while let Some(next) = self.chars.next() { + expression.push(next); + if next == ch { + break; + } } } _ => { diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index eb00c1cde..c76967acb 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -11,10 +11,15 @@ assert f"{f'{{}}'}" == '{}' # don't include escaped braces in nested f-strings assert f'{f"{{"}' == '{' assert f'{f"}}"}' == '}' assert f'{foo}' f"{foo}" 'foo' == 'barbarfoo' -#assert f'{"!:"}' == '!:' -#assert f"{1 != 2}" == 'True' +assert f'{"!:"}' == '!:' assert fr'x={4*10}\n' == 'x=40\\n' assert f'{16:0>+#10x}' == '00000+0x10' +assert f"{{{(lambda x: f'hello, {x}')('world}')}" == '{hello, world}' + +# Normally `!` cannot appear outside of delimiters in the expression but +# cpython makes an exception for `!=`, so we should too. + +# assert f'{1 != 2}' == 'True' # conversion flags