diff --git a/Cargo.lock b/Cargo.lock index 272786ef3..44ad4ceed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,6 +78,16 @@ dependencies = [ "libc 0.2.50 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "bincode" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "bit-set" version = "0.5.1" @@ -509,6 +519,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -517,6 +528,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -875,6 +887,7 @@ dependencies = [ "num-bigint 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "unic-emoji-char 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -883,6 +896,7 @@ dependencies = [ name = "rustpython_vm" version = "0.1.0" dependencies = [ + "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -904,7 +918,6 @@ dependencies = [ "rustpython_derive 0.1.0", "rustpython_parser 0.0.1", "serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)", "statrs 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-casing 0.1.0 (git+https://github.com/OddCoincidence/unicode-casing?rev=90d6d1f02b9cc04ffb55a5f1c3fa1455a84231fb)", @@ -970,6 +983,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" name = "serde" version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "serde_derive" @@ -1423,6 +1439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum autocfg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a6d640bee2da49f60a4068a7fae53acde8982514ab7bae8b8cea9e88cbcfd799" "checksum backtrace 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "cd5a90e2b463010cd0e0ce9a11d4a9d5d58d9f41d4a6ba3dcaf9e68b466e88b4" "checksum backtrace-sys 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)" = "797c830ac25ccc92a7f8a7b9862bde440715531514594a6154e3d4a54dd769b6" +"checksum bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "9f04a5e50dc80b3d5d35320889053637d15011aed5e66b66b37ae798c65da6f7" "checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80" "checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb" "checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" diff --git a/parser/Cargo.toml b/parser/Cargo.toml index e3c4e36bd..6cc7805c6 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -16,3 +16,4 @@ num-bigint = "0.2" num-traits = "0.2" unicode-xid = "0.1.0" unic-emoji-char = "0.9.0" +serde = { version = "1.0.66", features = ["derive"] } diff --git a/parser/src/ast.rs b/parser/src/ast.rs index ec0677fe8..cf7676a3b 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -4,6 +4,8 @@ pub use super::lexer::Location; use num_bigint::BigInt; +use serde::{Serialize, Deserialize}; + /* #[derive(Debug)] @@ -381,7 +383,7 @@ pub enum Number { } /// Transforms a value prior to formatting it. -#[derive(Copy, Clone, Debug, PartialEq)] +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum ConversionFlag { /// Converts by calling `str()`. Str, diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index f95563eac..269864daf 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -12,6 +12,7 @@ use std::collections::HashMap; use std::str::FromStr; use unic_emoji_char::is_emoji_presentation; use unicode_xid::UnicodeXID; +use serde::{Serialize, Deserialize}; #[derive(Clone, Copy, PartialEq, Debug)] struct IndentationLevel { @@ -71,7 +72,7 @@ pub enum LexicalErrorType { OtherError(String), } -#[derive(Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct Location { row: usize, column: usize, diff --git a/tests/snippets/stdlib_marshal.py b/tests/snippets/stdlib_marshal.py new file mode 100644 index 000000000..118bb1bdd --- /dev/null +++ b/tests/snippets/stdlib_marshal.py @@ -0,0 +1,7 @@ +import marshal +orig = compile("1 + 1", "", 'eval') + +dumped = marshal.dumps(orig) +loaded = marshal.loads(dumped) + +assert eval(loaded) == eval(orig) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 5ee8e81c4..4f6e4b1b2 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -6,8 +6,8 @@ edition = "2018" [dependencies] bitflags = "1.0.4" -num-complex = "0.2" -num-bigint = "0.2.1" +num-complex = { version = "0.2", features = ["serde"] } +num-bigint = { version = "0.2.1", features = ["serde"] } num-traits = "0.2" num-integer = "0.1.39" num-rational = "0.2.1" @@ -15,8 +15,7 @@ rand = "0.5" log = "0.3" rustpython_derive = {path = "../derive"} rustpython_parser = {path = "../parser"} -serde = "1.0.66" -serde_derive = "1.0.66" +serde = { version = "1.0.66", features = ["derive"] } serde_json = "1.0.26" byteorder = "1.2.6" regex = "1" @@ -30,6 +29,7 @@ lexical = "2.0.0" itertools = "^0.8.0" hexf = "0.1.0" indexmap = "1.0.2" +bincode = "1.1.4" # TODO: release and publish to crates.io [dependencies.unicode-casing] diff --git a/vm/src/bytecode.rs b/vm/src/bytecode.rs index 09bb7e70a..5291cd0c3 100644 --- a/vm/src/bytecode.rs +++ b/vm/src/bytecode.rs @@ -10,10 +10,11 @@ use num_complex::Complex64; use rustpython_parser::ast; use std::collections::{HashMap, HashSet}; use std::fmt; +use serde::{Serialize, Deserialize}; /// Primary container of a single code object. Each python function has /// a codeobject. Also a module has a codeobject. -#[derive(Clone, PartialEq)] +#[derive(Clone, PartialEq, Serialize, Deserialize)] pub struct CodeObject { pub instructions: Vec, pub label_map: HashMap, @@ -29,6 +30,7 @@ pub struct CodeObject { } bitflags! { + #[derive(Serialize, Deserialize)] pub struct FunctionOpArg: u8 { const HAS_DEFAULTS = 0x01; const HAS_KW_ONLY_DEFAULTS = 0x02; @@ -38,7 +40,7 @@ bitflags! { pub type Label = usize; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum NameScope { Local, NonLocal, @@ -46,7 +48,7 @@ pub enum NameScope { } /// A Single bytecode instruction. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Instruction { Import { name: String, @@ -187,14 +189,14 @@ pub enum Instruction { use self::Instruction::*; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum CallType { Positional(usize), Keyword(usize), Ex(bool), } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Constant { Integer { value: BigInt }, Float { value: f64 }, @@ -208,7 +210,7 @@ pub enum Constant { Ellipsis, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum ComparisonOperator { Greater, GreaterOrEqual, @@ -222,7 +224,7 @@ pub enum ComparisonOperator { IsNot, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum BinaryOperator { Power, Multiply, @@ -240,7 +242,7 @@ pub enum BinaryOperator { Or, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum UnaryOperator { Not, Invert, @@ -248,7 +250,7 @@ pub enum UnaryOperator { Plus, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub enum Varargs { None, Unnamed, diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index 78c78e333..d88991b9e 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -294,7 +294,7 @@ impl PyContext { let zip_type = create_type("zip", &type_type, &object_type); let bool_type = create_type("bool", &type_type, &int_type); let memoryview_type = create_type("memoryview", &type_type, &object_type); - let code_type = create_type("code", &type_type, &int_type); + let code_type = create_type("code", &type_type, &object_type); let range_type = create_type("range", &type_type, &object_type); let rangeiterator_type = create_type("range_iterator", &type_type, &object_type); let slice_type = create_type("slice", &type_type, &object_type); diff --git a/vm/src/stdlib/marshal.rs b/vm/src/stdlib/marshal.rs new file mode 100644 index 000000000..047559881 --- /dev/null +++ b/vm/src/stdlib/marshal.rs @@ -0,0 +1,24 @@ +use crate::obj::objcode::{PyCodeRef, PyCode}; +use crate::pyobject::{PyObjectRef, PyResult, IntoPyObject}; +use crate::vm::VirtualMachine; +use crate::obj::objbytes::{PyBytes, PyBytesRef}; +use crate::bytecode; + +fn marshal_dumps(co: PyCodeRef, vm: &VirtualMachine) -> PyResult { + PyBytes::new(bincode::serialize(&co.code).unwrap()).into_pyobject(vm) +} + +fn marshal_loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult { + let code = bincode::deserialize::(&code_bytes).unwrap(); + let pycode = PyCode { code }; + pycode.into_pyobject(vm) +} + +pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { + let ctx = &vm.ctx; + + py_module!(vm, "marshal", { + "loads" => ctx.new_rustfunc(marshal_loads), + "dumps" => ctx.new_rustfunc(marshal_dumps) + }) +} \ No newline at end of file diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index e475ba057..ef2d7f3a5 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -5,6 +5,7 @@ mod imp; mod itertools; pub(crate) mod json; mod keyword; +mod marshal; mod math; mod platform; mod pystruct; @@ -42,6 +43,7 @@ pub fn get_module_inits() -> HashMap { modules.insert("itertools".to_string(), Box::new(itertools::make_module)); modules.insert("json".to_string(), Box::new(json::make_module)); modules.insert("keyword".to_string(), Box::new(keyword::make_module)); + modules.insert("marshal".to_string(), Box::new(marshal::make_module)); modules.insert("math".to_string(), Box::new(math::make_module)); modules.insert("platform".to_string(), Box::new(platform::make_module)); modules.insert("re".to_string(), Box::new(re::make_module));