mschoi/RustPython
1
0
Fork 0
forked from Rust-related/RustPython
Code Pull Requests Actions Activity

Merge pull request #1491 from RustPython/bytecode-compression

Browse Source 
Compress bytecode using LZ4
This commit is contained in:
Windel Bouwman
2019-10-10 08:04:21 +02:00
committed by GitHub
parent b769675c74 64ab147f64
commit 161f1d4c98
8 changed files with 36 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
Cargo.lock generated
View File

@@ -205,6 +205,11 @@ name = "byte-tools"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "byteorder"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "byteorder"
version = "1.3.2"
@@ -675,6 +680,15 @@ dependencies = [
"cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "lz4-compress"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "maplit"
version = "1.0.2"
@@ -1150,7 +1164,9 @@ dependencies = [
name = "rustpython-bytecode"
version = "0.1.1"
dependencies = [
"bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lz4-compress 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"num-bigint 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"num-complex 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1173,7 +1189,6 @@ dependencies = [
name = "rustpython-derive"
version = "0.1.1"
dependencies = [
"bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro-hack 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1204,7 +1219,6 @@ version = "0.1.1"
dependencies = [
"adler32 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"arr_macro 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"blake2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2083,6 +2097,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum build_const 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39"
"checksum bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad807f2fc2bf185eeb98ff3a901bd46dc5ad58163d0fa4577ba0d25674d71708"
"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
"checksum byteorder 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0fc10e8cc6b2580fda3f36eb6dc5316657f812a3df879a44a66fc9f0fdbc4855"
"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
"checksum caseless 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "808dab3318747be122cb31d36de18d4d1c81277a76f8332a02b81a3d73463d7f"
"checksum cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)" = "4fc9a35e1f4290eb9e5fc54ba6cf40671ed2a2514c3eeb2b2a908dda2ea5a1be"
@@ -2141,6 +2156,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum libz-sys 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "2eb5e43362e38e2bca2fd5f5134c4d4564a23a5c28e9b95411652021a8675ebe"
"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
"checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7"
"checksum lz4-compress 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f966533a922a9bba9e95e594c1fdb3b9bf5fdcdb11e37e51ad84cd76e468b91"
"checksum maplit 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
"checksum md-5 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a18af3dcaf2b0219366cdb4e2af65a6101457b415c3d1a5c71dd9c2b7c77b9c8"

2
bytecode/Cargo.toml
View File

@@ -9,7 +9,9 @@ license = "MIT"
[dependencies]
bincode = "1.1"
bitflags = "1.1"
lz4-compress = "0.1.1"
num-bigint = { version = "0.2", features = ["serde"] }
num-complex = { version = "0.2", features = ["serde"] }
serde = { version = "1.0", features = ["derive"] }

12
bytecode/src/bytecode.rs
View File

@@ -378,6 +378,18 @@ impl CodeObject {
}
}
/// Load a code object from bytes
pub fn from_bytes(data: &[u8]) -> Result<Self, Box<dyn std::error::Error>> {
let data = lz4_compress::decompress(data)?;
bincode::deserialize::<Self>(&data).map_err(|e| e.into())
}
/// Serialize this bytecode to bytes.
pub fn to_bytes(&self) -> Vec<u8> {
let data = bincode::serialize(&self).expect("Code object must be serializable");
lz4_compress::compress(&data)
}
pub fn get_constants(&self) -> impl Iterator<Item = &Constant> {
self.instructions.iter().filter_map(|x| {
if let Instruction::LoadConst { value } = x {

1
derive/Cargo.toml
View File

@@ -19,6 +19,5 @@ quote = "0.6.11"
proc-macro2 = "0.4.27"
rustpython-compiler = { path = "../compiler", version = "0.1.1" }
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
bincode = "1.1"
proc-macro-hack = { version = "0.5", optional = true }
maplit = "1.0"

6
derive/src/compile_bytecode.rs
View File

@@ -14,7 +14,6 @@
//! ```
use crate::{extract_spans, Diagnostic};
use bincode;
use proc_macro2::{Span, TokenStream as TokenStream2};
use quote::quote;
use rustpython_bytecode::bytecode::{CodeObject, FrozenModule};
@@ -249,11 +248,11 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Dia
.into_iter()
.map(|(module_name, FrozenModule { code, package })| {
let module_name = LitStr::new(&module_name, Span::call_site());
let bytes = bincode::serialize(&code).expect("Failed to serialize");
let bytes = code.to_bytes();
let bytes = LitByteStr::new(&bytes, Span::call_site());
quote! {
#module_name.into() => ::rustpython_vm::bytecode::FrozenModule {
code: bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(
code: ::rustpython_vm::bytecode::CodeObject::from_bytes(
#bytes
).expect("Deserializing CodeObject failed"),
package: #package,
@@ -263,7 +262,6 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Dia
let output = quote! {
({
use ::rustpython_vm::__exports::bincode;
use ::rustpython_vm::__exports::hashmap;
hashmap! {
#(#modules),*

1
vm/Cargo.toml
View File

@@ -52,7 +52,6 @@ hex = "0.3.2"
hexf-parse = "0.1.0"
indexmap = "1.0.2"
crc = "^1.0.0"
bincode = "1.1.4"
unicode_categories = "0.1.1"
unicode_names2 = "0.2.2"
unicode-casing = "0.1.0"

1
vm/src/lib.rs
View File

@@ -79,6 +79,5 @@ pub use rustpython_bytecode::*;
#[doc(hidden)]
pub mod __exports {
pub use bincode;
pub use maplit::hashmap;
}

4
vm/src/stdlib/marshal.rs
View File

@@ -5,11 +5,11 @@ use crate::pyobject::{PyObjectRef, PyResult};
use crate::vm::VirtualMachine;
fn marshal_dumps(co: PyCodeRef, _vm: &VirtualMachine) -> PyBytes {
PyBytes::new(bincode::serialize(&co.code).unwrap())
PyBytes::new(co.code.to_bytes())
}
fn marshal_loads(code_bytes: PyBytesRef, vm: &VirtualMachine) -> PyResult<PyCode> {
let code = bincode::deserialize::<bytecode::CodeObject>(&code_bytes)
let code = bytecode::CodeObject::from_bytes(&code_bytes)
.map_err(|_| vm.new_value_error("Couldn't deserialize python bytecode".to_owned()))?;
Ok(PyCode { code })
}