mschoi/RustPython
1
0
Fork 0
forked from Rust-related/RustPython
Code Pull Requests Actions Activity

Split py_compile_bytecode into py_compile and py_freeze.

Browse Source
This commit is contained in:
Ben Lewis
2020-09-05 16:10:39 +12:00
parent a0d72943bf
commit c54b4bb833
9 changed files with 104 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Cargo.lock generated
View File

@@ -313,7 +313,7 @@ dependencies = [
"atty",
"bitflags",
"strsim 0.8.0",
"textwrap",
"textwrap 0.11.0",
"unicode-width",
"vec_map",
]
@@ -1649,6 +1649,7 @@ dependencies = [
"rustpython-compiler",
"syn",
"syn-ext",
"textwrap 0.12.1",
]
[[package]]
@@ -2062,6 +2063,15 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "textwrap"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789"
dependencies = [
"unicode-width",
]
[[package]]
name = "thiserror"
version = "1.0.20"

1
derive/Cargo.toml
View File

@@ -19,3 +19,4 @@ rustpython-compiler = { path = "../compiler", version = "0.1.1" }
rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
maplit = "1.0"
once_cell = "1.3.1"
textwrap = "0.12.1"

100
derive/src/compile_bytecode.rs
View File

@@ -1,6 +1,6 @@
//! Parsing and processing for this form:
//! ```ignore
//! py_compile_input!(
//! py_compile!(
//! // either:
//! source = "python_source_code",
//! // or
@@ -24,7 +24,8 @@ use std::env;
use std::fs;
use std::path::{Path, PathBuf};
use syn::parse::{Parse, ParseStream, Result as ParseResult};
use syn::{self, parse2, Lit, LitByteStr, LitStr, Meta, Token};
use syn::spanned::Spanned;
use syn::{self, parse2, Lit, LitByteStr, LitStr, Macro, Meta, MetaNameValue, Token};
static CARGO_MANIFEST_DIR: Lazy<PathBuf> = Lazy::new(|| {
PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"))
@@ -62,7 +63,25 @@ impl CompilationSource {
mode: compile::Mode,
module_name: String,
) -> Result<HashMap<String, FrozenModule>, Diagnostic> {
Ok(match &self.kind {
match &self.kind {
CompilationSourceKind::Dir(rel_path) => {
self.compile_dir(&CARGO_MANIFEST_DIR.join(rel_path), String::new(), mode)
}
_ => Ok(hashmap! {
module_name.clone() => FrozenModule {
code: self.compile_single(mode, module_name)?,
package: false,
},
}),
}
}
fn compile_single(
&self,
mode: compile::Mode,
module_name: String,
) -> Result<CodeObject, Diagnostic> {
match &self.kind {
CompilationSourceKind::File(rel_path) => {
let path = CARGO_MANIFEST_DIR.join(rel_path);
let source = fs::read_to_string(&path).map_err(|err| {
@@ -71,25 +90,17 @@ impl CompilationSource {
format!("Error reading file {:?}: {}", path, err),
)
})?;
hashmap! {
module_name.clone() => FrozenModule {
code: self.compile_string(&source, mode, module_name, || rel_path.display())?,
package: false,
},
}
self.compile_string(&source, mode, module_name, || rel_path.display())
}
CompilationSourceKind::SourceCode(code) => {
hashmap! {
module_name.clone() => FrozenModule {
code: self.compile_string(code, mode, module_name, || "string literal")?,
package: false,
},
}
self.compile_string(&textwrap::dedent(code), mode, module_name, || {
"string literal"
})
}
CompilationSourceKind::Dir(rel_path) => {
self.compile_dir(&CARGO_MANIFEST_DIR.join(rel_path), String::new(), mode)?
CompilationSourceKind::Dir(_) => {
unreachable!("Can't use compile_single with directory source")
}
})
}
}
fn compile_dir(
@@ -157,7 +168,7 @@ struct PyCompileInput {
}
impl PyCompileInput {
fn parse(&self) -> Result<PyCompileArgs, Diagnostic> {
fn parse(&self, allow_dir: bool) -> Result<PyCompileArgs, Diagnostic> {
let mut module_name = None;
let mut mode = None;
let mut source: Option<CompilationSource> = None;
@@ -214,6 +225,10 @@ impl PyCompileInput {
span: extract_spans(&name_value).unwrap(),
});
} else if ident == "dir" {
if !allow_dir {
bail_span!(ident, "py_compile doesn't accept dir")
}
assert_source_empty(&source)?;
let path = match &name_value.lit {
Lit::Str(s) => PathBuf::from(s.value()),
@@ -236,7 +251,7 @@ impl PyCompileInput {
let source = source.ok_or_else(|| {
Diagnostic::span_error(
self.span,
"Must have either file or source in py_compile_bytecode!()",
"Must have either file or source in py_compile!()/py_freeze!()",
)
})?;
@@ -249,11 +264,32 @@ impl PyCompileInput {
}
}
fn parse_meta(input: ParseStream) -> ParseResult<Meta> {
let path = input.call(syn::Path::parse_mod_style)?;
let eq_token: Token![=] = input.parse()?;
let span = input.span();
if input.peek(LitStr) {
Ok(Meta::NameValue(MetaNameValue {
path,
eq_token,
lit: Lit::Str(input.parse()?),
}))
} else if let Ok(mac) = input.parse::<Macro>() {
Ok(Meta::NameValue(MetaNameValue {
path,
eq_token,
lit: Lit::Str(LitStr::new(&mac.tokens.to_string(), mac.span())),
}))
} else {
Err(syn::Error::new(span, "Expected string or stringify macro"))
}
}
impl Parse for PyCompileInput {
fn parse(input: ParseStream) -> ParseResult<Self> {
let span = input.cursor().span();
let metas = input
.parse_terminated::<Meta, Token![,]>(Meta::parse)?
.parse_terminated::<Meta, Token![,]>(parse_meta)?
.into_iter()
.collect();
Ok(PyCompileInput { span, metas })
@@ -267,9 +303,27 @@ struct PyCompileArgs {
crate_name: syn::Ident,
}
pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result<TokenStream2, Diagnostic> {
pub fn impl_py_compile(input: TokenStream2) -> Result<TokenStream2, Diagnostic> {
let input: PyCompileInput = parse2(input)?;
let args = input.parse()?;
let args = input.parse(false)?;
let crate_name = args.crate_name;
let code = args.source.compile_single(args.mode, args.module_name)?;
let bytes = code.to_bytes();
let bytes = LitByteStr::new(&bytes, Span::call_site());
let output = quote! {
::#crate_name::bytecode::CodeObject::from_bytes(#bytes)
.expect("Deserializing CodeObject failed")
};
Ok(output)
}
pub fn impl_py_freeze(input: TokenStream2) -> Result<TokenStream2, Diagnostic> {
let input: PyCompileInput = parse2(input)?;
let args = input.parse(true)?;
let crate_name = args.crate_name;
let code_map = args.source.compile(args.mode, args.module_name)?;

9
derive/src/lib.rs
View File

@@ -62,6 +62,11 @@ pub fn pystruct_sequence(input: TokenStream) -> TokenStream {
}
#[proc_macro]
pub fn py_compile_bytecode(input: TokenStream) -> TokenStream {
result_to_tokens(compile_bytecode::impl_py_compile_bytecode(input.into()))
pub fn py_compile(input: TokenStream) -> TokenStream {
result_to_tokens(compile_bytecode::impl_py_compile(input.into()))
}
#[proc_macro]
pub fn py_freeze(input: TokenStream) -> TokenStream {
result_to_tokens(compile_bytecode::impl_py_freeze(input.into()))
}

2
examples/freeze/main.rs
View File

@@ -12,7 +12,7 @@ fn run(vm: &vm::VirtualMachine) -> vm::pyobject::PyResult<()> {
// the file parameter is relevant to the directory where the crate's Cargo.toml is located, see $CARGO_MANIFEST_DIR:
// https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates
let modules: HashMap<String, vm::bytecode::FrozenModule> =
vm::py_compile_bytecode!(file = "examples/freeze/freeze.py");
vm::py_freeze!(file = "examples/freeze/freeze.py");
let res = vm.run_code_obj(
vm.ctx

12
examples/mini_repl.rs
View File

@@ -10,7 +10,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
// this needs to be in scope in order to insert things into scope.globals
use vm::pyobject::ItemProtocol;
// This has to be a macro because it uses the py_compile_bytecode macro,
// This has to be a macro because it uses the py_compile macro,
// which compiles python source to optimized bytecode at compile time, so that
// the program you're embedding this into doesn't take longer to start up.
macro_rules! add_python_function {
@@ -19,15 +19,7 @@ macro_rules! add_python_function {
// (a PyRef is a special reference that points to something in the VirtualMachine)
use vm::pyobject::PyValue;
// you can safely assume that only one module will be created when passing a source literal
// to py_compile_bytecode. However, it is also possible to pass directories, which may
// return more modules.
let (_, vm::bytecode::FrozenModule { code, .. }): (String, _) =
vm::py_compile_bytecode!(source = $src)
.into_iter()
.collect::<Vec<_>>()
.pop()
.expect("No modules found in the provided source!");
let code = vm::py_compile!(source = $src);
// takes the first constant in the file that's a function
let def = code

2
vm/pylib-crate/src/lib.rs
View File

@@ -13,5 +13,5 @@ use {
};
#[cfg(feature = "compiled-bytecode")]
pub fn frozen_stdlib() -> HashMap<String, FrozenModule> {
rustpython_derive::py_compile_bytecode!(dir = "Lib", crate_name = "rustpython_pylib")
rustpython_derive::py_freeze!(dir = "Lib", crate_name = "rustpython_pylib")
}

2
vm/src/frozen.rs
View File

@@ -6,7 +6,7 @@ pub fn get_module_inits() -> HashMap<String, FrozenModule> {
macro_rules! ext_modules {
($($t:tt)*) => {
modules.extend(py_compile_bytecode!($($t)*));
modules.extend(py_freeze!($($t)*));
};
}

7
wasm/lib/src/browser_module.rs
View File

@@ -382,8 +382,7 @@ pub fn setup_browser_module(vm: &mut VirtualMachine) {
state
.stdlib_inits
.insert("_browser".to_owned(), Box::new(make_module));
state.frozen.extend(py_compile_bytecode!(
file = "src/browser.py",
module_name = "browser",
));
state
.frozen
.extend(py_freeze!(file = "src/browser.py", module_name = "browser",));
}