From d32cd5394813fd8a8936f428c6618f8d152db812 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 25 Aug 2022 05:02:50 +0900 Subject: [PATCH] Move pylib dependency out of vm --- Cargo.lock | 3 ++- Cargo.toml | 7 +++--- pylib/Lib | 1 + pylib/src/lib.rs | 21 ++---------------- src/lib.rs | 47 +++++++++++++++++++++++++++------------- vm/Cargo.toml | 3 +-- vm/src/frozen.rs | 24 +++++++++++++------- vm/src/vm/mod.rs | 2 +- vm/src/vm/setting.rs | 2 +- wasm/lib/Cargo.toml | 6 +++-- wasm/lib/src/vm_class.rs | 6 +++++ 11 files changed, 70 insertions(+), 52 deletions(-) create mode 120000 pylib/Lib diff --git a/Cargo.lock b/Cargo.lock index 00dba1aebe..638f9d2e8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1725,6 +1725,7 @@ dependencies = [ "python3-sys", "rustpython-compiler", "rustpython-parser", + "rustpython-pylib", "rustpython-stdlib", "rustpython-vm", "rustyline", @@ -2002,7 +2003,6 @@ dependencies = [ "rustpython-derive", "rustpython-jit", "rustpython-parser", - "rustpython-pylib", "rustyline", "schannel", "serde", @@ -2036,6 +2036,7 @@ dependencies = [ "parking_lot", "rustpython-common", "rustpython-parser", + "rustpython-pylib", "rustpython-stdlib", "rustpython-vm", "serde", diff --git a/Cargo.toml b/Cargo.toml index c655a2d1f6..4de697a72b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,9 +22,9 @@ members = [ default = ["threading", "stdlib", "zlib", "importlib", "encodings", "rustpython-parser/lalrpop"] importlib = ["rustpython-vm/importlib"] encodings = ["rustpython-vm/encodings"] -stdlib = ["rustpython-stdlib"] +stdlib = ["rustpython-stdlib", "rustpython-pylib"] flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] -freeze-stdlib = ["rustpython-vm/freeze-stdlib"] +freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] zlib = ["stdlib", "rustpython-stdlib/zlib"] @@ -35,7 +35,8 @@ ssl-vendor = ["rustpython-stdlib/ssl-vendor"] [dependencies] rustpython-compiler = { path = "compiler", version = "0.1.1" } rustpython-parser = { path = "compiler/parser", version = "0.1.1" } -rustpython-stdlib = {path = "stdlib", optional = true, default-features = false} +rustpython-pylib = { path = "pylib", optional = true, default-features = false } +rustpython-stdlib = { path = "stdlib", optional = true, default-features = false } rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] } cfg-if = "1.0.0" diff --git a/pylib/Lib b/pylib/Lib new file mode 120000 index 0000000000..47f928ff4d --- /dev/null +++ b/pylib/Lib @@ -0,0 +1 @@ +../Lib \ No newline at end of file diff --git a/pylib/src/lib.rs b/pylib/src/lib.rs index d11f304146..4ce3c8a7e0 100644 --- a/pylib/src/lib.rs +++ b/pylib/src/lib.rs @@ -6,27 +6,10 @@ // so build.rs sets this env var pub const LIB_PATH: &str = match option_env!("win_lib_path") { Some(s) => s, - None => concat!(env!("CARGO_MANIFEST_DIR"), "/../../Lib"), + None => concat!(env!("CARGO_MANIFEST_DIR"), "/../Lib"), }; -use rustpython_compiler_core::FrozenModule; - -pub fn frozen_builtins() -> impl Iterator { - rustpython_derive::py_freeze!( - dir = "../vm/Lib/python_builtins", - crate_name = "rustpython_compiler_core" - ) -} - -#[cfg(not(feature = "freeze-stdlib"))] -pub fn frozen_core() -> impl Iterator { - rustpython_derive::py_freeze!( - dir = "../vm/Lib/core_modules", - crate_name = "rustpython_compiler_core" - ) -} - #[cfg(feature = "freeze-stdlib")] -pub fn frozen_stdlib() -> impl Iterator { +pub fn frozen_stdlib() -> impl Iterator { rustpython_derive::py_freeze!(dir = "../Lib", crate_name = "rustpython_compiler_core") } diff --git a/src/lib.rs b/src/lib.rs index fae4c9e3be..8d7f662c65 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -248,6 +248,38 @@ fn add_stdlib(vm: &mut VirtualMachine) { let _ = vm; #[cfg(feature = "stdlib")] vm.add_native_modules(rustpython_stdlib::get_module_inits()); + + // if we're on freeze-stdlib, the core stdlib modules will be included anyway + #[cfg(feature = "freeze-stdlib")] + vm.add_frozen(rustpython_pylib::frozen_stdlib()); + + #[cfg(not(feature = "freeze-stdlib"))] + { + use rustpython_vm::common::rc::PyRc; + let state = PyRc::get_mut(&mut vm.state).unwrap(); + + #[allow(clippy::needless_collect)] // false positive + let path_list: Vec<_> = state.settings.path_list.drain(..).collect(); + + // add the current directory to sys.path + state.settings.path_list.push("".to_owned()); + + // BUILDTIME_RUSTPYTHONPATH should be set when distributing + if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") { + state + .settings + .path_list + .extend(split_paths(paths).map(|path| path.into_os_string().into_string().unwrap())) + } else { + #[cfg(feature = "rustpython-pylib")] + state + .settings + .path_list + .push(rustpython_pylib::LIB_PATH.to_owned()) + } + + state.settings.path_list.extend(path_list.into_iter()); + } } /// Create settings by examining command line arguments and environment @@ -264,21 +296,6 @@ fn create_settings(matches: &ArgMatches) -> Settings { let ignore_environment = settings.ignore_environment || settings.isolated; - // when rustpython-vm/pylib is enabled, Settings::default().path_list has pylib::LIB_PATH - let maybe_pylib = settings.path_list.pop(); - - // add the current directory to sys.path - settings.path_list.push("".to_owned()); - - // BUILDTIME_RUSTPYTHONPATH should be set when distributing - if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") { - settings - .path_list - .extend(split_paths(paths).map(|path| path.into_os_string().into_string().unwrap())) - } else { - settings.path_list.extend(maybe_pylib); - } - if !ignore_environment { settings.path_list.extend(get_paths("RUSTPYTHONPATH")); settings.path_list.extend(get_paths("PYTHONPATH")); diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 83c05c4823..ccbeb8a4b2 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -14,7 +14,7 @@ importlib = [] encodings = ["importlib"] vm-tracing-logging = [] flame-it = ["flame", "flamer"] -freeze-stdlib = ["rustpython-pylib/freeze-stdlib"] +freeze-stdlib = [] jit = ["rustpython-jit"] threading = ["rustpython-common/threading"] compiler = ["parser", "codegen", "rustpython-compiler"] @@ -31,7 +31,6 @@ rustpython-compiler-core = { path = "../compiler/core", version = "0.1.2" } rustpython-common = { path = "../common" } rustpython-derive = { path = "../derive", version = "0.1.2" } rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" } -rustpython-pylib = { path = "../pylib", version = "0.1.0" } num-complex = { version = "0.4.0", features = ["serde"] } num-bigint = { version = "0.4.3", features = ["serde"] } diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs index fa694ef4c5..44bcf23c57 100644 --- a/vm/src/frozen.rs +++ b/vm/src/frozen.rs @@ -1,6 +1,6 @@ use crate::bytecode::FrozenModule; -pub fn get_module_inits() -> impl Iterator { +pub fn core_frozen_inits() -> impl Iterator { let iter = std::iter::empty(); macro_rules! ext_modules { ($iter:ident, ($modules:expr)) => { @@ -20,16 +20,24 @@ pub fn get_module_inits() -> impl Iterator { // Python modules that the vm calls into, but are not actually part of the stdlib. They could // in theory be implemented in Rust, but are easiest to do in Python for one reason or another. // Includes _importlib_bootstrap and _importlib_bootstrap_external - ext_modules!(iter, (rustpython_pylib::frozen_builtins())); + ext_modules!( + iter, + (rustpython_derive::py_freeze!( + dir = "./Lib/python_builtins", + crate_name = "rustpython_compiler_core" + )) + ); - #[cfg(not(feature = "freeze-stdlib"))] // core stdlib Python modules that the vm calls into, but are still used in Python // application code, e.g. copyreg - ext_modules!(iter, (rustpython_pylib::frozen_core())); - - // if we're on freeze-stdlib, the core stdlib modules will be included anyway - #[cfg(feature = "freeze-stdlib")] - ext_modules!(iter, (rustpython_pylib::frozen_stdlib())); + #[cfg(not(feature = "freeze-stdlib"))] + ext_modules!( + iter, + (rustpython_derive::py_freeze!( + dir = "./Lib/core_modules", + crate_name = "rustpython_compiler_core" + )) + ); iter } diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index 619b9dcb0f..b1a1332b8c 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -176,7 +176,7 @@ impl VirtualMachine { panic!("Interpreters in same process must share the hash seed"); } - let frozen = frozen::get_module_inits().collect(); + let frozen = frozen::core_frozen_inits().collect(); PyRc::get_mut(&mut vm.state).unwrap().frozen = frozen; vm.builtins diff --git a/vm/src/vm/setting.rs b/vm/src/vm/setting.rs index 5ebcce01b8..62b0d6848c 100644 --- a/vm/src/vm/setting.rs +++ b/vm/src/vm/setting.rs @@ -89,7 +89,7 @@ impl Default for Settings { dev_mode: false, warn_default_encoding: false, warnopts: vec![], - path_list: vec![rustpython_pylib::LIB_PATH.to_owned()], + path_list: vec![], argv: vec![], hash_seed: None, stdio_unbuffered: false, diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 5109ea30d7..1145f63124 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -11,13 +11,15 @@ edition = "2021" crate-type = ["cdylib", "rlib"] [features] -default = ["freeze-stdlib", "rustpython-stdlib"] -freeze-stdlib = ["rustpython-vm/freeze-stdlib"] +default = ["stdlib"] +stdlib = ["freeze-stdlib", "rustpython-pylib", "rustpython-stdlib"] +freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] no-start-func = [] [dependencies] rustpython-common = { path = "../../common" } rustpython-parser = { path = "../../compiler/parser" } +rustpython-pylib = { path = "../../pylib", default-features = false, optional = true } rustpython-stdlib = { path = "../../stdlib", default-features = false, optional = true } # make sure no threading! otherwise wasm build will fail rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings"] } diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 30f586d857..1ce47a6bbb 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -42,6 +42,12 @@ impl StoredVirtualMachine { let mut settings = Settings::default(); settings.allow_external_library = false; let interp = Interpreter::with_init(settings, |vm| { + #[cfg(feature = "stdlib")] + vm.add_native_modules(rustpython_stdlib::get_module_inits()); + + #[cfg(feature = "freeze-stdlib")] + vm.add_frozen(rustpython_pylib::frozen_stdlib()); + vm.wasm_id = Some(id); js_module::setup_js_module(vm);