Merge pull request #4133 from youknowone/pylib

Split pylib from vm

Cargo.lock

@@ -1725,6 +1725,7 @@ dependencies = [
  "python3-sys",
  "rustpython-compiler",
  "rustpython-parser",
+ "rustpython-pylib",
  "rustpython-stdlib",
  "rustpython-vm",
  "rustyline",
@@ -2002,7 +2003,6 @@ dependencies = [
  "rustpython-derive",
  "rustpython-jit",
  "rustpython-parser",
- "rustpython-pylib",
  "rustyline",
  "schannel",
  "serde",
@@ -2036,6 +2036,7 @@ dependencies = [
  "parking_lot",
  "rustpython-common",
  "rustpython-parser",
+ "rustpython-pylib",
  "rustpython-stdlib",
  "rustpython-vm",
  "serde",

Cargo.toml

@@ -15,16 +15,16 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 resolver = "2"
 members = [
     "compiler", "compiler/ast", "compiler/core", "compiler/codegen", "compiler/parser",
-    ".", "common", "derive", "jit", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
+    ".", "common", "derive", "jit", "vm", "pylib", "stdlib", "wasm/lib",
 ]
 
 [features]
 default = ["threading", "stdlib", "zlib", "importlib", "encodings", "rustpython-parser/lalrpop"]
 importlib = ["rustpython-vm/importlib"]
 encodings = ["rustpython-vm/encodings"]
-stdlib = ["rustpython-stdlib"]
+stdlib = ["rustpython-stdlib", "rustpython-pylib"]
 flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
-freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
+freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 jit = ["rustpython-vm/jit"]
 threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"]
 zlib = ["stdlib", "rustpython-stdlib/zlib"]
@@ -35,7 +35,8 @@ ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
 [dependencies]
 rustpython-compiler = { path = "compiler", version = "0.1.1" }
 rustpython-parser = { path = "compiler/parser", version = "0.1.1" }
-rustpython-stdlib = {path = "stdlib", optional = true, default-features = false}
+rustpython-pylib = { path = "pylib", optional = true, default-features = false }
+rustpython-stdlib = { path = "stdlib", optional = true, default-features = false }
 rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
 
 cfg-if = "1.0.0"

pylib/Cargo.toml

@@ -12,8 +12,8 @@ include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"
 freeze-stdlib = []
 
 [dependencies]
-rustpython-compiler-core = { version = "0.1.2", path = "../../compiler/core" }
-rustpython-derive = { version = "0.1.2", path = "../../derive" }
+rustpython-compiler-core = { version = "0.1.2", path = "../compiler/core" }
+rustpython-derive = { version = "0.1.2", path = "../derive" }
 
 [build-dependencies]
 glob = "0.3"

pylib/Lib

@@ -0,0 +1 @@
+../Lib

pylib/src/lib.rs

@@ -0,0 +1,15 @@
+//! This crate includes the compiled python bytecode of the RustPython standard library. The most
+//! common way to use this crate is to just add the `"freeze-stdlib"` feature to `rustpython-vm`,
+//! in order to automatically include the python part of the standard library into the binary.
+
+// windows needs to read the symlink out of `Lib` as git turns it into a text file,
+// so build.rs sets this env var
+pub const LIB_PATH: &str = match option_env!("win_lib_path") {
+    Some(s) => s,
+    None => concat!(env!("CARGO_MANIFEST_DIR"), "/../Lib"),
+};
+
+#[cfg(feature = "freeze-stdlib")]
+pub fn frozen_stdlib() -> impl Iterator<Item = (String, rustpython_compiler_core::FrozenModule)> {
+    rustpython_derive::py_freeze!(dir = "../Lib", crate_name = "rustpython_compiler_core")
+}

src/lib.rs

@@ -248,6 +248,35 @@ fn add_stdlib(vm: &mut VirtualMachine) {
     let _ = vm;
     #[cfg(feature = "stdlib")]
     vm.add_native_modules(rustpython_stdlib::get_module_inits());
+
+    // if we're on freeze-stdlib, the core stdlib modules will be included anyway
+    #[cfg(feature = "freeze-stdlib")]
+    vm.add_frozen(rustpython_pylib::frozen_stdlib());
+
+    #[cfg(not(feature = "freeze-stdlib"))]
+    {
+        use rustpython_vm::common::rc::PyRc;
+        let state = PyRc::get_mut(&mut vm.state).unwrap();
+
+        #[allow(clippy::needless_collect)] // false positive
+        let path_list: Vec<_> = state.settings.path_list.drain(..).collect();
+
+        // BUILDTIME_RUSTPYTHONPATH should be set when distributing
+        if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") {
+            state
+                .settings
+                .path_list
+                .extend(split_paths(paths).map(|path| path.into_os_string().into_string().unwrap()))
+        } else {
+            #[cfg(feature = "rustpython-pylib")]
+            state
+                .settings
+                .path_list
+                .push(rustpython_pylib::LIB_PATH.to_owned())
+        }
+
+        state.settings.path_list.extend(path_list.into_iter());
+    }
 }
 
 /// Create settings by examining command line arguments and environment
@@ -264,21 +293,6 @@ fn create_settings(matches: &ArgMatches) -> Settings {
 
     let ignore_environment = settings.ignore_environment || settings.isolated;
 
-    // when rustpython-vm/pylib is enabled, Settings::default().path_list has pylib::LIB_PATH
-    let maybe_pylib = settings.path_list.pop();
-
-    // add the current directory to sys.path
-    settings.path_list.push("".to_owned());
-
-    // BUILDTIME_RUSTPYTHONPATH should be set when distributing
-    if let Some(paths) = option_env!("BUILDTIME_RUSTPYTHONPATH") {
-        settings
-            .path_list
-            .extend(split_paths(paths).map(|path| path.into_os_string().into_string().unwrap()))
-    } else {
-        settings.path_list.extend(maybe_pylib);
-    }
-
     if !ignore_environment {
         settings.path_list.extend(get_paths("RUSTPYTHONPATH"));
         settings.path_list.extend(get_paths("PYTHONPATH"));

vm/Cargo.toml

@@ -14,7 +14,7 @@ importlib = []
 encodings = ["importlib"]
 vm-tracing-logging = []
 flame-it = ["flame", "flamer"]
-freeze-stdlib = ["rustpython-pylib/freeze-stdlib"]
+freeze-stdlib = []
 jit = ["rustpython-jit"]
 threading = ["rustpython-common/threading"]
 compiler = ["parser", "codegen", "rustpython-compiler"]
@@ -31,7 +31,6 @@ rustpython-compiler-core = { path = "../compiler/core", version = "0.1.2" }
 rustpython-common = { path = "../common" }
 rustpython-derive = { path = "../derive", version = "0.1.2" }
 rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" }
-rustpython-pylib = { path = "pylib-crate", version = "0.1.0" }
 
 num-complex = { version = "0.4.0", features = ["serde"] }
 num-bigint = { version = "0.4.3", features = ["serde"] }

vm/pylib-crate/src/lib.rs

@@ -1,32 +0,0 @@
-//! This crate includes the compiled python bytecode of the RustPython standard library. The most
-//! common way to use this crate is to just add the `"freeze-stdlib"` feature to `rustpython-vm`,
-//! in order to automatically include the python part of the standard library into the binary.
-
-// windows needs to read the symlink out of `Lib` as git turns it into a text file,
-// so build.rs sets this env var
-pub const LIB_PATH: &str = match option_env!("win_lib_path") {
-    Some(s) => s,
-    None => concat!(env!("CARGO_MANIFEST_DIR"), "/../../Lib"),
-};
-
-use rustpython_compiler_core::FrozenModule;
-
-pub fn frozen_builtins() -> impl Iterator<Item = (String, FrozenModule)> {
-    rustpython_derive::py_freeze!(
-        dir = "../Lib/python_builtins",
-        crate_name = "rustpython_compiler_core"
-    )
-}
-
-#[cfg(not(feature = "freeze-stdlib"))]
-pub fn frozen_core() -> impl Iterator<Item = (String, FrozenModule)> {
-    rustpython_derive::py_freeze!(
-        dir = "../Lib/core_modules",
-        crate_name = "rustpython_compiler_core"
-    )
-}
-
-#[cfg(feature = "freeze-stdlib")]
-pub fn frozen_stdlib() -> impl Iterator<Item = (String, FrozenModule)> {
-    rustpython_derive::py_freeze!(dir = "../../Lib", crate_name = "rustpython_compiler_core")
-}

vm/src/frozen.rs

@@ -1,6 +1,6 @@
 use crate::bytecode::FrozenModule;
 
-pub fn get_module_inits() -> impl Iterator<Item = (String, FrozenModule)> {
+pub fn core_frozen_inits() -> impl Iterator<Item = (String, FrozenModule)> {
     let iter = std::iter::empty();
     macro_rules! ext_modules {
         ($iter:ident, ($modules:expr)) => {
@@ -20,16 +20,24 @@ pub fn get_module_inits() -> impl Iterator<Item = (String, FrozenModule)> {
     // Python modules that the vm calls into, but are not actually part of the stdlib. They could
     // in theory be implemented in Rust, but are easiest to do in Python for one reason or another.
     // Includes _importlib_bootstrap and _importlib_bootstrap_external
-    ext_modules!(iter, (rustpython_pylib::frozen_builtins()));
+    ext_modules!(
+        iter,
+        (rustpython_derive::py_freeze!(
+            dir = "./Lib/python_builtins",
+            crate_name = "rustpython_compiler_core"
+        ))
+    );
 
-    #[cfg(not(feature = "freeze-stdlib"))]
     // core stdlib Python modules that the vm calls into, but are still used in Python
     // application code, e.g. copyreg
-    ext_modules!(iter, (rustpython_pylib::frozen_core()));
-
-    // if we're on freeze-stdlib, the core stdlib modules will be included anyway
-    #[cfg(feature = "freeze-stdlib")]
-    ext_modules!(iter, (rustpython_pylib::frozen_stdlib()));
+    #[cfg(not(feature = "freeze-stdlib"))]
+    ext_modules!(
+        iter,
+        (rustpython_derive::py_freeze!(
+            dir = "./Lib/core_modules",
+            crate_name = "rustpython_compiler_core"
+        ))
+    );
 
     iter
 }

vm/src/vm/mod.rs

@@ -176,7 +176,7 @@ impl VirtualMachine {
             panic!("Interpreters in same process must share the hash seed");
         }
 
-        let frozen = frozen::get_module_inits().collect();
+        let frozen = frozen::core_frozen_inits().collect();
         PyRc::get_mut(&mut vm.state).unwrap().frozen = frozen;
 
         vm.builtins
@@ -227,6 +227,9 @@ impl VirtualMachine {
             panic!("Double Initialize Error");
         }
 
+        // add the current directory to sys.path
+        self.state_mut().settings.path_list.insert(0, "".to_owned());
+
         stdlib::builtins::make_module(self, self.builtins.clone().into());
         stdlib::sys::init_module(self, self.sys_module.as_ref(), self.builtins.as_ref());
 

vm/src/vm/setting.rs

@@ -89,7 +89,7 @@ impl Default for Settings {
             dev_mode: false,
             warn_default_encoding: false,
             warnopts: vec![],
-            path_list: vec![rustpython_pylib::LIB_PATH.to_owned()],
+            path_list: vec![],
             argv: vec![],
             hash_seed: None,
             stdio_unbuffered: false,

wasm/lib/Cargo.toml

@@ -11,13 +11,15 @@ edition = "2021"
 crate-type = ["cdylib", "rlib"]
 
 [features]
-default = ["freeze-stdlib", "rustpython-stdlib"]
-freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
+default = ["stdlib"]
+stdlib = ["freeze-stdlib", "rustpython-pylib", "rustpython-stdlib"]
+freeze-stdlib = ["rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"]
 no-start-func = []
 
 [dependencies]
 rustpython-common = { path = "../../common" }
 rustpython-parser = { path = "../../compiler/parser" }
+rustpython-pylib = { path = "../../pylib", default-features = false, optional = true }
 rustpython-stdlib = { path = "../../stdlib", default-features = false, optional = true }
 # make sure no threading! otherwise wasm build will fail
 rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings"] }

wasm/lib/src/vm_class.rs

@@ -42,6 +42,12 @@ impl StoredVirtualMachine {
         let mut settings = Settings::default();
         settings.allow_external_library = false;
         let interp = Interpreter::with_init(settings, |vm| {
+            #[cfg(feature = "stdlib")]
+            vm.add_native_modules(rustpython_stdlib::get_module_inits());
+
+            #[cfg(feature = "freeze-stdlib")]
+            vm.add_frozen(rustpython_pylib::frozen_stdlib());
+
             vm.wasm_id = Some(id);
 
             js_module::setup_js_module(vm);