diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 0b634ea5d..6135cb75d 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -136,11 +136,6 @@ class PyPicklerTests(AbstractPickleTests, unittest.TestCase): def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_buffers_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes @@ -166,42 +161,6 @@ class PyPicklerTests(AbstractPickleTests, unittest.TestCase): def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_oob_buffers_writable_to_readonly() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_buffers_error() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_bytearray_memoization() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_bytes_memoization() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_in_band_buffers() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_oob_buffers() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_oob_buffers_writable_to_readonly() - - class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, BigmemPickleTests, unittest.TestCase): @@ -250,11 +209,6 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_buffers_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index f4ce1c75b..6ec8b348f 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -67,11 +67,6 @@ class OptimizedPickleTests(AbstractPickleTests, unittest.TestCase): def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes return super().test_buffers_error() - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes - return super().test_builtin_functions() - # TODO: RUSTPYTHON @unittest.expectedFailure def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index 98993b415..c97ed31c4 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -114,9 +114,7 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup, bench: &MicroBenc settings.user_site_directory = false; Interpreter::with_init(settings, |vm| { - for (name, init) in rustpython_stdlib::get_module_inits() { - vm.add_native_module(name, init); - } + vm.add_native_module_defs(rustpython_stdlib::get_module_defs()); }) .enter(|vm| { let setup_code = vm diff --git a/crates/derive-impl/src/pymodule.rs b/crates/derive-impl/src/pymodule.rs index ebcea89ac..03a90bc70 100644 --- a/crates/derive-impl/src/pymodule.rs +++ b/crates/derive-impl/src/pymodule.rs @@ -58,7 +58,7 @@ struct ModuleContext { name: String, function_items: FunctionNursery, attribute_items: ItemNursery, - has_extend_module: bool, // TODO: check if `fn extend_module` exists + has_module_exec: bool, errors: Vec, } @@ -82,6 +82,12 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result Result Result) -> ::rustpython_vm::PyResult<()> { - __extend_module(vm, module); + pub(crate) fn module_exec(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> { + __module_exec(vm, module); Ok(()) } }); @@ -192,7 +198,7 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result, ) { diff --git a/crates/derive/src/lib.rs b/crates/derive/src/lib.rs index 5a3ff84c6..2457d456b 100644 --- a/crates/derive/src/lib.rs +++ b/crates/derive/src/lib.rs @@ -143,8 +143,8 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream { } /// This attribute must be applied to an inline module. -/// It defines a Python module in the form a `make_module` function in the module; -/// this has to be used in a `get_module_inits` to properly register the module. +/// It defines a Python module in the form of a `module_def` function in the module; +/// this has to be used in a `get_module_defs` to properly register the module. /// Additionally, this macro defines 'MODULE_NAME' and 'DOC' in the module. /// # Arguments /// - `name`: the name of the python module, diff --git a/crates/stdlib/src/array.rs b/crates/stdlib/src/array.rs index 463027d9b..3f33f23d7 100644 --- a/crates/stdlib/src/array.rs +++ b/crates/stdlib/src/array.rs @@ -1,33 +1,6 @@ // spell-checker:ignore typecode tofile tolist fromfile -use rustpython_vm::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let module = array::make_module(vm); - - let array = module - .get_attr("array", vm) - .expect("Expect array has array type."); - - let collections_abc = vm - .import("collections.abc", 0) - .expect("Expect collections exist."); - let abc = collections_abc - .get_attr("abc", vm) - .expect("Expect collections has abc submodule."); - let mutable_sequence = abc - .get_attr("MutableSequence", vm) - .expect("Expect collections.abc has MutableSequence type."); - - let register = &mutable_sequence - .get_attr("register", vm) - .expect("Expect collections.abc.MutableSequence has register method."); - register - .call((array,), vm) - .expect("Expect collections.abc.MutableSequence.register(array.array) not fail."); - - module -} +pub(crate) use array::module_def; #[pymodule(name = "array")] mod array { @@ -1658,4 +1631,25 @@ mod array { }; PyArray::from(array).into_ref_with_type(vm, cls) } + + // Register array.array as collections.abc.MutableSequence + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + __module_exec(vm, module); + + let array_type = module + .get_attr("array", vm) + .expect("array module has array type"); + + // vm.import returns the top-level module, so we need to get abc submodule + let collections_abc = vm.import("collections.abc", 0)?; + let abc = collections_abc.get_attr("abc", vm)?; + let mutable_sequence = abc.get_attr("MutableSequence", vm)?; + let register = mutable_sequence.get_attr("register", vm)?; + register.call((array_type,), vm)?; + + Ok(()) + } } diff --git a/crates/stdlib/src/contextvars.rs b/crates/stdlib/src/contextvars.rs index 5608b39d1..57864bf45 100644 --- a/crates/stdlib/src/contextvars.rs +++ b/crates/stdlib/src/contextvars.rs @@ -1,20 +1,9 @@ -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, class::StaticType}; +pub(crate) use _contextvars::module_def; + +use crate::vm::PyRef; use _contextvars::PyContext; use core::cell::RefCell; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let module = _contextvars::make_module(vm); - let token_type = module.get_attr("Token", vm).unwrap(); - token_type - .set_attr( - "MISSING", - _contextvars::ContextTokenMissing::static_type().to_owned(), - vm, - ) - .unwrap(); - module -} - thread_local! { // TODO: Vec doesn't seem to match copy behavior static CONTEXTS: RefCell>> = RefCell::default(); @@ -643,4 +632,17 @@ mod _contextvars { fn copy_context(vm: &VirtualMachine) -> PyContext { PyContext::current(vm).copy() } + + // Set Token.MISSING attribute + pub(crate) fn module_exec( + vm: &VirtualMachine, + module: &Py, + ) -> PyResult<()> { + __module_exec(vm, module); + + let token_type = module.get_attr("Token", vm)?; + token_type.set_attr("MISSING", ContextTokenMissing::static_type().to_owned(), vm)?; + + Ok(()) + } } diff --git a/crates/stdlib/src/gc.rs b/crates/stdlib/src/gc.rs index 36a77db85..9359119a6 100644 --- a/crates/stdlib/src/gc.rs +++ b/crates/stdlib/src/gc.rs @@ -2,7 +2,7 @@ pub(crate) use gc::module_def; #[pymodule] mod gc { - use crate::vm::{PyObjectRef, PyResult, VirtualMachine, function::FuncArgs}; + use crate::vm::{PyResult, VirtualMachine, function::FuncArgs}; #[pyfunction] fn collect(_args: FuncArgs, _vm: &VirtualMachine) -> i32 { @@ -40,15 +40,13 @@ mod gc { } #[pyfunction] - fn get_referents(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { - // RustPython does not track object references. - vm.ctx.new_tuple(vec![]).into() + fn get_referents(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_not_implemented_error("")) } #[pyfunction] - fn get_referrers(_args: FuncArgs, vm: &VirtualMachine) -> PyObjectRef { - // RustPython does not track object references. - vm.ctx.new_list(vec![]).into() + fn get_referrers(_args: FuncArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_not_implemented_error("")) } #[pyfunction] diff --git a/crates/stdlib/src/lib.rs b/crates/stdlib/src/lib.rs index 043176e68..82d037417 100644 --- a/crates/stdlib/src/lib.rs +++ b/crates/stdlib/src/lib.rs @@ -105,10 +105,7 @@ mod tkinter; use rustpython_common as common; use rustpython_vm as vm; -use crate::vm::{ - builtins, - stdlib::{StdlibDefFunc, StdlibInitFunc}, -}; +use crate::vm::{builtins, stdlib::StdlibDefFunc}; use alloc::borrow::Cow; /// Returns module definitions for multi-phase init modules. @@ -133,12 +130,13 @@ pub fn get_module_defs() -> impl Iterator, StdlibDefFu modules! { #[cfg(all())] { - "_asyncio" => _asyncio::module_def, + "array" => array::module_def, "binascii" => binascii::module_def, "_bisect" => bisect::module_def, "_blake2" => blake2::module_def, "_bz2" => bz2::module_def, "cmath" => cmath::module_def, + "_contextvars" => contextvars::module_def, "_csv" => csv::module_def, "faulthandler" => faulthandler::module_def, "gc" => gc::module_def, @@ -149,28 +147,45 @@ pub fn get_module_defs() -> impl Iterator, StdlibDefFu "_opcode" => opcode::module_def, "_random" => random::module_def, "_sha1" => sha1::module_def, + "_sha256" => sha256::module_def, "_sha3" => sha3::module_def, + "_sha512" => sha512::module_def, "_statistics" => statistics::module_def, "_struct" => pystruct::module_def, "_suggestions" => suggestions::module_def, "zlib" => zlib::module_def, + "pyexpat" => pyexpat::module_def, + "unicodedata" => unicodedata::module_def, } #[cfg(any(unix, target_os = "wasi"))] { "fcntl" => fcntl::module_def, } + #[cfg(any(unix, windows, target_os = "wasi"))] + { + "select" => select::module_def, + } #[cfg(not(target_arch = "wasm32"))] { "_multiprocessing" => multiprocessing::module_def, + "_socket" => socket::module_def, } #[cfg(not(any(target_os = "android", target_arch = "wasm32")))] { "_lzma" => lzma::module_def, } + #[cfg(all(feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32"))))] + { + "_sqlite3" => sqlite::module_def, + } #[cfg(all(not(target_arch = "wasm32"), feature = "ssl-rustls"))] { "_ssl" => ssl::module_def, } + #[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))] + { + "_ssl" => openssl::module_def, + } #[cfg(windows)] { "_overlapped" => overlapped::module_def, @@ -218,50 +233,3 @@ pub fn get_module_defs() -> impl Iterator, StdlibDefFu } } } - -/// Returns module initializers for single-phase init modules. -/// These are modules that don't use #[pymodule] macro or need custom initialization. -pub fn get_module_inits() -> impl Iterator, StdlibInitFunc)> { - macro_rules! modules { - { - $( - #[cfg($cfg:meta)] - { $( $key:expr => $val:expr),* $(,)? } - )* - } => {{ - [ - $( - $(#[cfg($cfg)] (Cow::<'static, str>::from($key), Box::new($val) as StdlibInitFunc),)* - )* - ] - .into_iter() - }}; - } - modules! { - #[cfg(all())] - { - "array" => array::make_module, - "_contextvars" => contextvars::make_module, - "pyexpat" => pyexpat::make_module, - "_sha256" => sha256::make_module, - "_sha512" => sha512::make_module, - "unicodedata" => unicodedata::make_module, - } - #[cfg(any(unix, windows, target_os = "wasi"))] - { - "select" => select::make_module, - } - #[cfg(not(target_arch = "wasm32"))] - { - "_socket" => socket::make_module, - } - #[cfg(all(feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32"))))] - { - "_sqlite3" => sqlite::make_module, - } - #[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))] - { - "_ssl" => openssl::make_module, - } - } -} diff --git a/crates/stdlib/src/math.rs b/crates/stdlib/src/math.rs index e72fc7990..238c7fd49 100644 --- a/crates/stdlib/src/math.rs +++ b/crates/stdlib/src/math.rs @@ -1,4 +1,4 @@ -pub(crate) use math::__module_def; +pub(crate) use math::module_def; use crate::vm::{VirtualMachine, builtins::PyBaseExceptionRef}; diff --git a/crates/stdlib/src/opcode.rs b/crates/stdlib/src/opcode.rs index 4f59a3358..f3ae1f207 100644 --- a/crates/stdlib/src/opcode.rs +++ b/crates/stdlib/src/opcode.rs @@ -1,4 +1,4 @@ -pub(crate) use opcode::__module_def; +pub(crate) use opcode::module_def; #[pymodule] mod opcode { diff --git a/crates/stdlib/src/openssl.rs b/crates/stdlib/src/openssl.rs index df153459f..e36189957 100644 --- a/crates/stdlib/src/openssl.rs +++ b/crates/stdlib/src/openssl.rs @@ -23,18 +23,11 @@ cfg_if::cfg_if! { } } -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; +pub(crate) use _ssl::module_def; + use openssl_probe::ProbeResult; use std::sync::LazyLock; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - // if openssl is vendored, it doesn't know the locations - // of system certificates - cache the probe result now. - #[cfg(openssl_vendored)] - LazyLock::force(&PROBE); - _ssl::make_module(vm) -} - // define our own copy of ProbeResult so we can handle the vendor case // easily, without having to have a bunch of cfgs cfg_if::cfg_if! { @@ -52,6 +45,8 @@ cfg_if::cfg_if! { #[allow(non_upper_case_globals)] #[pymodule(with(cert::ssl_cert, ssl_error::ssl_error, ossl101, ossl111, windows))] mod _ssl { + #[cfg(openssl_vendored)] + use super::PROBE; use super::{bio, probe}; // Import error types and helpers used in this module (others are exposed via pymodule(with(...))) @@ -67,8 +62,8 @@ mod _ssl { vm::{ AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::{ - PyBaseException, PyBaseExceptionRef, PyBytesRef, PyListRef, PyStrRef, PyType, - PyWeak, + PyBaseException, PyBaseExceptionRef, PyBytesRef, PyListRef, PyModule, PyStrRef, + PyType, PyWeak, }, class_or_notimplemented, convert::ToPyException, @@ -104,6 +99,16 @@ mod _ssl { // Import certificate types from parent module use super::cert::{self, cert_to_certificate, cert_to_py}; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + // if openssl is vendored, it doesn't know the locations + // of system certificates - cache the probe result now. + #[cfg(openssl_vendored)] + std::sync::LazyLock::force(&PROBE); + + __module_exec(vm, module); + Ok(()) + } + // Re-export PySSLCertificate to make it available in the _ssl module // It will be automatically exposed to Python via #[pyclass] #[allow(unused_imports)] diff --git a/crates/stdlib/src/pyexpat.rs b/crates/stdlib/src/pyexpat.rs index 89cf69077..f7c9cacad 100644 --- a/crates/stdlib/src/pyexpat.rs +++ b/crates/stdlib/src/pyexpat.rs @@ -2,18 +2,7 @@ // spell-checker: ignore libexpat -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, extend_module}; - -pub fn make_module(vm: &VirtualMachine) -> PyRef { - let module = _pyexpat::make_module(vm); - - extend_module!(vm, &module, { - "errors" => _errors::make_module(vm), - "model" => _model::make_module(vm), - }); - - module -} +pub(crate) use _pyexpat::module_def; macro_rules! create_property { ($ctx: expr, $attributes: expr, $name: expr, $class: expr, $element: ident) => { @@ -52,7 +41,7 @@ macro_rules! create_bool_property { mod _pyexpat { use crate::vm::{ Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine, - builtins::{PyBytesRef, PyStr, PyStrRef, PyType}, + builtins::{PyBytesRef, PyModule, PyStr, PyStrRef, PyType}, function::ArgBytesLike, function::{Either, IntoFuncArgs, OptionalArg}, }; @@ -60,6 +49,18 @@ mod _pyexpat { use std::io::Cursor; use xml::reader::XmlEvent; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + + // Add submodules + let model = PyModule::from_def(super::_model::module_def(&vm.ctx)).into_ref(&vm.ctx); + let errors = PyModule::from_def(super::_errors::module_def(&vm.ctx)).into_ref(&vm.ctx); + module.set_attr("model", model, vm)?; + module.set_attr("errors", errors, vm)?; + + Ok(()) + } + type MutableObject = PyRwLock; #[pyattr(name = "version_info")] diff --git a/crates/stdlib/src/select.rs b/crates/stdlib/src/select.rs index 3c2f5e63c..09a85cf1c 100644 --- a/crates/stdlib/src/select.rs +++ b/crates/stdlib/src/select.rs @@ -1,25 +1,13 @@ // spell-checker:disable +pub(crate) use decl::module_def; + use crate::vm::{ - PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef, - builtins::PyModule, + PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef, }; use core::mem; use std::io; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - #[cfg(windows)] - crate::vm::windows::init_winsock(); - - #[cfg(unix)] - { - use crate::vm::class::PyClassImpl; - decl::poll::PyPoll::make_class(&vm.ctx); - } - - decl::make_module(vm) -} - #[cfg(unix)] mod platform { pub use libc::{FD_ISSET, FD_SET, FD_SETSIZE, FD_ZERO, fd_set, select, timeval}; @@ -221,13 +209,27 @@ fn sec_to_timeval(sec: f64) -> timeval { mod decl { use super::*; use crate::vm::{ - PyObjectRef, PyResult, VirtualMachine, - builtins::PyTypeRef, + Py, PyObjectRef, PyResult, VirtualMachine, + builtins::{PyModule, PyTypeRef}, convert::ToPyException, function::{Either, OptionalOption}, stdlib::time, }; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + #[cfg(windows)] + crate::vm::windows::init_winsock(); + + #[cfg(unix)] + { + use crate::vm::class::PyClassImpl; + poll::PyPoll::make_class(&vm.ctx); + } + + __module_exec(vm, module); + Ok(()) + } + #[pyattr] fn error(vm: &VirtualMachine) -> PyTypeRef { vm.ctx.exceptions.os_error.to_owned() @@ -545,7 +547,7 @@ mod decl { pub(super) mod epoll { use super::*; use crate::vm::{ - Py, PyPayload, + Py, PyPayload, PyRef, builtins::PyType, common::lock::{PyRwLock, PyRwLockReadGuard}, convert::{IntoPyException, ToPyObject}, diff --git a/crates/stdlib/src/sha256.rs b/crates/stdlib/src/sha256.rs index 5d031968a..bf4a39373 100644 --- a/crates/stdlib/src/sha256.rs +++ b/crates/stdlib/src/sha256.rs @@ -1,14 +1,7 @@ -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let _ = vm.import("_hashlib", 0); - _sha256::make_module(vm) -} - #[pymodule] mod _sha256 { use crate::hashlib::_hashlib::{HashArgs, local_sha224, local_sha256}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult { @@ -19,4 +12,12 @@ mod _sha256 { fn sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_sha256(args).into_pyobject(vm)) } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } + +pub(crate) use _sha256::module_def; diff --git a/crates/stdlib/src/sha512.rs b/crates/stdlib/src/sha512.rs index baf63fdac..303c0067d 100644 --- a/crates/stdlib/src/sha512.rs +++ b/crates/stdlib/src/sha512.rs @@ -1,14 +1,7 @@ -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; - -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - let _ = vm.import("_hashlib", 0); - _sha512::make_module(vm) -} - #[pymodule] mod _sha512 { use crate::hashlib::_hashlib::{HashArgs, local_sha384, local_sha512}; - use crate::vm::{PyPayload, PyResult, VirtualMachine}; + use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; #[pyfunction] fn sha384(args: HashArgs, vm: &VirtualMachine) -> PyResult { @@ -19,4 +12,12 @@ mod _sha512 { fn sha512(args: HashArgs, vm: &VirtualMachine) -> PyResult { Ok(local_sha512(args).into_pyobject(vm)) } + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + let _ = vm.import("_hashlib", 0); + __module_exec(vm, module); + Ok(()) + } } + +pub(crate) use _sha512::module_def; diff --git a/crates/stdlib/src/socket.rs b/crates/stdlib/src/socket.rs index ea5f39cb4..b0ebd681e 100644 --- a/crates/stdlib/src/socket.rs +++ b/crates/stdlib/src/socket.rs @@ -1,27 +1,32 @@ // spell-checker:disable -use crate::vm::{PyRef, VirtualMachine, builtins::PyModule}; +pub(crate) use _socket::module_def; + #[cfg(feature = "ssl")] pub(super) use _socket::{PySocket, SelectKind, sock_select, timeout_error_msg}; -pub fn make_module(vm: &VirtualMachine) -> PyRef { - #[cfg(windows)] - crate::vm::windows::init_winsock(); - _socket::make_module(vm) -} - #[pymodule] mod _socket { use crate::common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard}; use crate::vm::{ AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, - builtins::{PyBaseExceptionRef, PyListRef, PyOSError, PyStrRef, PyTupleRef, PyTypeRef}, + builtins::{ + PyBaseExceptionRef, PyListRef, PyModule, PyOSError, PyStrRef, PyTupleRef, PyTypeRef, + }, common::os::ErrorExt, convert::{IntoPyException, ToPyObject, TryFromBorrowedObject, TryFromObject}, function::{ArgBytesLike, ArgMemoryBuffer, Either, FsPath, OptionalArg, OptionalOption}, types::{Constructor, DefaultConstructor, Initializer, Representable}, utils::ToCString, }; + + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + #[cfg(windows)] + crate::vm::windows::init_winsock(); + + __module_exec(vm, module); + Ok(()) + } use core::{ mem::MaybeUninit, net::{Ipv4Addr, Ipv6Addr, SocketAddr}, diff --git a/crates/stdlib/src/sqlite.rs b/crates/stdlib/src/sqlite.rs index 9b7b810b2..506c9da58 100644 --- a/crates/stdlib/src/sqlite.rs +++ b/crates/stdlib/src/sqlite.rs @@ -8,15 +8,7 @@ // spell-checker:ignore cantlock commithook foreignkey notnull primarykey gettemppath autoindex convpath // spell-checker:ignore dbmoved vnode nbytes -use rustpython_vm::{AsObject, PyRef, VirtualMachine, builtins::PyModule}; - -// pub(crate) use _sqlite::make_module; -pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { - // TODO: sqlite version check - let module = _sqlite::make_module(vm); - _sqlite::setup_module(module.as_object(), vm); - module -} +pub(crate) use _sqlite::module_def; #[pymodule] mod _sqlite { @@ -58,8 +50,8 @@ mod _sqlite { TryFromBorrowedObject, VirtualMachine, atomic_func, builtins::{ PyBaseException, PyBaseExceptionRef, PyByteArray, PyBytes, PyDict, PyDictRef, PyFloat, - PyInt, PyIntRef, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef, - PyUtf8Str, PyUtf8StrRef, + PyInt, PyIntRef, PyModule, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, + PyTypeRef, PyUtf8Str, PyUtf8StrRef, }, convert::IntoObject, function::{ @@ -852,26 +844,26 @@ mod _sqlite { .expect("enable traceback not initialize") } - pub(super) fn setup_module(module: &PyObject, vm: &VirtualMachine) { + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + for (name, code) in ERROR_CODES { let name = vm.ctx.intern_str(*name); let code = vm.new_pyobj(*code); - module.set_attr(name, code, vm).unwrap(); + module.set_attr(name, code, vm)?; } - setup_module_exceptions(module, vm); + setup_module_exceptions(module.as_object(), vm); let _ = CONVERTERS.set(vm.ctx.new_dict()); let _ = ADAPTERS.set(vm.ctx.new_dict()); let _ = USER_FUNCTION_EXCEPTION.set(PyAtomicRef::from(None)); let _ = ENABLE_TRACEBACK.set(Radium::new(false)); - module - .set_attr("converters", converters().to_owned(), vm) - .unwrap(); - module - .set_attr("adapters", adapters().to_owned(), vm) - .unwrap(); + module.set_attr("converters", converters().to_owned(), vm)?; + module.set_attr("adapters", adapters().to_owned(), vm)?; + + Ok(()) } #[pyattr] diff --git a/crates/stdlib/src/unicodedata.rs b/crates/stdlib/src/unicodedata.rs index 68d9a17e5..a859e39df 100644 --- a/crates/stdlib/src/unicodedata.rs +++ b/crates/stdlib/src/unicodedata.rs @@ -4,35 +4,12 @@ // spell-checker:ignore nfkc unistr unidata +pub(crate) use unicodedata::module_def; + use crate::vm::{ - PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyModule, - builtins::PyStr, convert::TryFromBorrowedObject, + PyObject, PyResult, VirtualMachine, builtins::PyStr, convert::TryFromBorrowedObject, }; -pub fn make_module(vm: &VirtualMachine) -> PyRef { - let module = unicodedata::make_module(vm); - - let ucd: PyObjectRef = unicodedata::Ucd::new(unic_ucd_age::UNICODE_VERSION) - .into_ref(&vm.ctx) - .into(); - - for attr in [ - "category", - "lookup", - "name", - "bidirectional", - "east_asian_width", - "normalize", - "mirrored", - ] { - crate::vm::extend_module!(vm, &module, { - attr => ucd.get_attr(attr, vm).unwrap(), - }); - } - - module -} - enum NormalizeForm { Nfc, Nfkc, @@ -60,7 +37,8 @@ impl<'a> TryFromBorrowedObject<'a> for NormalizeForm { #[pymodule] mod unicodedata { use crate::vm::{ - PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyStrRef, + Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + builtins::{PyModule, PyStrRef}, function::OptionalArg, }; use itertools::Itertools; @@ -73,6 +51,27 @@ mod unicodedata { use unic_ucd_category::GeneralCategory; use unicode_bidi_mirroring::is_mirroring; + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + + // Add UCD methods as module-level functions + let ucd: PyObjectRef = Ucd::new(UNICODE_VERSION).into_ref(&vm.ctx).into(); + + for attr in [ + "category", + "lookup", + "name", + "bidirectional", + "east_asian_width", + "normalize", + "mirrored", + ] { + module.set_attr(attr, ucd.get_attr(attr, vm)?, vm)?; + } + + Ok(()) + } + #[pyattr] #[pyclass(name = "UCD")] #[derive(Debug, PyPayload)] diff --git a/crates/vm/src/import.rs b/crates/vm/src/import.rs index a2ea2c7d0..0b51c63fd 100644 --- a/crates/vm/src/import.rs +++ b/crates/vm/src/import.rs @@ -104,13 +104,8 @@ pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult { // Phase 1: Create module from definition let module = PyModule::from_def(def).into_ref(&vm.ctx); - // Initialize module dict - let dict = vm.ctx.new_dict(); - dict.set_item("__name__", vm.ctx.new_str(def.name.as_str()).into(), vm)?; - if let Some(doc) = def.doc { - dict.set_item("__doc__", vm.ctx.new_str(doc.as_str()).into(), vm)?; - } - module.set_attr("__dict__", dict, vm)?; + // Initialize module dict using proper method + PyModule::__init_dict_from_def(vm, &module); // Add to sys.modules BEFORE exec (critical for circular import handling) sys_modules.set_item(module_name, module.clone().into(), vm)?; diff --git a/crates/vm/src/stdlib/builtins.rs b/crates/vm/src/stdlib/builtins.rs index 7be55261e..dbe9431f6 100644 --- a/crates/vm/src/stdlib/builtins.rs +++ b/crates/vm/src/stdlib/builtins.rs @@ -1100,7 +1100,7 @@ pub fn init_module(vm: &VirtualMachine, module: &Py) { crate::protocol::VecBuffer::make_class(&vm.ctx); - builtins::extend_module(vm, module).unwrap(); + builtins::module_exec(vm, module).unwrap(); let debug_mode: bool = vm.state.config.settings.optimize == 0; // Create dynamic ExceptionGroup with multiple inheritance (BaseExceptionGroup + Exception) diff --git a/crates/vm/src/stdlib/imp.rs b/crates/vm/src/stdlib/imp.rs index 06ecbaeb5..ba60904aa 100644 --- a/crates/vm/src/stdlib/imp.rs +++ b/crates/vm/src/stdlib/imp.rs @@ -135,13 +135,8 @@ mod _imp { // Phase 1: Create module from definition let module = PyModule::from_def(def).into_ref(&vm.ctx); - // Initialize module dict - let dict = vm.ctx.new_dict(); - dict.set_item("__name__", vm.ctx.new_str(def.name.as_str()).into(), vm)?; - if let Some(doc) = def.doc { - dict.set_item("__doc__", vm.ctx.new_str(doc.as_str()).into(), vm)?; - } - module.set_attr("__dict__", dict, vm)?; + // Initialize module dict using proper method + PyModule::__init_dict_from_def(vm, &module); // Add to sys.modules BEFORE exec (critical for circular import handling) sys_modules.set_item(&*name, module.clone().into(), vm)?; diff --git a/crates/vm/src/stdlib/io.rs b/crates/vm/src/stdlib/io.rs index 9cb0b7d7d..0578f1947 100644 --- a/crates/vm/src/stdlib/io.rs +++ b/crates/vm/src/stdlib/io.rs @@ -95,7 +95,7 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = _io::make_module(vm); #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] - fileio::extend_module(vm, &module).unwrap(); + fileio::module_exec(vm, &module).unwrap(); let unsupported_operation = _io::unsupported_operation().to_owned(); extend_module!(vm, &module, { diff --git a/crates/vm/src/stdlib/nt.rs b/crates/vm/src/stdlib/nt.rs index a32959808..1ed7544a4 100644 --- a/crates/vm/src/stdlib/nt.rs +++ b/crates/vm/src/stdlib/nt.rs @@ -6,7 +6,7 @@ pub use module::raw_set_handle_inheritable; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = module::make_module(vm); - super::os::extend_module(vm, &module); + super::os::module_exec(vm, &module); module } diff --git a/crates/vm/src/stdlib/os.rs b/crates/vm/src/stdlib/os.rs index aa72fd9fd..4645e4a28 100644 --- a/crates/vm/src/stdlib/os.rs +++ b/crates/vm/src/stdlib/os.rs @@ -1896,7 +1896,7 @@ impl SupportFunc { } } -pub fn extend_module(vm: &VirtualMachine, module: &Py) { +pub fn module_exec(vm: &VirtualMachine, module: &Py) { let support_funcs = _os::support_funcs(); let supports_fd = PySet::default().into_ref(&vm.ctx); let supports_dir_fd = PySet::default().into_ref(&vm.ctx); diff --git a/crates/vm/src/stdlib/posix.rs b/crates/vm/src/stdlib/posix.rs index 7ac5129d8..a47936beb 100644 --- a/crates/vm/src/stdlib/posix.rs +++ b/crates/vm/src/stdlib/posix.rs @@ -16,7 +16,7 @@ pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> nix::Result<()> pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = module::make_module(vm); - super::os::extend_module(vm, &module); + super::os::module_exec(vm, &module); module } diff --git a/crates/vm/src/stdlib/posix_compat.rs b/crates/vm/src/stdlib/posix_compat.rs index 0fabd6a1f..09333a87e 100644 --- a/crates/vm/src/stdlib/posix_compat.rs +++ b/crates/vm/src/stdlib/posix_compat.rs @@ -5,7 +5,7 @@ use crate::{PyRef, VirtualMachine, builtins::PyModule}; pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef { let module = module::make_module(vm); - super::os::extend_module(vm, &module); + super::os::module_exec(vm, &module); module } diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 86eb15810..5409601c5 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -1609,7 +1609,7 @@ mod sys { } pub(crate) fn init_module(vm: &VirtualMachine, module: &Py, builtins: &Py) { - sys::extend_module(vm, module).unwrap(); + sys::module_exec(vm, module).unwrap(); let modules = vm.ctx.new_dict(); modules diff --git a/crates/vm/src/vm/interpreter.rs b/crates/vm/src/vm/interpreter.rs index 567bc40a2..529a89d10 100644 --- a/crates/vm/src/vm/interpreter.rs +++ b/crates/vm/src/vm/interpreter.rs @@ -38,7 +38,7 @@ impl Interpreter { /// use rustpython_vm::Interpreter; /// Interpreter::with_init(Default::default(), |vm| { /// // put this line to add stdlib to the vm - /// // vm.add_native_modules(rustpython_stdlib::get_module_inits()); + /// // vm.add_native_module_defs(rustpython_stdlib::get_module_defs()); /// }).enter(|vm| { /// vm.run_code_string(vm.new_scope_with_builtins(), "print(1)", "<...>".to_owned()); /// }); diff --git a/crates/vm/src/vm/mod.rs b/crates/vm/src/vm/mod.rs index 54def5d75..3149cb092 100644 --- a/crates/vm/src/vm/mod.rs +++ b/crates/vm/src/vm/mod.rs @@ -479,6 +479,7 @@ impl VirtualMachine { } /// Can only be used in the initialization closure passed to [`Interpreter::with_init`] + #[deprecated(note = "use add_native_module_def(s) with multi-phase module initialization")] pub fn add_native_module(&mut self, name: S, module: stdlib::StdlibInitFunc) where S: Into>, @@ -486,6 +487,7 @@ impl VirtualMachine { self.state_mut().module_inits.insert(name.into(), module); } + #[deprecated(note = "use add_native_module_defs with multi-phase module initialization")] pub fn add_native_modules(&mut self, iter: I) where I: IntoIterator, stdlib::StdlibInitFunc)>, @@ -1444,7 +1446,7 @@ fn test_nested_frozen() { use rustpython_vm as vm; vm::Interpreter::with_init(Default::default(), |vm| { - // vm.add_native_modules(rustpython_stdlib::get_module_inits()); + // vm.add_native_module_defs(rustpython_stdlib::get_module_defs()); vm.add_frozen(rustpython_vm::py_freeze!( dir = "../../extra_tests/snippets" )); diff --git a/crates/wasm/src/browser_module.rs b/crates/wasm/src/browser_module.rs index 9b6219cab..2ba6fe27c 100644 --- a/crates/wasm/src/browser_module.rs +++ b/crates/wasm/src/browser_module.rs @@ -1,6 +1,6 @@ use rustpython_vm::VirtualMachine; -pub(crate) use _browser::make_module; +pub(crate) use _browser::module_def; #[pymodule] mod _browser { @@ -259,6 +259,6 @@ mod _browser { } pub fn setup_browser_module(vm: &mut VirtualMachine) { - vm.add_native_module("_browser".to_owned(), Box::new(make_module)); + vm.add_native_module_def("_browser".to_owned(), module_def); vm.add_frozen(py_freeze!(dir = "Lib")); } diff --git a/crates/wasm/src/js_module.rs b/crates/wasm/src/js_module.rs index 11d23bdf6..371c9c936 100644 --- a/crates/wasm/src/js_module.rs +++ b/crates/wasm/src/js_module.rs @@ -621,8 +621,8 @@ mod _js { } } -pub(crate) use _js::make_module; +pub(crate) use _js::module_def; pub fn setup_js_module(vm: &mut VirtualMachine) { - vm.add_native_module("_js".to_owned(), Box::new(make_module)); + vm.add_native_module_def("_js".to_owned(), module_def); } diff --git a/crates/wasm/src/vm_class.rs b/crates/wasm/src/vm_class.rs index 0339a4782..71f2e9a83 100644 --- a/crates/wasm/src/vm_class.rs +++ b/crates/wasm/src/vm_class.rs @@ -7,10 +7,8 @@ use alloc::rc::{Rc, Weak}; use core::cell::RefCell; use js_sys::{Object, TypeError}; use rustpython_vm::{ - Interpreter, PyObjectRef, PyPayload, PyRef, PyResult, Settings, VirtualMachine, - builtins::{PyModule, PyWeak}, - compiler::Mode, - scope::Scope, + Interpreter, PyObjectRef, PyRef, PyResult, Settings, VirtualMachine, builtins::PyWeak, + compiler::Mode, scope::Scope, }; use std::collections::HashMap; use wasm_bindgen::prelude::*; @@ -24,16 +22,17 @@ pub(crate) struct StoredVirtualMachine { } #[pymodule] -mod _window {} +mod _window { + use super::{js_module, wasm_builtins}; + use rustpython_vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule}; -fn init_window_module(vm: &VirtualMachine) -> PyRef { - let module = _window::make_module(vm); - - extend_module!(vm, &module, { - "window" => js_module::PyJsValue::new(wasm_builtins::window()).into_ref(&vm.ctx), - }); - - module + pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py) -> PyResult<()> { + __module_exec(vm, module); + extend_module!(vm, module, { + "window" => js_module::PyJsValue::new(wasm_builtins::window()).into_ref(&vm.ctx), + }); + Ok(()) + } } impl StoredVirtualMachine { @@ -43,7 +42,7 @@ impl StoredVirtualMachine { settings.allow_external_library = false; let interp = Interpreter::with_init(settings, |vm| { #[cfg(feature = "freeze-stdlib")] - vm.add_native_modules(rustpython_stdlib::get_module_inits()); + vm.add_native_module_defs(rustpython_stdlib::get_module_defs()); #[cfg(feature = "freeze-stdlib")] vm.add_frozen(rustpython_pylib::FROZEN_STDLIB); @@ -52,7 +51,7 @@ impl StoredVirtualMachine { js_module::setup_js_module(vm); if inject_browser_module { - vm.add_native_module("_window".to_owned(), Box::new(init_window_module)); + vm.add_native_module_def("_window".to_owned(), _window::module_def); setup_browser_module(vm); } diff --git a/examples/call_between_rust_and_python.rs b/examples/call_between_rust_and_python.rs index dee170584..f916af114 100644 --- a/examples/call_between_rust_and_python.rs +++ b/examples/call_between_rust_and_python.rs @@ -6,10 +6,7 @@ pub fn main() { let interp = rustpython::InterpreterConfig::new() .init_stdlib() .init_hook(Box::new(|vm| { - vm.add_native_module( - "rust_py_module".to_owned(), - Box::new(rust_py_module::make_module), - ); + vm.add_native_module_def("rust_py_module".to_owned(), rust_py_module::module_def); })) .interpreter(); diff --git a/examples/generator.rs b/examples/generator.rs index 27733a191..fc53e2d50 100644 --- a/examples/generator.rs +++ b/examples/generator.rs @@ -43,7 +43,7 @@ gen() fn main() -> ExitCode { let interp = vm::Interpreter::with_init(Default::default(), |vm| { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); + vm.add_native_module_defs(rustpython_stdlib::get_module_defs()); }); let result = py_main(&interp); vm::common::os::exit_code(interp.run(|_vm| result)) diff --git a/examples/package_embed.rs b/examples/package_embed.rs index e82e71f5c..4a3deaf7b 100644 --- a/examples/package_embed.rs +++ b/examples/package_embed.rs @@ -20,7 +20,7 @@ fn main() -> ExitCode { let mut settings = vm::Settings::default(); settings.path_list.push("Lib".to_owned()); let interp = vm::Interpreter::with_init(settings, |vm| { - vm.add_native_modules(rustpython_stdlib::get_module_inits()); + vm.add_native_module_defs(rustpython_stdlib::get_module_defs()); }); let result = py_main(&interp); let result = result.map(|result| { diff --git a/src/interpreter.rs b/src/interpreter.rs index 3141f692a..0ad7763a1 100644 --- a/src/interpreter.rs +++ b/src/interpreter.rs @@ -1,4 +1,6 @@ -use rustpython_vm::{Interpreter, PyRef, Settings, VirtualMachine, builtins::PyModule}; +use rustpython_vm::{ + Interpreter, PyRef, Settings, VirtualMachine, builtins::PyModule, stdlib::StdlibDefFunc, +}; pub type InitHook = Box; @@ -34,9 +36,9 @@ pub type InitHook = Box; /// /// let interpreter = rustpython::InterpreterConfig::new() /// .init_stdlib() -/// .add_native_module( +/// .add_native_module_def( /// "your_module_name".to_owned(), -/// your_module::make_module, +/// your_module::module_def, /// ) /// .interpreter(); /// ``` @@ -78,7 +80,14 @@ impl InterpreterConfig { self } + /// Adds a native module definition to the interpreter. + pub fn add_native_module_def(self, name: String, def_func: StdlibDefFunc) -> Self { + self.init_hook(Box::new(move |vm| vm.add_native_module_def(name, def_func))) + } + /// Adds a native module to the interpreter. + #[deprecated(note = "use add_native_module_def with multi-phase module initialization")] + #[allow(deprecated)] pub fn add_native_module( self, name: String, @@ -101,9 +110,7 @@ impl InterpreterConfig { /// Initializes all standard library modules for the given VM. #[cfg(feature = "stdlib")] pub fn init_stdlib(vm: &mut VirtualMachine) { - // Add multi-phase init modules first (they're checked first in import_builtin) vm.add_native_module_defs(rustpython_stdlib::get_module_defs()); - vm.add_native_modules(rustpython_stdlib::get_module_inits()); #[cfg(feature = "freeze-stdlib")] setup_frozen_stdlib(vm); diff --git a/src/lib.rs b/src/lib.rs index cf5227a70..63eeee927 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,7 @@ //! use rustpython_vm::{pymodule, py_freeze}; //! fn main() { //! rustpython::run(|vm| { -//! vm.add_native_module("my_mod".to_owned(), Box::new(my_mod::make_module)); +//! vm.add_native_module_def("my_mod".to_owned(), my_mod::module_def); //! vm.add_frozen(py_freeze!(source = "def foo(): pass", module_name = "other_thing")); //! }); //! }