Rust-related/RustPython
1
0
Fork 1
mirror of https://github.com/RustPython/RustPython.git synced 2026-06-02 19:39:49 +09:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6740 from youknowone/multi-phase-module-init

Browse Source 
Multi phase module init
This commit is contained in:
Jeong, YunWon
2026-01-22 12:12:43 +09:00
committed by GitHub
parent 512c84dbc0 20376451eb
commit 47df6dd270
108 changed files with 1230 additions and 1111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
Lib/test/test_pickle.py vendored
View File

@@ -136,11 +136,6 @@ class PyPicklerTests(AbstractPickleTests, unittest.TestCase):
def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_buffers_error()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_builtin_functions()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes
@@ -166,42 +161,6 @@ class PyPicklerTests(AbstractPickleTests, unittest.TestCase):
def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_oob_buffers_writable_to_readonly()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_buffers_error()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_builtin_functions()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_bytearray_memoization()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_bytes_memoization()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_in_band_buffers()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_oob_buffers()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_oob_buffers_writable_to_readonly()
class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests,
BigmemPickleTests, unittest.TestCase):
@@ -250,11 +209,6 @@ class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests,
def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_buffers_error()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_builtin_functions()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes

5
Lib/test/test_pickletools.py vendored
View File

@@ -67,11 +67,6 @@ class OptimizedPickleTests(AbstractPickleTests, unittest.TestCase):
def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_buffers_error()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes
return super().test_builtin_functions()
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes

7
benches/execution.rs
View File

@@ -24,10 +24,9 @@ fn bench_rustpython_code(b: &mut Bencher, name: &str, source: &str) {
settings.path_list.push("Lib/".to_string());
settings.write_bytecode = false;
settings.user_site_directory = false;
Interpreter::with_init(settings, |vm| {
vm.add_native_modules(rustpython_stdlib::get_module_inits());
})
.enter(|vm| {
let builder = Interpreter::builder(settings);
let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx);
builder.add_native_modules(&defs).build().enter(|vm| {
// Note: bench_cpython is both compiling and executing the code.
// As such we compile the code in the benchmark loop as well.
b.iter(|| {

10
benches/microbenchmarks.rs
View File

@@ -113,12 +113,10 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenc
settings.write_bytecode = false;
settings.user_site_directory = false;
Interpreter::with_init(settings, |vm| {
for (name, init) in rustpython_stdlib::get_module_inits() {
vm.add_native_module(name, init);
}
})
.enter(|vm| {
let builder = Interpreter::builder(settings);
let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx);
let interp = builder.add_native_modules(&defs).build();
interp.enter(|vm| {
let setup_code = vm
.compile(&bench.setup, Mode::Exec, bench.name.to_owned())
.expect("Error compiling setup code");

4
crates/common/src/crt_fd.rs
View File

@@ -5,8 +5,10 @@ use alloc::fmt;
use core::cmp;
use std::{ffi, io};
#[cfg(unix)]
use std::os::fd::AsFd;
#[cfg(not(windows))]
use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd};
use std::os::fd::{AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd};
#[cfg(windows)]
use std::os::windows::io::BorrowedHandle;

33
crates/derive-impl/src/pymodule.rs
View File

@@ -58,7 +58,7 @@ struct ModuleContext {
name: String,
function_items: FunctionNursery,
attribute_items: ItemNursery,
has_extend_module: bool, // TODO: check if `fn extend_module` exists
has_module_exec: bool,
errors: Vec<syn::Error>,
}
@@ -82,6 +82,12 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To
// collect to context
for item in items.iter_mut() {
// Check if module_exec function is already defined
if let Item::Fn(func) = item
&& func.sig.ident == "module_exec"
{
context.has_module_exec = true;
}
if matches!(item, Item::Impl(_) | Item::Trait(_)) {
// #[pyclass] implementations
continue;
@@ -123,7 +129,7 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To
pub(crate) const DOC: Option<&'static str> = #doc;
},
parse_quote! {
pub(crate) fn __module_def(
pub(crate) fn module_def(
ctx: &::rustpython_vm::Context,
) -> &'static ::rustpython_vm::builtins::PyModuleDef {
DEF.get_or_init(|| {
@@ -133,29 +139,17 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To
methods: METHOD_DEFS,
slots: Default::default(),
};
def.slots.exec = Some(extend_module);
def.slots.exec = Some(module_exec);
def
})
}
},
parse_quote! {
#[allow(dead_code)]
pub(crate) fn make_module(
vm: &::rustpython_vm::VirtualMachine
) -> ::rustpython_vm::PyRef<::rustpython_vm::builtins::PyModule> {
use ::rustpython_vm::PyPayload;
let module = ::rustpython_vm::builtins::PyModule::from_def(__module_def(&vm.ctx)).into_ref(&vm.ctx);
__init_dict(vm, &module);
extend_module(vm, &module).unwrap();
module
}
},
]);
}
if !is_submodule && !context.has_extend_module {
if !is_submodule && !context.has_module_exec {
items.push(parse_quote! {
pub(crate) fn extend_module(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> {
__extend_module(vm, module);
pub(crate) fn module_exec(vm: &::rustpython_vm::VirtualMachine, module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>) -> ::rustpython_vm::PyResult<()> {
__module_exec(vm, module);
Ok(())
}
});
@@ -192,11 +186,10 @@ pub fn impl_pymodule(attr: PunctuatedNestedMeta, module_item: Item) -> Result<To
}
},
parse_quote! {
pub(crate) fn __extend_module(
pub(crate) fn __module_exec(
vm: &::rustpython_vm::VirtualMachine,
module: &::rustpython_vm::Py<::rustpython_vm::builtins::PyModule>,
) {
module.__init_methods(vm).unwrap();
__init_attributes(vm, module);
}
},

4
crates/derive/src/lib.rs
View File

@@ -143,8 +143,8 @@ pub fn pyexception(attr: TokenStream, item: TokenStream) -> TokenStream {
}
/// This attribute must be applied to an inline module.
/// It defines a Python module in the form a `make_module` function in the module;
/// this has to be used in a `get_module_inits` to properly register the module.
/// It defines a Python module in the form of a `module_def` function in the module;
/// this has to be used in a `add_native_module` to properly register the module.
/// Additionally, this macro defines 'MODULE_NAME' and 'DOC' in the module.
/// # Arguments
/// - `name`: the name of the python module,

4
crates/stdlib/src/opcode.rs → crates/stdlib/src/_opcode.rs
View File

@@ -1,7 +1,7 @@
pub(crate) use opcode::make_module;
pub(crate) use _opcode::module_def;
#[pymodule]
mod opcode {
mod _opcode {
use crate::vm::{
AsObject, PyObjectRef, PyResult, VirtualMachine,
builtins::{PyInt, PyIntRef},

34
crates/stdlib/src/sqlite.rs → crates/stdlib/src/_sqlite3.rs
View File

@@ -8,18 +8,10 @@
// spell-checker:ignore cantlock commithook foreignkey notnull primarykey gettemppath autoindex convpath
// spell-checker:ignore dbmoved vnode nbytes
use rustpython_vm::{AsObject, PyRef, VirtualMachine, builtins::PyModule};
// pub(crate) use _sqlite::make_module;
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
// TODO: sqlite version check
let module = _sqlite::make_module(vm);
_sqlite::setup_module(module.as_object(), vm);
module
}
pub(crate) use _sqlite3::module_def;
#[pymodule]
mod _sqlite {
mod _sqlite3 {
use libsqlite3_sys::{
SQLITE_BLOB, SQLITE_DETERMINISTIC, SQLITE_FLOAT, SQLITE_INTEGER, SQLITE_NULL,
SQLITE_OPEN_CREATE, SQLITE_OPEN_READWRITE, SQLITE_OPEN_URI, SQLITE_TEXT, SQLITE_TRACE_STMT,
@@ -58,8 +50,8 @@ mod _sqlite {
TryFromBorrowedObject, VirtualMachine, atomic_func,
builtins::{
PyBaseException, PyBaseExceptionRef, PyByteArray, PyBytes, PyDict, PyDictRef, PyFloat,
PyInt, PyIntRef, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType, PyTypeRef,
PyUtf8Str, PyUtf8StrRef,
PyInt, PyIntRef, PyModule, PySlice, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType,
PyTypeRef, PyUtf8Str, PyUtf8StrRef,
},
convert::IntoObject,
function::{
@@ -852,26 +844,26 @@ mod _sqlite {
.expect("enable traceback not initialize")
}
pub(super) fn setup_module(module: &PyObject, vm: &VirtualMachine) {
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
__module_exec(vm, module);
for (name, code) in ERROR_CODES {
let name = vm.ctx.intern_str(*name);
let code = vm.new_pyobj(*code);
module.set_attr(name, code, vm).unwrap();
module.set_attr(name, code, vm)?;
}
setup_module_exceptions(module, vm);
setup_module_exceptions(module.as_object(), vm);
let _ = CONVERTERS.set(vm.ctx.new_dict());
let _ = ADAPTERS.set(vm.ctx.new_dict());
let _ = USER_FUNCTION_EXCEPTION.set(PyAtomicRef::from(None));
let _ = ENABLE_TRACEBACK.set(Radium::new(false));
module
.set_attr("converters", converters().to_owned(), vm)
.unwrap();
module
.set_attr("adapters", adapters().to_owned(), vm)
.unwrap();
module.set_attr("converters", converters().to_owned(), vm)?;
module.set_attr("adapters", adapters().to_owned(), vm)?;
Ok(())
}
#[pyattr]

50
crates/stdlib/src/array.rs
View File

@@ -1,33 +1,6 @@
// spell-checker:ignore typecode tofile tolist fromfile
use rustpython_vm::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = array::make_module(vm);
let array = module
.get_attr("array", vm)
.expect("Expect array has array type.");
let collections_abc = vm
.import("collections.abc", 0)
.expect("Expect collections exist.");
let abc = collections_abc
.get_attr("abc", vm)
.expect("Expect collections has abc submodule.");
let mutable_sequence = abc
.get_attr("MutableSequence", vm)
.expect("Expect collections.abc has MutableSequence type.");
let register = &mutable_sequence
.get_attr("register", vm)
.expect("Expect collections.abc.MutableSequence has register method.");
register
.call((array,), vm)
.expect("Expect collections.abc.MutableSequence.register(array.array) not fail.");
module
}
pub(crate) use array::module_def;
#[pymodule(name = "array")]
mod array {
@@ -1658,4 +1631,25 @@ mod array {
};
PyArray::from(array).into_ref_with_type(vm, cls)
}
// Register array.array as collections.abc.MutableSequence
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::vm::builtins::PyModule>,
) -> PyResult<()> {
__module_exec(vm, module);
let array_type = module
.get_attr("array", vm)
.expect("array module has array type");
// vm.import returns the top-level module, so we need to get abc submodule
let collections_abc = vm.import("collections.abc", 0)?;
let abc = collections_abc.get_attr("abc", vm)?;
let mutable_sequence = abc.get_attr("MutableSequence", vm)?;
let register = mutable_sequence.get_attr("register", vm)?;
register.call((array_type,), vm)?;
Ok(())
}
}

2
crates/stdlib/src/binascii.rs
View File

@@ -1,7 +1,7 @@
// spell-checker:ignore hexlify unhexlify uuencodes CRCTAB rlecode rledecode
pub(super) use decl::crc32;
pub(crate) use decl::make_module;
pub(crate) use decl::module_def;
use rustpython_vm::{VirtualMachine, builtins::PyBaseExceptionRef, convert::ToPyException};
const PAD: u8 = 61u8;

2
crates/stdlib/src/bisect.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _bisect::make_module;
pub(crate) use _bisect::module_def;
#[pymodule]
mod _bisect {

2
crates/stdlib/src/blake2.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:ignore usedforsecurity HASHXOF
pub(crate) use _blake2::make_module;
pub(crate) use _blake2::module_def;
#[pymodule]
mod _blake2 {

2
crates/stdlib/src/bz2.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:ignore compresslevel
pub(crate) use _bz2::make_module;
pub(crate) use _bz2::module_def;
#[pymodule]
mod _bz2 {

2
crates/stdlib/src/cmath.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use cmath::make_module;
pub(crate) use cmath::module_def;
#[pymodule]
mod cmath {

30
crates/stdlib/src/contextvars.rs
View File

@@ -1,20 +1,9 @@
use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, class::StaticType};
pub(crate) use _contextvars::module_def;
use crate::vm::PyRef;
use _contextvars::PyContext;
use core::cell::RefCell;
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = _contextvars::make_module(vm);
let token_type = module.get_attr("Token", vm).unwrap();
token_type
.set_attr(
"MISSING",
_contextvars::ContextTokenMissing::static_type().to_owned(),
vm,
)
.unwrap();
module
}
thread_local! {
// TODO: Vec doesn't seem to match copy behavior
static CONTEXTS: RefCell<Vec<PyRef<PyContext>>> = RefCell::default();
@@ -643,4 +632,17 @@ mod _contextvars {
fn copy_context(vm: &VirtualMachine) -> PyContext {
PyContext::current(vm).copy()
}
// Set Token.MISSING attribute
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::vm::builtins::PyModule>,
) -> PyResult<()> {
__module_exec(vm, module);
let token_type = module.get_attr("Token", vm)?;
token_type.set_attr("MISSING", ContextTokenMissing::static_type().to_owned(), vm)?;
Ok(())
}
}

2
crates/stdlib/src/csv.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _csv::make_module;
pub(crate) use _csv::module_def;
#[pymodule]
mod _csv {

2
crates/stdlib/src/faulthandler.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use decl::make_module;
pub(crate) use decl::module_def;
#[allow(static_mut_refs)] // TODO: group code only with static mut refs
#[pymodule(name = "faulthandler")]

2
crates/stdlib/src/fcntl.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:disable
pub(crate) use fcntl::make_module;
pub(crate) use fcntl::module_def;
#[pymodule]
mod fcntl {

2
crates/stdlib/src/gc.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use gc::make_module;
pub(crate) use gc::module_def;
#[pymodule]
mod gc {

2
crates/stdlib/src/grp.rs
View File

@@ -1,5 +1,5 @@
// spell-checker:disable
pub(crate) use grp::make_module;
pub(crate) use grp::module_def;
#[pymodule]
mod grp {

2
crates/stdlib/src/hashlib.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:ignore usedforsecurity HASHXOF
pub(crate) use _hashlib::make_module;
pub(crate) use _hashlib::module_def;
#[pymodule]
pub mod _hashlib {

2
crates/stdlib/src/json.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _json::make_module;
pub(crate) use _json::module_def;
mod machinery;
#[pymodule]

212
crates/stdlib/src/lib.rs
View File

@@ -38,10 +38,10 @@ mod json;
#[cfg(not(any(target_os = "ios", target_arch = "wasm32")))]
mod locale;
mod _opcode;
mod math;
#[cfg(any(unix, windows))]
mod mmap;
mod opcode;
mod pyexpat;
mod pystruct;
mod random;
@@ -65,6 +65,11 @@ mod posixshmem;
#[cfg(unix)]
mod posixsubprocess;
// libc is missing constants on redox
#[cfg(all(
feature = "sqlite",
not(any(target_os = "android", target_arch = "wasm32"))
))]
mod _sqlite3;
#[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))]
mod grp;
#[cfg(windows)]
@@ -75,11 +80,6 @@ mod resource;
mod scproxy;
#[cfg(any(unix, windows, target_os = "wasi"))]
mod select;
#[cfg(all(
feature = "sqlite",
not(any(target_os = "android", target_arch = "wasm32"))
))]
mod sqlite;
#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))]
mod openssl;
@@ -105,131 +105,87 @@ mod tkinter;
use rustpython_common as common;
use rustpython_vm as vm;
use crate::vm::{builtins, stdlib::StdlibInitFunc};
use alloc::borrow::Cow;
use crate::vm::{Context, builtins};
pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInitFunc)> {
macro_rules! modules {
{
$(
#[cfg($cfg:meta)]
{ $( $key:expr => $val:expr),* $(,)? }
)*
} => {{
[
$(
$(#[cfg($cfg)] (Cow::<'static, str>::from($key), Box::new($val) as StdlibInitFunc),)*
)*
]
.into_iter()
}};
}
modules! {
#[cfg(all())]
{
"array" => array::make_module,
"binascii" => binascii::make_module,
"_bisect" => bisect::make_module,
"_bz2" => bz2::make_module,
"cmath" => cmath::make_module,
"_contextvars" => contextvars::make_module,
"_csv" => csv::make_module,
"faulthandler" => faulthandler::make_module,
"gc" => gc::make_module,
"_hashlib" => hashlib::make_module,
"_sha1" => sha1::make_module,
"_sha3" => sha3::make_module,
"_sha256" => sha256::make_module,
"_sha512" => sha512::make_module,
"_md5" => md5::make_module,
"_blake2" => blake2::make_module,
"_json" => json::make_module,
"math" => math::make_module,
"pyexpat" => pyexpat::make_module,
"_opcode" => opcode::make_module,
"_random" => random::make_module,
"_statistics" => statistics::make_module,
"_struct" => pystruct::make_module,
"unicodedata" => unicodedata::make_module,
"zlib" => zlib::make_module,
"_statistics" => statistics::make_module,
"_suggestions" => suggestions::make_module,
// crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module,
}
/// Returns module definitions for multi-phase init modules.
/// These modules are added to sys.modules BEFORE their exec function runs,
/// allowing safe circular imports.
pub fn stdlib_module_defs(ctx: &Context) -> Vec<&'static builtins::PyModuleDef> {
vec![
_opcode::module_def(ctx),
array::module_def(ctx),
binascii::module_def(ctx),
bisect::module_def(ctx),
blake2::module_def(ctx),
bz2::module_def(ctx),
cmath::module_def(ctx),
contextvars::module_def(ctx),
csv::module_def(ctx),
faulthandler::module_def(ctx),
#[cfg(any(unix, target_os = "wasi"))]
{
"fcntl" => fcntl::make_module,
}
#[cfg(any(unix, windows, target_os = "wasi"))]
{
"select" => select::make_module,
}
#[cfg(not(target_arch = "wasm32"))]
{
"_multiprocessing" => multiprocessing::make_module,
"_socket" => socket::make_module,
}
#[cfg(not(any(target_os = "android", target_arch = "wasm32")))]
{
"_lzma" => lzma::make_module,
}
#[cfg(all(feature = "sqlite", not(any(target_os = "android", target_arch = "wasm32"))))]
{
"_sqlite3" => sqlite::make_module,
}
#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-rustls"))]
{
"_ssl" => ssl::make_module,
}
#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))]
{
"_ssl" => openssl::make_module,
}
#[cfg(windows)]
{
"_overlapped" => overlapped::make_module,
}
// Unix-only
#[cfg(unix)]
{
"_posixsubprocess" => posixsubprocess::make_module,
}
#[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))]
{
"_posixshmem" => posixshmem::make_module,
}
#[cfg(any(unix, windows))]
{
"mmap" => mmap::make_module,
}
#[cfg(all(unix, not(target_os = "redox")))]
{
"syslog" => syslog::make_module,
"resource" => resource::make_module,
}
#[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))]
{
"termios" => termios::make_module,
}
fcntl::module_def(ctx),
gc::module_def(ctx),
#[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))]
{
"grp" => grp::make_module,
}
#[cfg(target_os = "macos")]
{
"_scproxy" => scproxy::make_module,
}
#[cfg(not(any(target_os = "android", target_os = "ios", target_os = "windows", target_arch = "wasm32", target_os = "redox")))]
{
"_uuid" => uuid::make_module,
}
grp::module_def(ctx),
hashlib::module_def(ctx),
json::module_def(ctx),
#[cfg(not(any(target_os = "ios", target_arch = "wasm32")))]
{
"_locale" => locale::make_module,
}
locale::module_def(ctx),
#[cfg(not(any(target_os = "android", target_arch = "wasm32")))]
lzma::module_def(ctx),
math::module_def(ctx),
md5::module_def(ctx),
#[cfg(any(unix, windows))]
mmap::module_def(ctx),
#[cfg(not(target_arch = "wasm32"))]
multiprocessing::module_def(ctx),
#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-openssl"))]
openssl::module_def(ctx),
#[cfg(windows)]
overlapped::module_def(ctx),
#[cfg(unix)]
posixsubprocess::module_def(ctx),
#[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))]
posixshmem::module_def(ctx),
pyexpat::module_def(ctx),
pystruct::module_def(ctx),
random::module_def(ctx),
#[cfg(all(unix, not(target_os = "redox")))]
resource::module_def(ctx),
#[cfg(target_os = "macos")]
scproxy::module_def(ctx),
#[cfg(any(unix, windows, target_os = "wasi"))]
select::module_def(ctx),
sha1::module_def(ctx),
sha256::module_def(ctx),
sha3::module_def(ctx),
sha512::module_def(ctx),
#[cfg(not(target_arch = "wasm32"))]
socket::module_def(ctx),
#[cfg(all(
feature = "sqlite",
not(any(target_os = "android", target_arch = "wasm32"))
))]
_sqlite3::module_def(ctx),
#[cfg(all(not(target_arch = "wasm32"), feature = "ssl-rustls"))]
ssl::module_def(ctx),
statistics::module_def(ctx),
suggestions::module_def(ctx),
#[cfg(all(unix, not(target_os = "redox")))]
syslog::module_def(ctx),
#[cfg(all(unix, not(any(target_os = "ios", target_os = "redox"))))]
termios::module_def(ctx),
#[cfg(feature = "tkinter")]
{
"_tkinter" => tkinter::make_module,
}
}
tkinter::module_def(ctx),
unicodedata::module_def(ctx),
#[cfg(not(any(
target_os = "android",
target_os = "ios",
target_os = "windows",
target_arch = "wasm32",
target_os = "redox"
)))]
uuid::module_def(ctx),
zlib::module_def(ctx),
]
}

2
crates/stdlib/src/locale.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:ignore abday abmon yesexpr noexpr CRNCYSTR RADIXCHAR AMPM THOUSEP
pub(crate) use _locale::make_module;
pub(crate) use _locale::module_def;
#[cfg(windows)]
#[repr(C)]

2
crates/stdlib/src/lzma.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:ignore ARMTHUMB
pub(crate) use _lzma::make_module;
pub(crate) use _lzma::module_def;
#[pymodule]
mod _lzma {

2
crates/stdlib/src/math.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use math::make_module;
pub(crate) use math::module_def;
use crate::vm::{VirtualMachine, builtins::PyBaseExceptionRef};

2
crates/stdlib/src/md5.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _md5::make_module;
pub(crate) use _md5::module_def;
#[pymodule]
mod _md5 {

2
crates/stdlib/src/mmap.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:disable
//! mmap module
pub(crate) use mmap::make_module;
pub(crate) use mmap::module_def;
#[pymodule]
mod mmap {

2
crates/stdlib/src/multiprocessing.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _multiprocessing::make_module;
pub(crate) use _multiprocessing::module_def;
#[cfg(windows)]
#[pymodule]

35
crates/stdlib/src/openssl.rs
View File

@@ -23,32 +23,25 @@ cfg_if::cfg_if! {
}
}
use crate::vm::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) use _ssl::module_def;
use openssl_probe::ProbeResult;
use std::sync::LazyLock;
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
// if openssl is vendored, it doesn't know the locations
// of system certificates - cache the probe result now.
#[cfg(openssl_vendored)]
LazyLock::force(&PROBE);
_ssl::make_module(vm)
}
// define our own copy of ProbeResult so we can handle the vendor case
// easily, without having to have a bunch of cfgs
cfg_if::cfg_if! {
if #[cfg(openssl_vendored)] {
static PROBE: LazyLock<ProbeResult> = LazyLock::new(openssl_probe::probe);
fn probe() -> &'static ProbeResult { &PROBE }
} else {
static EMPTY_PROBE: LazyLock<ProbeResult> = LazyLock::new(|| ProbeResult { cert_file: None, cert_dir: vec![] });
fn probe() -> &'static ProbeResult {
&EMPTY_PROBE
}
static PROBE: LazyLock<ProbeResult> = LazyLock::new(|| ProbeResult { cert_file: None, cert_dir: vec![] });
}
}
fn probe() -> &'static ProbeResult {
&PROBE
}
#[allow(non_upper_case_globals)]
#[pymodule(with(cert::ssl_cert, ssl_error::ssl_error, ossl101, ossl111, windows))]
mod _ssl {
@@ -67,8 +60,8 @@ mod _ssl {
vm::{
AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
builtins::{
PyBaseException, PyBaseExceptionRef, PyBytesRef, PyListRef, PyStrRef, PyType,
PyWeak,
PyBaseException, PyBaseExceptionRef, PyBytesRef, PyListRef, PyModule, PyStrRef,
PyType, PyWeak,
},
class_or_notimplemented,
convert::ToPyException,
@@ -104,6 +97,16 @@ mod _ssl {
// Import certificate types from parent module
use super::cert::{self, cert_to_certificate, cert_to_py};
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
// if openssl is vendored, it doesn't know the locations
// of system certificates - cache the probe result now.
#[cfg(openssl_vendored)]
std::sync::LazyLock::force(&super::PROBE);
__module_exec(vm, module);
Ok(())
}
// Re-export PySSLCertificate to make it available in the _ssl module
// It will be automatically exposed to Python via #[pyclass]
#[allow(unused_imports)]

2
crates/stdlib/src/overlapped.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:disable
pub(crate) use _overlapped::make_module;
pub(crate) use _overlapped::module_def;
#[allow(non_snake_case)]
#[pymodule]

2
crates/stdlib/src/posixshmem.rs
View File

@@ -1,5 +1,5 @@
#[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))]
pub(crate) use _posixshmem::make_module;
pub(crate) use _posixshmem::module_def;
#[cfg(all(unix, not(target_os = "redox"), not(target_os = "android")))]
#[pymodule]

2
crates/stdlib/src/posixsubprocess.rs
View File

@@ -22,7 +22,7 @@ use alloc::ffi::CString;
use core::{convert::Infallible as Never, ffi::CStr, marker::PhantomData, ops::Deref};
pub(crate) use _posixsubprocess::make_module;
pub(crate) use _posixsubprocess::module_def;
#[pymodule]
mod _posixsubprocess {

31
crates/stdlib/src/pyexpat.rs
View File

@@ -2,18 +2,7 @@
// spell-checker: ignore libexpat
use crate::vm::{PyRef, VirtualMachine, builtins::PyModule, extend_module};
pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = _pyexpat::make_module(vm);
extend_module!(vm, &module, {
"errors" => _errors::make_module(vm),
"model" => _model::make_module(vm),
});
module
}
pub(crate) use _pyexpat::module_def;
macro_rules! create_property {
($ctx: expr, $attributes: expr, $name: expr, $class: expr, $element: ident) => {
@@ -52,7 +41,8 @@ macro_rules! create_bool_property {
mod _pyexpat {
use crate::vm::{
Context, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine,
builtins::{PyBytesRef, PyStr, PyStrRef, PyType},
builtins::{PyBytesRef, PyModule, PyStr, PyStrRef, PyType},
extend_module,
function::ArgBytesLike,
function::{Either, IntoFuncArgs, OptionalArg},
};
@@ -60,6 +50,21 @@ mod _pyexpat {
use std::io::Cursor;
use xml::reader::XmlEvent;
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
__module_exec(vm, module);
// Add submodules
let model = super::_model::module_def(&vm.ctx).create_module(vm)?;
let errors = super::_errors::module_def(&vm.ctx).create_module(vm)?;
extend_module!(vm, module, {
"model" => model,
"errors" => errors,
});
Ok(())
}
type MutableObject = PyRwLock<PyObjectRef>;
#[pyattr(name = "version_info")]

2
crates/stdlib/src/pystruct.rs
View File

@@ -5,7 +5,7 @@
//! Use this rust module to do byte packing:
//! <https://docs.rs/byteorder/1.2.6/byteorder/>
pub(crate) use _struct::make_module;
pub(crate) use _struct::module_def;
#[pymodule]
pub(crate) mod _struct {

2
crates/stdlib/src/random.rs
View File

@@ -1,6 +1,6 @@
//! Random module.
pub(crate) use _random::make_module;
pub(crate) use _random::module_def;
#[pymodule]
mod _random {

2
crates/stdlib/src/re.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use re::make_module;
pub(crate) use re::module_def;
#[pymodule]
mod re {

2
crates/stdlib/src/resource.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:disable
pub(crate) use resource::make_module;
pub(crate) use resource::module_def;
#[pymodule]
mod resource {

2
crates/stdlib/src/scproxy.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _scproxy::make_module;
pub(crate) use _scproxy::module_def;
#[pymodule]
mod _scproxy {

38
crates/stdlib/src/select.rs
View File

@@ -1,25 +1,13 @@
// spell-checker:disable
pub(crate) use decl::module_def;
use crate::vm::{
PyObject, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef,
builtins::PyModule,
PyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, builtins::PyListRef,
};
use core::mem;
use std::io;
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
#[cfg(windows)]
crate::vm::windows::init_winsock();
#[cfg(unix)]
{
use crate::vm::class::PyClassImpl;
decl::poll::PyPoll::make_class(&vm.ctx);
}
decl::make_module(vm)
}
#[cfg(unix)]
mod platform {
pub use libc::{FD_ISSET, FD_SET, FD_SETSIZE, FD_ZERO, fd_set, select, timeval};
@@ -221,13 +209,27 @@ fn sec_to_timeval(sec: f64) -> timeval {
mod decl {
use super::*;
use crate::vm::{
PyObjectRef, PyResult, VirtualMachine,
builtins::PyTypeRef,
Py, PyObjectRef, PyResult, VirtualMachine,
builtins::{PyModule, PyTypeRef},
convert::ToPyException,
function::{Either, OptionalOption},
stdlib::time,
};
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
#[cfg(windows)]
crate::vm::windows::init_winsock();
#[cfg(unix)]
{
use crate::vm::class::PyClassImpl;
poll::PyPoll::make_class(&vm.ctx);
}
__module_exec(vm, module);
Ok(())
}
#[pyattr]
fn error(vm: &VirtualMachine) -> PyTypeRef {
vm.ctx.exceptions.os_error.to_owned()
@@ -545,7 +547,7 @@ mod decl {
pub(super) mod epoll {
use super::*;
use crate::vm::{
Py, PyPayload,
Py, PyPayload, PyRef,
builtins::PyType,
common::lock::{PyRwLock, PyRwLockReadGuard},
convert::{IntoPyException, ToPyObject},

2
crates/stdlib/src/sha1.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _sha1::make_module;
pub(crate) use _sha1::module_def;
#[pymodule]
mod _sha1 {

17
crates/stdlib/src/sha256.rs
View File

@@ -1,14 +1,7 @@
use crate::vm::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let _ = vm.import("_hashlib", 0);
_sha256::make_module(vm)
}
#[pymodule]
mod _sha256 {
use crate::hashlib::_hashlib::{HashArgs, local_sha224, local_sha256};
use crate::vm::{PyPayload, PyResult, VirtualMachine};
use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule};
#[pyfunction]
fn sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult {
@@ -19,4 +12,12 @@ mod _sha256 {
fn sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult {
Ok(local_sha256(args).into_pyobject(vm))
}
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
let _ = vm.import("_hashlib", 0);
__module_exec(vm, module);
Ok(())
}
}
pub(crate) use _sha256::module_def;

2
crates/stdlib/src/sha3.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _sha3::make_module;
pub(crate) use _sha3::module_def;
#[pymodule]
mod _sha3 {

17
crates/stdlib/src/sha512.rs
View File

@@ -1,14 +1,7 @@
use crate::vm::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let _ = vm.import("_hashlib", 0);
_sha512::make_module(vm)
}
#[pymodule]
mod _sha512 {
use crate::hashlib::_hashlib::{HashArgs, local_sha384, local_sha512};
use crate::vm::{PyPayload, PyResult, VirtualMachine};
use crate::vm::{Py, PyPayload, PyResult, VirtualMachine, builtins::PyModule};
#[pyfunction]
fn sha384(args: HashArgs, vm: &VirtualMachine) -> PyResult {
@@ -19,4 +12,12 @@ mod _sha512 {
fn sha512(args: HashArgs, vm: &VirtualMachine) -> PyResult {
Ok(local_sha512(args).into_pyobject(vm))
}
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
let _ = vm.import("_hashlib", 0);
__module_exec(vm, module);
Ok(())
}
}
pub(crate) use _sha512::module_def;

21
crates/stdlib/src/socket.rs
View File

@@ -1,27 +1,32 @@
// spell-checker:disable
use crate::vm::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) use _socket::module_def;
#[cfg(feature = "ssl")]
pub(super) use _socket::{PySocket, SelectKind, sock_select, timeout_error_msg};
pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
#[cfg(windows)]
crate::vm::windows::init_winsock();
_socket::make_module(vm)
}
#[pymodule]
mod _socket {
use crate::common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard};
use crate::vm::{
AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
builtins::{PyBaseExceptionRef, PyListRef, PyOSError, PyStrRef, PyTupleRef, PyTypeRef},
builtins::{
PyBaseExceptionRef, PyListRef, PyModule, PyOSError, PyStrRef, PyTupleRef, PyTypeRef,
},
common::os::ErrorExt,
convert::{IntoPyException, ToPyObject, TryFromBorrowedObject, TryFromObject},
function::{ArgBytesLike, ArgMemoryBuffer, Either, FsPath, OptionalArg, OptionalOption},
types::{Constructor, DefaultConstructor, Initializer, Representable},
utils::ToCString,
};
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
#[cfg(windows)]
crate::vm::windows::init_winsock();
__module_exec(vm, module);
Ok(())
}
use core::{
mem::MaybeUninit,
net::{Ipv4Addr, Ipv6Addr, SocketAddr},

2
crates/stdlib/src/ssl.rs
View File

@@ -25,7 +25,7 @@ mod compat;
// SSL exception types (shared with openssl backend)
mod error;
pub(crate) use _ssl::make_module;
pub(crate) use _ssl::module_def;
#[allow(non_snake_case)]
#[allow(non_upper_case_globals)]

2
crates/stdlib/src/statistics.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _statistics::make_module;
pub(crate) use _statistics::module_def;
#[pymodule]
mod _statistics {

2
crates/stdlib/src/suggestions.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _suggestions::make_module;
pub(crate) use _suggestions::module_def;
#[pymodule]
mod _suggestions {

2
crates/stdlib/src/syslog.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:ignore logoption openlog setlogmask upto NDELAY ODELAY
pub(crate) use syslog::make_module;
pub(crate) use syslog::module_def;
#[pymodule(name = "syslog")]
mod syslog {

2
crates/stdlib/src/termios.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:disable
pub(crate) use self::termios::make_module;
pub(crate) use self::termios::module_def;
#[pymodule]
mod termios {

2
crates/stdlib/src/tkinter.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:ignore createcommand
pub(crate) use self::_tkinter::make_module;
pub(crate) use self::_tkinter::module_def;
#[pymodule]
mod _tkinter {

53
crates/stdlib/src/unicodedata.rs
View File

@@ -4,35 +4,12 @@
// spell-checker:ignore nfkc unistr unidata
pub(crate) use unicodedata::module_def;
use crate::vm::{
PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyModule,
builtins::PyStr, convert::TryFromBorrowedObject,
PyObject, PyResult, VirtualMachine, builtins::PyStr, convert::TryFromBorrowedObject,
};
pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = unicodedata::make_module(vm);
let ucd: PyObjectRef = unicodedata::Ucd::new(unic_ucd_age::UNICODE_VERSION)
.into_ref(&vm.ctx)
.into();
for attr in [
"category",
"lookup",
"name",
"bidirectional",
"east_asian_width",
"normalize",
"mirrored",
] {
crate::vm::extend_module!(vm, &module, {
attr => ucd.get_attr(attr, vm).unwrap(),
});
}
module
}
enum NormalizeForm {
Nfc,
Nfkc,
@@ -60,7 +37,8 @@ impl<'a> TryFromBorrowedObject<'a> for NormalizeForm {
#[pymodule]
mod unicodedata {
use crate::vm::{
PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, builtins::PyStrRef,
Py, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
builtins::{PyModule, PyStrRef},
function::OptionalArg,
};
use itertools::Itertools;
@@ -73,6 +51,27 @@ mod unicodedata {
use unic_ucd_category::GeneralCategory;
use unicode_bidi_mirroring::is_mirroring;
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
__module_exec(vm, module);
// Add UCD methods as module-level functions
let ucd: PyObjectRef = Ucd::new(UNICODE_VERSION).into_ref(&vm.ctx).into();
for attr in [
"category",
"lookup",
"name",
"bidirectional",
"east_asian_width",
"normalize",
"mirrored",
] {
module.set_attr(attr, ucd.get_attr(attr, vm)?, vm)?;
}
Ok(())
}
#[pyattr]
#[pyclass(name = "UCD")]
#[derive(Debug, PyPayload)]

2
crates/stdlib/src/uuid.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _uuid::make_module;
pub(crate) use _uuid::module_def;
#[pymodule]
mod _uuid {

2
crates/stdlib/src/zlib.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:ignore compressobj decompressobj zdict chunksize zlibmodule miniz chunker
pub(crate) use zlib::make_module;
pub(crate) use zlib::module_def;
#[pymodule]
mod zlib {

2
crates/vm/src/builtins/mod.rs
View File

@@ -49,7 +49,7 @@ pub use mappingproxy::PyMappingProxy;
pub(crate) mod memory;
pub use memory::PyMemoryView;
pub(crate) mod module;
pub use module::{PyModule, PyModuleDef};
pub use module::{PyModule, PyModuleDef, PyModuleSlots};
pub(crate) mod namespace;
pub use namespace::PyNamespace;
pub(crate) mod object;

40
crates/vm/src/builtins/module.rs
View File

@@ -41,6 +41,46 @@ impl core::fmt::Debug for PyModuleSlots {
}
}
impl PyModuleDef {
/// Create a module from this definition (Phase 1 of multi-phase init).
///
/// This performs:
/// 1. Create module object (using create slot if provided)
/// 2. Initialize module dict from def
/// 3. Add methods to module
///
/// Does NOT add to sys.modules or call exec slot.
pub fn create_module(&'static self, vm: &VirtualMachine) -> PyResult<PyRef<PyModule>> {
use crate::PyPayload;
// Create module (use create slot if provided, else default creation)
let module = if let Some(create) = self.slots.create {
// Custom module creation
let spec = vm.ctx.new_str(self.name.as_str());
create(vm, spec.as_object(), self)?
} else {
// Default module creation
PyModule::from_def(self).into_ref(&vm.ctx)
};
// Initialize module dict and methods
PyModule::__init_dict_from_def(vm, &module);
module.__init_methods(vm)?;
Ok(module)
}
/// Execute the module's exec slot (Phase 2 of multi-phase init).
///
/// Calls the exec slot if present. Returns Ok(()) if no exec slot.
pub fn exec_module(&'static self, vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
if let Some(exec) = self.slots.exec {
exec(vm, module)?;
}
Ok(())
}
}
#[allow(clippy::new_without_default)] // avoid Default implementation
#[pyclass(module = false, name = "module")]
#[derive(Debug)]

38
crates/vm/src/import.rs
View File

@@ -87,16 +87,36 @@ pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult {
}
pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult {
let make_module_func = vm.state.module_inits.get(module_name).ok_or_else(|| {
vm.new_import_error(
format!("Cannot import builtin module {module_name}"),
vm.ctx.new_str(module_name),
)
})?;
let module = make_module_func(vm);
let sys_modules = vm.sys_module.get_attr("modules", vm)?;
sys_modules.set_item(module_name, module.as_object().to_owned(), vm)?;
Ok(module.into())
// Check if already in sys.modules (handles recursive imports)
if let Ok(module) = sys_modules.get_item(module_name, vm) {
return Ok(module);
}
// Try multi-phase init first (preferred for modules that import other modules)
if let Some(&def) = vm.state.module_defs.get(module_name) {
// Phase 1: Create and initialize module
let module = def.create_module(vm)?;
// Add to sys.modules BEFORE exec (critical for circular import handling)
sys_modules.set_item(module_name, module.clone().into(), vm)?;
// Phase 2: Call exec slot (can safely import other modules now)
// If exec fails, remove the partially-initialized module from sys.modules
if let Err(e) = def.exec_module(vm, &module) {
let _ = sys_modules.del_item(module_name, vm);
return Err(e);
}
return Ok(module.into());
}
// Module not found in module_defs
Err(vm.new_import_error(
format!("Cannot import builtin module {module_name}"),
vm.ctx.new_str(module_name),
))
}
#[cfg(feature = "rustpython-compiler")]

2
crates/vm/src/lib.rs
View File

@@ -99,7 +99,7 @@ pub use self::object::{
AsObject, Py, PyAtomicRef, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact,
PyResult, PyWeakRef,
};
pub use self::vm::{Context, Interpreter, Settings, VirtualMachine};
pub use self::vm::{Context, Interpreter, InterpreterBuilder, Settings, VirtualMachine};
pub use rustpython_common as common;
pub use rustpython_compiler_core::{bytecode, frozen};

2
crates/vm/src/stdlib/_abc.rs
View File

@@ -3,7 +3,7 @@
//! This module provides the C implementation of Abstract Base Classes (ABCs)
//! as defined in PEP 3119.
pub(crate) use _abc::make_module;
pub(crate) use _abc::module_def;
#[pymodule]
mod _abc {

10
crates/vm/src/stdlib/ast.rs
View File

@@ -1,14 +1,16 @@
//! `ast` standard module for abstract syntax trees.
//!
//! This module makes use of the parser logic, and translates all ast nodes
//! into python ast.AST objects.
pub(crate) use python::_ast::module_def;
mod pyast;
use crate::builtins::{PyInt, PyStr};
use crate::stdlib::ast::module::{Mod, ModInteractive};
use crate::stdlib::ast::node::BoxedSlice;
use crate::stdlib::ast::python::_ast;
use crate::{
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact, PyResult,
TryFromObject, VirtualMachine,
@@ -365,9 +367,3 @@ pub const PY_COMPILE_FLAGS_MASK: i32 = PY_COMPILE_FLAG_AST_ONLY
| CO_FUTURE_BARRY_AS_BDFL
| CO_FUTURE_GENERATOR_STOP
| CO_FUTURE_ANNOTATIONS;
pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = _ast::make_module(vm);
pyast::extend_module_nodes(vm, &module);
module
}

9
crates/vm/src/stdlib/ast/python.rs
View File

@@ -96,4 +96,13 @@ pub(crate) mod _ast {
#[pyattr(name = "PyCF_TYPE_COMMENTS")]
use super::PY_CF_TYPE_COMMENTS;
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::builtins::PyModule>,
) -> PyResult<()> {
__module_exec(vm, module);
super::super::pyast::extend_module_nodes(vm, module);
Ok(())
}
}

2
crates/vm/src/stdlib/atexit.rs
View File

@@ -1,5 +1,5 @@
pub use atexit::_run_exitfuncs;
pub(crate) use atexit::make_module;
pub(crate) use atexit::module_def;
#[pymodule]
mod atexit {

5
crates/vm/src/stdlib/builtins.rs
View File

@@ -2,7 +2,7 @@
//!
//! Implements the list of [builtin Python functions](https://docs.python.org/3/library/builtins.html).
use crate::{Py, VirtualMachine, builtins::PyModule, class::PyClassImpl};
pub(crate) use builtins::{__module_def, DOC};
pub(crate) use builtins::{DOC, module_def};
pub use builtins::{ascii, print, reversed};
#[pymodule]
@@ -1100,7 +1100,8 @@ pub fn init_module(vm: &VirtualMachine, module: &Py<PyModule>) {
crate::protocol::VecBuffer::make_class(&vm.ctx);
builtins::extend_module(vm, module).unwrap();
module.__init_methods(vm).unwrap();
builtins::module_exec(vm, module).unwrap();
let debug_mode: bool = vm.state.config.settings.optimize == 0;
// Create dynamic ExceptionGroup with multiple inheritance (BaseExceptionGroup + Exception)

2
crates/vm/src/stdlib/codecs.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _codecs::make_module;
pub(crate) use _codecs::module_def;
#[pymodule]
mod _codecs {

2
crates/vm/src/stdlib/collections.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _collections::make_module;
pub(crate) use _collections::module_def;
#[pymodule]
mod _collections {

64
crates/vm/src/stdlib/ctypes.rs
View File

@@ -10,8 +10,8 @@ mod structure;
mod union;
use crate::{
AsObject, Py, PyObjectRef, PyRef, PyResult, VirtualMachine,
builtins::{PyModule, PyStr, PyType},
AsObject, Py, PyObjectRef, PyResult, VirtualMachine,
builtins::{PyStr, PyType},
class::PyClassImpl,
types::TypeDataRef,
};
@@ -85,32 +85,9 @@ impl PyType {
}
// Dynamic type check helpers for PyCData
// These check if an object's type's metaclass is a subclass of a specific metaclass
pub(crate) use _ctypes::module_def;
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = _ctypes::make_module(vm);
let ctx = &vm.ctx;
PyCSimpleType::make_class(ctx);
array::PyCArrayType::make_class(ctx);
pointer::PyCPointerType::make_class(ctx);
structure::PyCStructType::make_class(ctx);
union::PyCUnionType::make_class(ctx);
function::PyCFuncPtrType::make_class(ctx);
extend_module!(vm, &module, {
"_CData" => PyCData::make_class(ctx),
"_SimpleCData" => PyCSimple::make_class(ctx),
"Array" => PyCArray::make_class(ctx),
"CField" => PyCField::make_class(ctx),
"CFuncPtr" => function::PyCFuncPtr::make_class(ctx),
"_Pointer" => PyCPointer::make_class(ctx),
"_pointer_type_cache" => ctx.new_dict(),
"_array_type_cache" => ctx.new_dict(),
"Structure" => PyCStructure::make_class(ctx),
"CThunkObject" => function::PyCThunk::make_class(ctx),
"Union" => PyCUnion::make_class(ctx),
});
module
}
// These check if an object's type's metaclass is a subclass of a specific metaclass
/// Size of long double - platform dependent
/// x86_64 macOS/Linux: 16 bytes (80-bit extended + padding)
@@ -1300,4 +1277,37 @@ pub(crate) mod _ctypes {
Ok(S_OK)
}
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::builtins::PyModule>,
) -> PyResult<()> {
use super::*;
__module_exec(vm, module);
let ctx = &vm.ctx;
PyCSimpleType::make_class(ctx);
array::PyCArrayType::make_class(ctx);
pointer::PyCPointerType::make_class(ctx);
structure::PyCStructType::make_class(ctx);
union::PyCUnionType::make_class(ctx);
function::PyCFuncPtrType::make_class(ctx);
extend_module!(vm, module, {
"_CData" => PyCData::make_class(ctx),
"_SimpleCData" => PyCSimple::make_class(ctx),
"Array" => PyCArray::make_class(ctx),
"CField" => PyCField::make_class(ctx),
"CFuncPtr" => function::PyCFuncPtr::make_class(ctx),
"_Pointer" => PyCPointer::make_class(ctx),
"_pointer_type_cache" => ctx.new_dict(),
"_array_type_cache" => ctx.new_dict(),
"Structure" => PyCStructure::make_class(ctx),
"CThunkObject" => function::PyCThunk::make_class(ctx),
"Union" => PyCUnion::make_class(ctx),
});
Ok(())
}
}

35
crates/vm/src/stdlib/errno.rs
View File

@@ -1,25 +1,26 @@
// spell-checker:disable
use crate::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) use errno_mod::module_def;
#[pymodule]
mod errno {}
#[pymodule(name = "errno")]
mod errno_mod {
use crate::{Py, PyResult, VirtualMachine, builtins::PyModule};
pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = errno::make_module(vm);
let errorcode = vm.ctx.new_dict();
extend_module!(vm, &module, {
"errorcode" => errorcode.clone(),
});
for (name, code) in ERROR_CODES {
let name = vm.ctx.intern_str(*name);
let code = vm.new_pyobj(*code);
errorcode
.set_item(&*code, name.to_owned().into(), vm)
.unwrap();
module.set_attr(name, code, vm).unwrap();
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
__module_exec(vm, module);
let errorcode = vm.ctx.new_dict();
extend_module!(vm, module, {
"errorcode" => errorcode.clone(),
});
for (name, code) in super::ERROR_CODES {
let name = vm.ctx.intern_str(*name);
let code = vm.new_pyobj(*code);
errorcode.set_item(&*code, name.to_owned().into(), vm)?;
module.set_attr(name, code, vm)?;
}
Ok(())
}
module
}
#[cfg(any(unix, windows, target_os = "wasi"))]

2
crates/vm/src/stdlib/functools.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _functools::make_module;
pub(crate) use _functools::module_def;
#[pymodule]
mod _functools {

49
crates/vm/src/stdlib/imp.rs
View File

@@ -1,6 +1,6 @@
use crate::frozen::FrozenModule;
use crate::{VirtualMachine, builtins::PyBaseExceptionRef};
pub(crate) use _imp::make_module;
pub(crate) use _imp::module_def;
pub use crate::vm::resolve_frozen_alias;
@@ -84,7 +84,7 @@ fn find_frozen(name: &str, vm: &VirtualMachine) -> Result<FrozenModule, FrozenEr
#[pymodule(with(lock))]
mod _imp {
use crate::{
PyObjectRef, PyRef, PyResult, VirtualMachine,
PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
builtins::{PyBytesRef, PyCode, PyMemoryView, PyModule, PyStrRef},
function::OptionalArg,
import, version,
@@ -106,7 +106,7 @@ mod _imp {
#[pyfunction]
fn is_builtin(name: PyStrRef, vm: &VirtualMachine) -> bool {
vm.state.module_inits.contains_key(name.as_str())
vm.state.module_defs.contains_key(name.as_str())
}
#[pyfunction]
@@ -119,19 +119,44 @@ mod _imp {
let sys_modules = vm.sys_module.get_attr("modules", vm).unwrap();
let name: PyStrRef = spec.get_attr("name", vm)?.try_into_value(vm)?;
let module = if let Ok(module) = sys_modules.get_item(&*name, vm) {
module
} else if let Some(make_module_func) = vm.state.module_inits.get(name.as_str()) {
make_module_func(vm).into()
} else {
vm.ctx.none()
};
Ok(module)
// Check sys.modules first
if let Ok(module) = sys_modules.get_item(&*name, vm) {
return Ok(module);
}
// Try multi-phase init modules first (they need special handling)
if let Some(&def) = vm.state.module_defs.get(name.as_str()) {
// Phase 1: Create module (use create slot if provided, else default creation)
let module = if let Some(create) = def.slots.create {
// Custom module creation
create(vm, &spec, def)?
} else {
// Default module creation
PyModule::from_def(def).into_ref(&vm.ctx)
};
// Initialize module dict and methods
// Corresponds to PyModule_FromDefAndSpec: md_def, _add_methods_to_object, PyModule_SetDocString
PyModule::__init_dict_from_def(vm, &module);
module.__init_methods(vm)?;
// Add to sys.modules BEFORE exec (critical for circular import handling)
sys_modules.set_item(&*name, module.clone().into(), vm)?;
// Phase 2: Call exec slot (can safely import other modules now)
if let Some(exec) = def.slots.exec {
exec(vm, &module)?;
}
return Ok(module.into());
}
Ok(vm.ctx.none())
}
#[pyfunction]
fn exec_builtin(_mod: PyRef<PyModule>) -> i32 {
// TODO: Should we do something here?
// For multi-phase init modules, exec is already called in create_builtin
0
}

38
crates/vm/src/stdlib/io.rs
View File

@@ -1,6 +1,8 @@
/*
* I/O core tools.
*/
pub(crate) use _io::module_def;
cfg_if::cfg_if! {
if #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] {
use crate::common::crt_fd::Offset;
@@ -19,7 +21,7 @@ cfg_if::cfg_if! {
}
use crate::{
PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine,
PyObjectRef, PyResult, TryFromObject, VirtualMachine,
builtins::{PyBaseExceptionRef, PyModule},
common::os::ErrorExt,
convert::{IntoPyException, ToPyException},
@@ -89,23 +91,6 @@ impl IntoPyException for std::io::Error {
}
}
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let ctx = &vm.ctx;
let module = _io::make_module(vm);
#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
fileio::extend_module(vm, &module).unwrap();
let unsupported_operation = _io::unsupported_operation().to_owned();
extend_module!(vm, &module, {
"UnsupportedOperation" => unsupported_operation,
"BlockingIOError" => ctx.exceptions.blocking_io_error.to_owned(),
});
module
}
// not used on all platforms
#[derive(Copy, Clone)]
#[repr(transparent)]
@@ -4743,8 +4728,23 @@ mod _io {
assert_eq!(buffered.getvalue(), data);
}
}
}
pub(crate) fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
// Call auto-generated initialization first
__module_exec(vm, module);
// Initialize FileIO types on non-WASM platforms
#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
super::fileio::module_exec(vm, module)?;
let unsupported_operation = unsupported_operation().to_owned();
extend_module!(vm, module, {
"UnsupportedOperation" => unsupported_operation,
"BlockingIOError" => vm.ctx.exceptions.blocking_io_error.to_owned(),
});
Ok(())
}
}
// disable FileIO on WASM
#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
#[pymodule]

2
crates/vm/src/stdlib/itertools.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use decl::make_module;
pub(crate) use decl::module_def;
#[pymodule(name = "itertools")]
mod decl {

2
crates/vm/src/stdlib/marshal.rs
View File

@@ -1,5 +1,5 @@
// spell-checker:ignore pyfrozen pycomplex
pub(crate) use decl::make_module;
pub(crate) use decl::module_def;
#[pymodule(name = "marshal")]
mod decl {

141
crates/vm/src/stdlib/mod.rs
View File

@@ -62,95 +62,66 @@ mod winapi;
#[cfg(windows)]
mod winreg;
use crate::{PyRef, VirtualMachine, builtins::PyModule};
use alloc::borrow::Cow;
use std::collections::HashMap;
use crate::{Context, builtins::PyModuleDef};
pub type StdlibInitFunc = Box<py_dyn_fn!(dyn Fn(&VirtualMachine) -> PyRef<PyModule>)>;
pub type StdlibMap = HashMap<Cow<'static, str>, StdlibInitFunc, ahash::RandomState>;
pub fn get_module_inits() -> StdlibMap {
macro_rules! modules {
{
$(
#[cfg($cfg:meta)]
{ $( $key:expr => $val:expr),* $(,)? }
)*
} => {{
let modules = [
$(
$(#[cfg($cfg)] (Cow::<'static, str>::from($key), Box::new($val) as StdlibInitFunc),)*
)*
];
modules.into_iter().collect()
}};
}
modules! {
#[cfg(all())]
{
"_abc" => _abc::make_module,
"atexit" => atexit::make_module,
"_codecs" => codecs::make_module,
"_collections" => collections::make_module,
"errno" => errno::make_module,
"_functools" => functools::make_module,
"itertools" => itertools::make_module,
"_io" => io::make_module,
"marshal" => marshal::make_module,
"_operator" => operator::make_module,
"_signal" => signal::make_module,
"_sre" => sre::make_module,
"_stat" => stat::make_module,
"_sysconfig" => sysconfig::make_module,
"_string" => string::make_module,
"time" => time::make_module,
"_typing" => typing::make_module,
"_weakref" => weakref::make_module,
"_imp" => imp::make_module,
"_warnings" => warnings::make_module,
sys::sysconfigdata_name() => sysconfigdata::make_module,
}
// parser related modules:
/// Returns module definitions for multi-phase init modules.
///
/// These modules use multi-phase initialization pattern:
/// 1. Create module from def and add to sys.modules
/// 2. Call exec slot (can safely import other modules without circular import issues)
pub fn builtin_module_defs(ctx: &Context) -> Vec<&'static PyModuleDef> {
vec![
_abc::module_def(ctx),
#[cfg(feature = "ast")]
{
"_ast" => ast::make_module,
}
// compiler related modules:
#[cfg(feature = "compiler")]
{
"_symtable" => symtable::make_module,
}
#[cfg(any(unix, target_os = "wasi"))]
{
"posix" => posix::make_module,
// "fcntl" => fcntl::make_module,
}
#[cfg(feature = "threading")]
{
"_thread" => thread::make_module,
}
// Unix-only
#[cfg(all(
unix,
not(any(target_os = "ios", target_os = "wasi", target_os = "redox"))
))]
{
"pwd" => pwd::make_module,
}
// Windows-only
#[cfg(windows)]
{
"nt" => nt::make_module,
"msvcrt" => msvcrt::make_module,
"_winapi" => winapi::make_module,
"winreg" => winreg::make_module,
}
ast::module_def(ctx),
atexit::module_def(ctx),
codecs::module_def(ctx),
collections::module_def(ctx),
#[cfg(all(
any(target_os = "linux", target_os = "macos", target_os = "windows"),
not(any(target_env = "musl", target_env = "sgx"))
))]
{
"_ctypes" => ctypes::make_module,
}
}
ctypes::module_def(ctx),
errno::module_def(ctx),
functools::module_def(ctx),
imp::module_def(ctx),
io::module_def(ctx),
itertools::module_def(ctx),
marshal::module_def(ctx),
#[cfg(windows)]
msvcrt::module_def(ctx),
#[cfg(windows)]
nt::module_def(ctx),
operator::module_def(ctx),
#[cfg(any(unix, target_os = "wasi"))]
posix::module_def(ctx),
#[cfg(all(
any(not(target_arch = "wasm32"), target_os = "wasi"),
not(any(unix, windows))
))]
posix::module_def(ctx),
#[cfg(all(
unix,
not(any(target_os = "ios", target_os = "wasi", target_os = "redox"))
))]
pwd::module_def(ctx),
signal::module_def(ctx),
sre::module_def(ctx),
stat::module_def(ctx),
string::module_def(ctx),
#[cfg(feature = "compiler")]
symtable::module_def(ctx),
sysconfigdata::module_def(ctx),
sysconfig::module_def(ctx),
#[cfg(feature = "threading")]
thread::module_def(ctx),
time::module_def(ctx),
typing::module_def(ctx),
warnings::module_def(ctx),
weakref::module_def(ctx),
#[cfg(windows)]
winapi::module_def(ctx),
#[cfg(windows)]
winreg::module_def(ctx),
]
}

20
crates/vm/src/stdlib/nt.rs
View File

@@ -1,19 +1,12 @@
// spell-checker:disable
use crate::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) use module::module_def;
pub use module::raw_set_handle_inheritable;
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = module::make_module(vm);
super::os::extend_module(vm, &module);
module
}
#[pymodule(name = "nt", with(super::os::_os))]
pub(crate) mod module {
use crate::{
PyResult, TryFromObject, VirtualMachine,
Py, PyResult, TryFromObject, VirtualMachine,
builtins::{PyBaseExceptionRef, PyDictRef, PyListRef, PyStrRef, PyTupleRef},
common::{crt_fd, suppress_iph, windows::ToWideString},
convert::ToPyException,
@@ -1923,4 +1916,13 @@ pub(crate) mod module {
pub(crate) fn support_funcs() -> Vec<SupportFunc> {
Vec::new()
}
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::builtins::PyModule>,
) -> PyResult<()> {
__module_exec(vm, module);
super::super::os::module_exec(vm, module)?;
Ok(())
}
}

2
crates/vm/src/stdlib/operator.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _operator::make_module;
pub(crate) use _operator::module_def;
#[pymodule]
mod _operator {

24
crates/vm/src/stdlib/os.rs
View File

@@ -149,6 +149,8 @@ pub(super) mod _os {
use super::{DirFd, FollowSymlinks, SupportFunc};
#[cfg(windows)]
use crate::common::windows::ToWideString;
#[cfg(any(unix, windows))]
use crate::utils::ToCString;
use crate::{
AsObject, Py, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject,
builtins::{
@@ -168,7 +170,6 @@ pub(super) mod _os {
protocol::PyIterReturn,
recursion::ReprGuard,
types::{IterNext, Iterable, PyStructSequence, Representable, SelfIter},
utils::ToCString,
vm::VirtualMachine,
};
use core::time::Duration;
@@ -906,9 +907,11 @@ pub(super) mod _os {
#[pyarg(any, default)]
#[pystruct_sequence(skip)]
pub st_blocks: i64,
#[cfg(windows)]
#[pyarg(any, default)]
#[pystruct_sequence(skip)]
pub st_reparse_tag: u32,
#[cfg(windows)]
#[pyarg(any, default)]
#[pystruct_sequence(skip)]
pub st_file_attributes: u32,
@@ -943,13 +946,8 @@ pub(super) mod _os {
#[cfg(windows)]
let st_reparse_tag = stat.st_reparse_tag;
#[cfg(not(windows))]
let st_reparse_tag = 0;
#[cfg(windows)]
let st_file_attributes = stat.st_file_attributes;
#[cfg(not(windows))]
let st_file_attributes = 0;
// On Windows, combine st_ino and st_ino_high into a 128-bit value
// like _pystat_l128_from_l64_l64
@@ -986,7 +984,9 @@ pub(super) mod _os {
st_blksize,
#[cfg(not(windows))]
st_blocks,
#[cfg(windows)]
st_reparse_tag,
#[cfg(windows)]
st_file_attributes,
}
}
@@ -1897,21 +1897,21 @@ impl SupportFunc {
}
}
pub fn extend_module(vm: &VirtualMachine, module: &Py<PyModule>) {
pub fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
let support_funcs = _os::support_funcs();
let supports_fd = PySet::default().into_ref(&vm.ctx);
let supports_dir_fd = PySet::default().into_ref(&vm.ctx);
let supports_follow_symlinks = PySet::default().into_ref(&vm.ctx);
for support in support_funcs {
let func_obj = module.get_attr(support.name, vm).unwrap();
let func_obj = module.get_attr(support.name, vm)?;
if support.fd.unwrap_or(false) {
supports_fd.clone().add(func_obj.clone(), vm).unwrap();
supports_fd.clone().add(func_obj.clone(), vm)?;
}
if support.dir_fd.unwrap_or(false) {
supports_dir_fd.clone().add(func_obj.clone(), vm).unwrap();
supports_dir_fd.clone().add(func_obj.clone(), vm)?;
}
if support.follow_symlinks.unwrap_or(false) {
supports_follow_symlinks.clone().add(func_obj, vm).unwrap();
supports_follow_symlinks.clone().add(func_obj, vm)?;
}
}
@@ -1921,6 +1921,8 @@ pub fn extend_module(vm: &VirtualMachine, module: &Py<PyModule>) {
"supports_follow_symlinks" => supports_follow_symlinks,
"error" => vm.ctx.exceptions.os_error.to_owned(),
});
Ok(())
}
#[cfg(not(windows))]

18
crates/vm/src/stdlib/posix.rs
View File

@@ -1,8 +1,9 @@
// spell-checker:disable
use crate::{PyRef, VirtualMachine, builtins::PyModule};
use std::os::fd::BorrowedFd;
pub(crate) use module::module_def;
pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> nix::Result<()> {
use nix::fcntl;
let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?);
@@ -14,12 +15,6 @@ pub fn set_inheritable(fd: BorrowedFd<'_>, inheritable: bool) -> nix::Result<()>
Ok(())
}
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = module::make_module(vm);
super::os::extend_module(vm, &module);
module
}
#[pymodule(name = "posix", with(super::os::_os))]
pub mod module {
use crate::{
@@ -2606,4 +2601,13 @@ pub mod module {
}
Ok(buf)
}
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::builtins::PyModule>,
) -> PyResult<()> {
__module_exec(vm, module);
super::super::os::module_exec(vm, module)?;
Ok(())
}
}

18
crates/vm/src/stdlib/posix_compat.rs
View File

@@ -1,18 +1,13 @@
// spell-checker:disable
//! `posix` compatible module for `not(any(unix, windows))`
use crate::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = module::make_module(vm);
super::os::extend_module(vm, &module);
module
}
pub(crate) use module::module_def;
#[pymodule(name = "posix", with(super::os::_os))]
pub(crate) mod module {
use crate::{
PyObjectRef, PyResult, VirtualMachine,
Py, PyObjectRef, PyResult, VirtualMachine,
builtins::PyStrRef,
convert::IntoPyException,
ospath::OsPath,
@@ -71,4 +66,13 @@ pub(crate) mod module {
pub(crate) fn support_funcs() -> Vec<SupportFunc> {
Vec::new()
}
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::builtins::PyModule>,
) -> PyResult<()> {
__module_exec(vm, module);
super::super::os::module_exec(vm, module)?;
Ok(())
}
}

2
crates/vm/src/stdlib/pwd.rs
View File

@@ -1,6 +1,6 @@
// spell-checker:disable
pub(crate) use pwd::make_module;
pub(crate) use pwd::module_def;
#[pymodule]
mod pwd {

30
crates/vm/src/stdlib/signal.rs
View File

@@ -1,24 +1,12 @@
// spell-checker:disable
use crate::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = _signal::make_module(vm);
#[cfg(any(unix, windows))]
_signal::init_signal_handlers(&module, vm);
module
}
pub(crate) use _signal::module_def;
#[pymodule]
pub(crate) mod _signal {
#[cfg(any(unix, windows))]
use crate::{
Py,
convert::{IntoPyException, TryFromBorrowedObject},
};
use crate::{PyObjectRef, PyResult, VirtualMachine, signal};
use crate::convert::{IntoPyException, TryFromBorrowedObject};
use crate::{Py, PyObjectRef, PyResult, VirtualMachine, signal};
#[cfg(unix)]
use crate::{
builtins::PyTypeRef,
@@ -682,4 +670,16 @@ pub(crate) mod _signal {
// TODO: handle _res < 1, support warn_on_full_buffer
}
}
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::builtins::PyModule>,
) -> PyResult<()> {
__module_exec(vm, module);
#[cfg(any(unix, windows))]
init_signal_handlers(module, vm);
Ok(())
}
}

2
crates/vm/src/stdlib/sre.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _sre::make_module;
pub(crate) use _sre::module_def;
#[pymodule]
mod _sre {

8
crates/vm/src/stdlib/stat.rs
View File

@@ -1,7 +1,7 @@
use crate::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) use _stat::module_def;
#[pymodule]
mod stat {
mod _stat {
// Use libc::mode_t for Mode to match the system's definition
#[cfg(unix)]
type Mode = libc::mode_t;
@@ -522,7 +522,3 @@ mod stat {
result
}
}
pub fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
stat::make_module(vm)
}

2
crates/vm/src/stdlib/string.rs
View File

@@ -1,7 +1,7 @@
/* String builtin module
*/
pub(crate) use _string::make_module;
pub(crate) use _string::module_def;
#[pymodule]
mod _string {

4
crates/vm/src/stdlib/symtable.rs
View File

@@ -1,7 +1,7 @@
pub(crate) use symtable::make_module;
pub(crate) use _symtable::module_def;
#[pymodule]
mod symtable {
mod _symtable {
use crate::{
PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
builtins::{PyDictRef, PyStrRef},

22
crates/vm/src/stdlib/sys.rs
View File

@@ -1,8 +1,6 @@
use crate::{Py, PyResult, VirtualMachine, builtins::PyModule, convert::ToPyObject};
pub(crate) use sys::{
__module_def, DOC, MAXSIZE, RUST_MULTIARCH, UnraisableHookArgsData, multiarch,
};
pub(crate) use sys::{DOC, MAXSIZE, RUST_MULTIARCH, UnraisableHookArgsData, module_def, multiarch};
#[pymodule(name = "_jit")]
mod sys_jit {
@@ -80,7 +78,7 @@ mod sys {
#[pyattr(name = "abiflags")]
const ABIFLAGS_ATTR: &str = "t"; // 't' for free-threaded (no GIL)
// Internal constant used for sysconfigdata_name
pub(crate) const ABIFLAGS: &str = "t";
pub const ABIFLAGS: &str = "t";
#[pyattr(name = "api_version")]
const API_VERSION: u32 = 0x0; // what C api?
#[pyattr(name = "copyright")]
@@ -96,7 +94,7 @@ mod sys {
#[pyattr(name = "maxunicode")]
const MAXUNICODE: u32 = core::char::MAX as u32;
#[pyattr(name = "platform")]
pub(crate) const PLATFORM: &str = {
pub const PLATFORM: &str = {
cfg_if::cfg_if! {
if #[cfg(target_os = "linux")] {
"linux"
@@ -168,9 +166,11 @@ mod sys {
#[pyattr]
fn builtin_module_names(vm: &VirtualMachine) -> PyTupleRef {
let mut module_names: Vec<_> = vm.state.module_inits.keys().cloned().collect();
module_names.push("sys".into());
module_names.push("builtins".into());
let mut module_names: Vec<String> =
vm.state.module_defs.keys().map(|&s| s.to_owned()).collect();
module_names.push("sys".to_owned());
module_names.push("builtins".to_owned());
module_names.sort();
vm.ctx.new_tuple(
module_names
@@ -1609,7 +1609,8 @@ mod sys {
}
pub(crate) fn init_module(vm: &VirtualMachine, module: &Py<PyModule>, builtins: &Py<PyModule>) {
sys::extend_module(vm, module).unwrap();
module.__init_methods(vm).unwrap();
sys::module_exec(vm, module).unwrap();
let modules = vm.ctx.new_dict();
modules
@@ -1620,7 +1621,8 @@ pub(crate) fn init_module(vm: &VirtualMachine, module: &Py<PyModule>, builtins:
.unwrap();
// Create sys._jit submodule
let jit_module = sys_jit::make_module(vm);
let jit_def = sys_jit::module_def(&vm.ctx);
let jit_module = jit_def.create_module(vm).unwrap();
extend_module!(vm, module, {
"__doc__" => sys::DOC.to_owned().to_pyobject(vm),

6
crates/vm/src/stdlib/sysconfig.rs
View File

@@ -1,7 +1,7 @@
pub(crate) use sysconfig::make_module;
pub(crate) use _sysconfig::module_def;
#[pymodule(name = "_sysconfig")]
pub(crate) mod sysconfig {
#[pymodule]
pub(crate) mod _sysconfig {
use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject};
#[pyfunction]

26
crates/vm/src/stdlib/sysconfigdata.rs
View File

@@ -1,11 +1,29 @@
// spell-checker: words LDSHARED ARFLAGS CPPFLAGS CCSHARED BASECFLAGS BLDSHARED
pub(crate) use _sysconfigdata::make_module;
pub(crate) use _sysconfigdata::module_def;
#[pymodule]
pub(crate) mod _sysconfigdata {
use crate::stdlib::sys::{RUST_MULTIARCH, multiarch};
use crate::{VirtualMachine, builtins::PyDictRef, convert::ToPyObject};
mod _sysconfigdata {
use crate::stdlib::sys::{RUST_MULTIARCH, multiarch, sysconfigdata_name};
use crate::{
Py, PyResult, VirtualMachine,
builtins::{PyDictRef, PyModule},
convert::ToPyObject,
};
fn module_exec(vm: &VirtualMachine, module: &Py<PyModule>) -> PyResult<()> {
// Set build_time_vars attribute
let build_time_vars = build_time_vars(vm);
module.set_attr("build_time_vars", build_time_vars, vm)?;
// Ensure the module is registered under the platform-specific name
// (import_builtin() already handles this, but double-check for safety)
let sys_modules = vm.sys_module.get_attr("modules", vm)?;
let sysconfigdata_name = sysconfigdata_name();
sys_modules.set_item(sysconfigdata_name.as_str(), module.to_owned().into(), vm)?;
Ok(())
}
#[pyattr]
fn build_time_vars(vm: &VirtualMachine) -> PyDictRef {

2
crates/vm/src/stdlib/thread.rs
View File

@@ -4,7 +4,7 @@ pub(crate) use _thread::after_fork_child;
#[cfg_attr(target_arch = "wasm32", allow(unused_imports))]
pub(crate) use _thread::{
CurrentFrameSlot, HandleEntry, RawRMutex, ShutdownEntry, get_all_current_frames, get_ident,
init_main_thread_ident, make_module,
init_main_thread_ident, module_def,
};
#[pymodule]

26
crates/vm/src/stdlib/time.rs
View File

@@ -3,18 +3,10 @@
// See also:
// https://docs.python.org/3/library/time.html
use crate::{PyRef, VirtualMachine, builtins::PyModule};
pub use decl::time;
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
#[cfg(not(target_env = "msvc"))]
#[cfg(not(target_arch = "wasm32"))]
unsafe {
c_tzset()
};
decl::make_module(vm)
}
pub(crate) use decl::module_def;
#[cfg(not(target_env = "msvc"))]
#[cfg(not(target_arch = "wasm32"))]
@@ -34,7 +26,7 @@ unsafe extern "C" {
#[pymodule(name = "time", with(platform))]
mod decl {
use crate::{
AsObject, PyObjectRef, PyResult, VirtualMachine,
AsObject, Py, PyObjectRef, PyResult, VirtualMachine,
builtins::{PyStrRef, PyTypeRef, PyUtf8StrRef},
function::{Either, FuncArgs, OptionalArg},
types::{PyStructSequence, struct_sequence_new},
@@ -581,6 +573,20 @@ mod decl {
#[allow(unused_imports)]
use super::platform::*;
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::builtins::PyModule>,
) -> PyResult<()> {
#[cfg(not(target_env = "msvc"))]
#[cfg(not(target_arch = "wasm32"))]
unsafe {
super::c_tzset()
};
__module_exec(vm, module);
Ok(())
}
}
#[cfg(unix)]

54
crates/vm/src/stdlib/typing.rs
View File

@@ -1,10 +1,11 @@
// spell-checker:ignore typevarobject funcobj
use crate::{Context, PyPayload, PyRef, VirtualMachine, class::PyClassImpl, stdlib::PyModule};
use crate::{Context, class::PyClassImpl};
pub use crate::stdlib::typevar::{
Generic, ParamSpec, ParamSpecArgs, ParamSpecKwargs, TypeVar, TypeVarTuple,
set_typeparam_default,
};
pub(crate) use decl::module_def;
pub use decl::*;
/// Initialize typing types (call extend_class)
@@ -12,32 +13,12 @@ pub fn init(ctx: &Context) {
NoDefault::extend_class(ctx, ctx.types.typing_no_default_type);
}
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
let module = decl::make_module(vm);
TypeVar::make_class(&vm.ctx);
ParamSpec::make_class(&vm.ctx);
TypeVarTuple::make_class(&vm.ctx);
ParamSpecArgs::make_class(&vm.ctx);
ParamSpecKwargs::make_class(&vm.ctx);
Generic::make_class(&vm.ctx);
extend_module!(vm, &module, {
"NoDefault" => vm.ctx.typing_no_default.clone(),
"TypeVar" => TypeVar::class(&vm.ctx).to_owned(),
"ParamSpec" => ParamSpec::class(&vm.ctx).to_owned(),
"TypeVarTuple" => TypeVarTuple::class(&vm.ctx).to_owned(),
"ParamSpecArgs" => ParamSpecArgs::class(&vm.ctx).to_owned(),
"ParamSpecKwargs" => ParamSpecKwargs::class(&vm.ctx).to_owned(),
"Generic" => Generic::class(&vm.ctx).to_owned(),
"Union" => vm.ctx.types.union_type.to_owned(),
});
module
}
#[pymodule(name = "_typing")]
pub(crate) mod decl {
use crate::{
Py, PyObjectRef, PyPayload, PyResult, VirtualMachine,
builtins::{PyStrRef, PyTupleRef, PyType, PyTypeRef, pystr::AsPyStr, type_},
class::PyClassImpl,
function::{FuncArgs, IntoFuncArgs},
protocol::PyNumberMethods,
types::{AsNumber, Constructor, Representable},
@@ -206,4 +187,33 @@ pub(crate) mod decl {
// &AS_MAPPING
// }
// }
pub(crate) fn module_exec(
vm: &VirtualMachine,
module: &Py<crate::builtins::PyModule>,
) -> PyResult<()> {
__module_exec(vm, module);
use super::{Generic, ParamSpec, ParamSpecArgs, ParamSpecKwargs, TypeVar, TypeVarTuple};
TypeVar::make_class(&vm.ctx);
ParamSpec::make_class(&vm.ctx);
TypeVarTuple::make_class(&vm.ctx);
ParamSpecArgs::make_class(&vm.ctx);
ParamSpecKwargs::make_class(&vm.ctx);
Generic::make_class(&vm.ctx);
extend_module!(vm, module, {
"NoDefault" => vm.ctx.typing_no_default.clone(),
"TypeVar" => TypeVar::class(&vm.ctx).to_owned(),
"ParamSpec" => ParamSpec::class(&vm.ctx).to_owned(),
"TypeVarTuple" => TypeVarTuple::class(&vm.ctx).to_owned(),
"ParamSpecArgs" => ParamSpecArgs::class(&vm.ctx).to_owned(),
"ParamSpecKwargs" => ParamSpecKwargs::class(&vm.ctx).to_owned(),
"Generic" => Generic::class(&vm.ctx).to_owned(),
"Union" => vm.ctx.types.union_type.to_owned(),
});
Ok(())
}
}

2
crates/vm/src/stdlib/warnings.rs
View File

@@ -1,4 +1,4 @@
pub(crate) use _warnings::make_module;
pub(crate) use _warnings::module_def;
use crate::{Py, PyResult, VirtualMachine, builtins::PyType};

2
crates/vm/src/stdlib/weakref.rs
View File

@@ -4,7 +4,7 @@
//! - [python weakref module](https://docs.python.org/3/library/weakref.html)
//! - [rust weak struct](https://doc.rust-lang.org/std/rc/struct.Weak.html)
//!
pub(crate) use _weakref::make_module;
pub(crate) use _weakref::module_def;
#[pymodule]
mod _weakref {

2
crates/vm/src/stdlib/winapi.rs
View File

@@ -1,7 +1,7 @@
// spell-checker:disable
#![allow(non_snake_case)]
pub(crate) use _winapi::make_module;
pub(crate) use _winapi::module_def;
#[pymodule]
mod _winapi {

6
crates/vm/src/stdlib/winreg.rs
View File

@@ -1,11 +1,7 @@
// spell-checker:disable
#![allow(non_snake_case)]
use crate::{PyRef, VirtualMachine, builtins::PyModule};
pub(crate) fn make_module(vm: &VirtualMachine) -> PyRef<PyModule> {
winreg::make_module(vm)
}
pub(crate) use winreg::module_def;
#[pymodule]
mod winreg {

361
crates/vm/src/vm/interpreter.rs
View File

@@ -1,7 +1,276 @@
use super::{Context, PyConfig, VirtualMachine, setting::Settings, thread};
use crate::{PyResult, getpath, stdlib::atexit, vm::PyBaseExceptionRef};
use super::{Context, PyConfig, PyGlobalState, VirtualMachine, setting::Settings, thread};
use crate::{
PyResult, builtins, common::rc::PyRc, frozen::FrozenModule, getpath, py_freeze, stdlib::atexit,
vm::PyBaseExceptionRef,
};
use core::sync::atomic::Ordering;
type InitFunc = Box<dyn FnOnce(&mut VirtualMachine)>;
/// Configuration builder for constructing an Interpreter.
///
/// This is the preferred way to configure and create an interpreter with custom modules.
/// Modules must be registered before the interpreter is built,
/// similar to CPython's `PyImport_AppendInittab` which must be called before `Py_Initialize`.
///
/// # Example
/// ```
/// use rustpython_vm::Interpreter;
///
/// let builder = Interpreter::builder(Default::default());
/// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx))
/// let interp = builder.build();
/// ```
pub struct InterpreterBuilder {
settings: Settings,
pub ctx: PyRc<Context>,
module_defs: Vec<&'static builtins::PyModuleDef>,
frozen_modules: Vec<(&'static str, FrozenModule)>,
init_hooks: Vec<InitFunc>,
}
/// Private helper to initialize a VM with settings, context, and custom initialization.
fn initialize_main_vm<F>(
settings: Settings,
ctx: PyRc<Context>,
module_defs: Vec<&'static builtins::PyModuleDef>,
frozen_modules: Vec<(&'static str, FrozenModule)>,
init_hooks: Vec<InitFunc>,
init: F,
) -> (VirtualMachine, PyRc<PyGlobalState>)
where
F: FnOnce(&mut VirtualMachine),
{
use crate::codecs::CodecsRegistry;
use crate::common::hash::HashSecret;
use crate::common::lock::PyMutex;
use crate::warn::WarningsState;
use crossbeam_utils::atomic::AtomicCell;
use std::sync::atomic::AtomicBool;
let paths = getpath::init_path_config(&settings);
let config = PyConfig::new(settings, paths);
crate::types::TypeZoo::extend(&ctx);
crate::exceptions::ExceptionZoo::extend(&ctx);
// Build module_defs map from builtin modules + additional modules
let mut all_module_defs: std::collections::BTreeMap<
&'static str,
&'static builtins::PyModuleDef,
> = crate::stdlib::builtin_module_defs(&ctx)
.into_iter()
.chain(module_defs)
.map(|def| (def.name.as_str(), def))
.collect();
// Register sysconfigdata under platform-specific name as well
if let Some(&sysconfigdata_def) = all_module_defs.get("_sysconfigdata") {
use std::sync::OnceLock;
static SYSCONFIGDATA_NAME: OnceLock<&'static str> = OnceLock::new();
let leaked_name = *SYSCONFIGDATA_NAME.get_or_init(|| {
let name = crate::stdlib::sys::sysconfigdata_name();
Box::leak(name.into_boxed_str())
});
all_module_defs.insert(leaked_name, sysconfigdata_def);
}
// Create hash secret
let seed = match config.settings.hash_seed {
Some(seed) => seed,
None => super::process_hash_secret_seed(),
};
let hash_secret = HashSecret::new(seed);
// Create codec registry and warnings state
let codec_registry = CodecsRegistry::new(&ctx);
let warnings = WarningsState::init_state(&ctx);
// Create int_max_str_digits
let int_max_str_digits = AtomicCell::new(match config.settings.int_max_str_digits {
-1 => 4300,
other => other,
} as usize);
// Initialize frozen modules (core + user-provided)
let mut frozen: std::collections::HashMap<&'static str, FrozenModule, ahash::RandomState> =
core_frozen_inits().collect();
frozen.extend(frozen_modules);
// Create PyGlobalState
let global_state = PyRc::new(PyGlobalState {
config,
module_defs: all_module_defs,
frozen,
stacksize: AtomicCell::new(0),
thread_count: AtomicCell::new(0),
hash_secret,
atexit_funcs: PyMutex::default(),
codec_registry,
finalizing: AtomicBool::new(false),
warnings,
override_frozen_modules: AtomicCell::new(0),
before_forkers: PyMutex::default(),
after_forkers_child: PyMutex::default(),
after_forkers_parent: PyMutex::default(),
int_max_str_digits,
switch_interval: AtomicCell::new(0.005),
global_trace_func: PyMutex::default(),
global_profile_func: PyMutex::default(),
#[cfg(feature = "threading")]
main_thread_ident: AtomicCell::new(0),
#[cfg(feature = "threading")]
thread_frames: parking_lot::Mutex::new(std::collections::HashMap::new()),
#[cfg(feature = "threading")]
thread_handles: parking_lot::Mutex::new(Vec::new()),
#[cfg(feature = "threading")]
shutdown_handles: parking_lot::Mutex::new(Vec::new()),
});
// Create VM with the global state
// Note: Don't clone here - init_hooks need exclusive access to mutate state
let mut vm = VirtualMachine::new(ctx, global_state);
// Execute initialization hooks (can mutate vm.state)
for hook in init_hooks {
hook(&mut vm);
}
// Call custom init function (can mutate vm.state)
init(&mut vm);
vm.initialize();
// Clone global_state for Interpreter after all initialization is done
let global_state = vm.state.clone();
(vm, global_state)
}
impl InterpreterBuilder {
/// Create a new interpreter configuration with default settings.
pub fn new() -> Self {
Self {
settings: Settings::default(),
ctx: Context::genesis().clone(),
module_defs: Vec::new(),
frozen_modules: Vec::new(),
init_hooks: Vec::new(),
}
}
/// Set custom settings for the interpreter.
///
/// If called multiple times, only the last settings will be used.
pub fn settings(mut self, settings: Settings) -> Self {
self.settings = settings;
self
}
/// Add a single native module definition.
///
/// # Example
/// ```
/// use rustpython_vm::{Interpreter, builtins::PyModuleDef};
///
/// let builder = Interpreter::builder(Default::default());
/// // Note: In practice, use module_def from your #[pymodule]
/// // let def = mymodule::module_def(&builder.ctx);
/// // let interp = builder.add_native_module(def).build();
/// let interp = builder.build();
/// ```
pub fn add_native_module(self, def: &'static builtins::PyModuleDef) -> Self {
self.add_native_modules(&[def])
}
/// Add multiple native module definitions.
///
/// # Example
/// ```
/// use rustpython_vm::Interpreter;
///
/// let builder = Interpreter::builder(Default::default());
/// // In practice, use module_defs from rustpython_stdlib:
/// // let defs = rustpython_stdlib::stdlib_module_defs(&builder.ctx);
/// // let interp = builder.add_native_modules(&defs).build();
/// let interp = builder.build();
/// ```
pub fn add_native_modules(mut self, defs: &[&'static builtins::PyModuleDef]) -> Self {
self.module_defs.extend_from_slice(defs);
self
}
/// Add a custom initialization hook.
///
/// Hooks are executed in the order they are added during interpreter creation.
/// This function will be called after modules are registered but before
/// the VM is initialized, allowing for additional customization.
///
/// # Example
/// ```
/// use rustpython_vm::Interpreter;
///
/// let interp = Interpreter::builder(Default::default())
/// .init_hook(|vm| {
/// // Custom initialization
/// })
/// .build();
/// ```
pub fn init_hook<F>(mut self, init: F) -> Self
where
F: FnOnce(&mut VirtualMachine) + 'static,
{
self.init_hooks.push(Box::new(init));
self
}
/// Add frozen modules to the interpreter.
///
/// Frozen modules are Python modules compiled into the binary.
/// This method accepts any iterator of (name, FrozenModule) pairs.
///
/// # Example
/// ```
/// use rustpython_vm::Interpreter;
///
/// let interp = Interpreter::builder(Default::default())
/// // In practice: .add_frozen_modules(rustpython_pylib::FROZEN_STDLIB)
/// .build();
/// ```
pub fn add_frozen_modules<I>(mut self, frozen: I) -> Self
where
I: IntoIterator<Item = (&'static str, FrozenModule)>,
{
self.frozen_modules.extend(frozen);
self
}
/// Build the interpreter.
///
/// This consumes the configuration and returns a fully initialized Interpreter.
pub fn build(self) -> Interpreter {
let (vm, global_state) = initialize_main_vm(
self.settings,
self.ctx,
self.module_defs,
self.frozen_modules,
self.init_hooks,
|_| {}, // No additional init needed
);
Interpreter { global_state, vm }
}
/// Alias for `build()` for compatibility with the `interpreter()` pattern.
pub fn interpreter(self) -> Interpreter {
self.build()
}
}
impl Default for InterpreterBuilder {
fn default() -> Self {
Self::new()
}
}
/// The general interface for the VM
///
/// # Examples
@@ -21,43 +290,49 @@ use core::sync::atomic::Ordering;
/// });
/// ```
pub struct Interpreter {
pub global_state: PyRc<PyGlobalState>,
vm: VirtualMachine,
}
impl Interpreter {
/// Create a new interpreter configuration builder.
///
/// # Example
/// ```
/// use rustpython_vm::Interpreter;
///
/// let builder = Interpreter::builder(Default::default());
/// // In practice, add stdlib: builder.add_native_modules(&stdlib_module_defs(&builder.ctx))
/// let interp = builder.build();
/// ```
pub fn builder(settings: Settings) -> InterpreterBuilder {
InterpreterBuilder::new().settings(settings)
}
/// This is a bare unit to build up an interpreter without the standard library.
/// To create an interpreter with the standard library with the `rustpython` crate, use `rustpython::InterpreterConfig`.
/// To create an interpreter with the standard library with the `rustpython` crate, use `rustpython::InterpreterBuilder`.
/// To create an interpreter without the `rustpython` crate, but only with `rustpython-vm`,
/// try to build one from the source code of `InterpreterConfig`. It will not be a one-liner but it also will not be too hard.
/// try to build one from the source code of `InterpreterBuilder`. It will not be a one-liner but it also will not be too hard.
pub fn without_stdlib(settings: Settings) -> Self {
Self::with_init(settings, |_| {})
}
/// Create with initialize function taking mutable vm reference.
/// ```
/// use rustpython_vm::Interpreter;
/// Interpreter::with_init(Default::default(), |vm| {
/// // put this line to add stdlib to the vm
/// // vm.add_native_modules(rustpython_stdlib::get_module_inits());
/// }).enter(|vm| {
/// vm.run_code_string(vm.new_scope_with_builtins(), "print(1)", "<...>".to_owned());
/// });
/// ```
///
/// Note: This is a legacy API. To add stdlib, use `Interpreter::builder()` instead.
pub fn with_init<F>(settings: Settings, init: F) -> Self
where
F: FnOnce(&mut VirtualMachine),
{
// Compute path configuration from settings
let paths = getpath::init_path_config(&settings);
let config = PyConfig::new(settings, paths);
let ctx = Context::genesis();
crate::types::TypeZoo::extend(ctx);
crate::exceptions::ExceptionZoo::extend(ctx);
let mut vm = VirtualMachine::new(config, ctx.clone());
init(&mut vm);
vm.initialize();
Self { vm }
let (vm, global_state) = initialize_main_vm(
settings,
Context::genesis().clone(),
Vec::new(), // No module_defs
Vec::new(), // No frozen_modules
Vec::new(), // No init_hooks
init,
);
Self { global_state, vm }
}
/// Run a function with the main virtual machine and return a PyResult of the result.
@@ -157,6 +432,44 @@ impl Interpreter {
}
}
fn core_frozen_inits() -> impl Iterator<Item = (&'static str, FrozenModule)> {
let iter = core::iter::empty();
macro_rules! ext_modules {
($iter:ident, $($t:tt)*) => {
let $iter = $iter.chain(py_freeze!($($t)*));
};
}
// keep as example but use file one now
// ext_modules!(
// iter,
// source = "initialized = True; print(\"Hello world!\")\n",
// module_name = "__hello__",
// );
// Python modules that the vm calls into, but are not actually part of the stdlib. They could
// in theory be implemented in Rust, but are easiest to do in Python for one reason or another.
// Includes _importlib_bootstrap and _importlib_bootstrap_external
ext_modules!(
iter,
dir = "./Lib/python_builtins",
crate_name = "rustpython_compiler_core"
);
// core stdlib Python modules that the vm calls into, but are still used in Python
// application code, e.g. copyreg
// FIXME: Initializing core_modules here results duplicated frozen module generation for core_modules.
// We need a way to initialize this modules for both `Interpreter::without_stdlib()` and `InterpreterBuilder::new().init_stdlib().interpreter()`
// #[cfg(not(feature = "freeze-stdlib"))]
ext_modules!(
iter,
dir = "./Lib/core_modules",
crate_name = "rustpython_compiler_core"
);
iter
}
#[cfg(test)]
mod tests {
use super::*;

197
crates/vm/src/vm/mod.rs
View File

@@ -19,7 +19,7 @@ mod vm_ops;
use crate::{
AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult,
builtins::{
PyBaseExceptionRef, PyDict, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned,
self, PyBaseExceptionRef, PyDict, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned,
PyStrRef, PyTypeRef,
code::PyCode,
dict::{PyDictItems, PyDictKeys, PyDictValues},
@@ -56,7 +56,7 @@ use std::{
};
pub use context::Context;
pub use interpreter::Interpreter;
pub use interpreter::{Interpreter, InterpreterBuilder};
pub(crate) use method::PyMethod;
pub use setting::{CheckHashPycsMode, Paths, PyConfig, Settings};
@@ -102,7 +102,7 @@ struct ExceptionStack {
pub struct PyGlobalState {
pub config: PyConfig,
pub module_inits: stdlib::StdlibMap,
pub module_defs: std::collections::BTreeMap<&'static str, &'static builtins::PyModuleDef>,
pub frozen: HashMap<&'static str, FrozenModule, ahash::RandomState>,
pub stacksize: AtomicCell<usize>,
pub thread_count: AtomicCell<usize>,
@@ -144,7 +144,7 @@ pub fn process_hash_secret_seed() -> u32 {
impl VirtualMachine {
/// Create a new `VirtualMachine` structure.
fn new(config: PyConfig, ctx: PyRc<Context>) -> Self {
pub(crate) fn new(ctx: PyRc<Context>, state: PyRc<PyGlobalState>) -> Self {
flame_guard!("new VirtualMachine");
// make a new module without access to the vm; doesn't
@@ -158,8 +158,8 @@ impl VirtualMachine {
};
// Hard-core modules:
let builtins = new_module(stdlib::builtins::__module_def(&ctx));
let sys_module = new_module(stdlib::sys::__module_def(&ctx));
let builtins = new_module(stdlib::builtins::module_def(&ctx));
let sys_module = new_module(stdlib::sys::module_def(&ctx));
let import_func = ctx.none();
let profile_func = RefCell::new(ctx.none());
@@ -169,23 +169,7 @@ impl VirtualMachine {
const { RefCell::new([const { None }; signal::NSIG]) },
));
let module_inits = stdlib::get_module_inits();
let seed = match config.settings.hash_seed {
Some(seed) => seed,
None => process_hash_secret_seed(),
};
let hash_secret = HashSecret::new(seed);
let codec_registry = CodecsRegistry::new(&ctx);
let warnings = WarningsState::init_state(&ctx);
let int_max_str_digits = AtomicCell::new(match config.settings.int_max_str_digits {
-1 => 4300,
other => other,
} as usize);
let mut vm = Self {
let vm = Self {
builtins,
sys_module,
ctx,
@@ -200,34 +184,7 @@ impl VirtualMachine {
signal_handlers,
signal_rx: None,
repr_guards: RefCell::default(),
state: PyRc::new(PyGlobalState {
config,
module_inits,
frozen: HashMap::default(),
stacksize: AtomicCell::new(0),
thread_count: AtomicCell::new(0),
hash_secret,
atexit_funcs: PyMutex::default(),
codec_registry,
finalizing: AtomicBool::new(false),
warnings,
override_frozen_modules: AtomicCell::new(0),
before_forkers: PyMutex::default(),
after_forkers_child: PyMutex::default(),
after_forkers_parent: PyMutex::default(),
int_max_str_digits,
switch_interval: AtomicCell::new(0.005),
global_trace_func: PyMutex::default(),
global_profile_func: PyMutex::default(),
#[cfg(feature = "threading")]
main_thread_ident: AtomicCell::new(0),
#[cfg(feature = "threading")]
thread_frames: parking_lot::Mutex::new(HashMap::new()),
#[cfg(feature = "threading")]
thread_handles: parking_lot::Mutex::new(Vec::new()),
#[cfg(feature = "threading")]
shutdown_handles: parking_lot::Mutex::new(Vec::new()),
}),
state,
initialized: false,
recursion_depth: Cell::new(0),
c_stack_soft_limit: Cell::new(Self::calculate_c_stack_soft_limit()),
@@ -245,9 +202,6 @@ impl VirtualMachine {
panic!("Interpreters in same process must share the hash seed");
}
let frozen = core_frozen_inits().collect();
PyRc::get_mut(&mut vm.state).unwrap().frozen = frozen;
vm.builtins.init_dict(
vm.ctx.intern_str("builtins"),
Some(vm.ctx.intern_str(stdlib::builtins::DOC.unwrap()).to_owned()),
@@ -275,7 +229,7 @@ impl VirtualMachine {
});
let guide_message = if cfg!(feature = "freeze-stdlib") {
"`rustpython_pylib` maybe not set while using `freeze-stdlib` feature. Try using `rustpython::InterpreterConfig::init_stdlib` or manually call `vm.add_frozen(rustpython_pylib::FROZEN_STDLIB)` in `rustpython_vm::Interpreter::with_init`."
"`rustpython_pylib` may not be set while using `freeze-stdlib` feature. Try using `rustpython::InterpreterBuilder::init_stdlib` or manually call `builder.add_frozen_modules(rustpython_pylib::FROZEN_STDLIB)` in `rustpython_vm::Interpreter::builder()`."
} else if !env_set {
"Neither RUSTPYTHONPATH nor PYTHONPATH is set. Try setting one of them to the stdlib directory."
} else if path_contains_env {
@@ -470,34 +424,6 @@ impl VirtualMachine {
self.initialized = true;
}
fn state_mut(&mut self) -> &mut PyGlobalState {
PyRc::get_mut(&mut self.state)
.expect("there should not be multiple threads while a user has a mut ref to a vm")
}
/// Can only be used in the initialization closure passed to [`Interpreter::with_init`]
pub fn add_native_module<S>(&mut self, name: S, module: stdlib::StdlibInitFunc)
where
S: Into<Cow<'static, str>>,
{
self.state_mut().module_inits.insert(name.into(), module);
}
pub fn add_native_modules<I>(&mut self, iter: I)
where
I: IntoIterator<Item = (Cow<'static, str>, stdlib::StdlibInitFunc)>,
{
self.state_mut().module_inits.extend(iter);
}
/// Can only be used in the initialization closure passed to [`Interpreter::with_init`]
pub fn add_frozen<I>(&mut self, frozen: I)
where
I: IntoIterator<Item = (&'static str, FrozenModule)>,
{
self.state_mut().frozen.extend(frozen);
}
/// Set the custom signal channel for the interpreter
pub fn set_user_signal_channel(&mut self, signal_rx: signal::UserSignalReceiver) {
self.signal_rx = Some(signal_rx);
@@ -1381,89 +1307,52 @@ pub fn resolve_frozen_alias(name: &str) -> &str {
}
}
fn core_frozen_inits() -> impl Iterator<Item = (&'static str, FrozenModule)> {
let iter = core::iter::empty();
macro_rules! ext_modules {
($iter:ident, $($t:tt)*) => {
let $iter = $iter.chain(py_freeze!($($t)*));
};
}
// keep as example but use file one now
// ext_modules!(
// iter,
// source = "initialized = True; print(\"Hello world!\")\n",
// module_name = "__hello__",
// );
// Python modules that the vm calls into, but are not actually part of the stdlib. They could
// in theory be implemented in Rust, but are easiest to do in Python for one reason or another.
// Includes _importlib_bootstrap and _importlib_bootstrap_external
ext_modules!(
iter,
dir = "./Lib/python_builtins",
crate_name = "rustpython_compiler_core"
);
// core stdlib Python modules that the vm calls into, but are still used in Python
// application code, e.g. copyreg
// FIXME: Initializing core_modules here results duplicated frozen module generation for core_modules.
// We need a way to initialize this modules for both `Interpreter::without_stdlib()` and `InterpreterConfig::new().init_stdlib().interpreter()`
// #[cfg(not(feature = "freeze-stdlib"))]
ext_modules!(
iter,
dir = "./Lib/core_modules",
crate_name = "rustpython_compiler_core"
);
iter
}
#[test]
fn test_nested_frozen() {
use rustpython_vm as vm;
vm::Interpreter::with_init(Default::default(), |vm| {
// vm.add_native_modules(rustpython_stdlib::get_module_inits());
vm.add_frozen(rustpython_vm::py_freeze!(
vm::Interpreter::builder(Default::default())
.add_frozen_modules(rustpython_vm::py_freeze!(
dir = "../../extra_tests/snippets"
));
})
.enter(|vm| {
let scope = vm.new_scope_with_builtins();
))
.build()
.enter(|vm| {
let scope = vm.new_scope_with_builtins();
let source = "from dir_module.dir_module_inner import value2";
let code_obj = vm
.compile(source, vm::compiler::Mode::Exec, "<embedded>".to_owned())
.map_err(|err| vm.new_syntax_error(&err, Some(source)))
.unwrap();
let source = "from dir_module.dir_module_inner import value2";
let code_obj = vm
.compile(source, vm::compiler::Mode::Exec, "<embedded>".to_owned())
.map_err(|err| vm.new_syntax_error(&err, Some(source)))
.unwrap();
if let Err(e) = vm.run_code_obj(code_obj, scope) {
vm.print_exception(e);
panic!();
}
})
if let Err(e) = vm.run_code_obj(code_obj, scope) {
vm.print_exception(e);
panic!();
}
})
}
#[test]
fn frozen_origname_matches() {
use rustpython_vm as vm;
vm::Interpreter::with_init(Default::default(), |_vm| {}).enter(|vm| {
let check = |name, expected| {
let module = import::import_frozen(vm, name).unwrap();
let origname: PyStrRef = module
.get_attr("__origname__", vm)
.unwrap()
.try_into_value(vm)
.unwrap();
assert_eq!(origname.as_str(), expected);
};
vm::Interpreter::builder(Default::default())
.build()
.enter(|vm| {
let check = |name, expected| {
let module = import::import_frozen(vm, name).unwrap();
let origname: PyStrRef = module
.get_attr("__origname__", vm)
.unwrap()
.try_into_value(vm)
.unwrap();
assert_eq!(origname.as_str(), expected);
};
check("_frozen_importlib", "importlib._bootstrap");
check(
"_frozen_importlib_external",
"importlib._bootstrap_external",
);
});
check("_frozen_importlib", "importlib._bootstrap");
check(
"_frozen_importlib_external",
"importlib._bootstrap_external",
);
});
}

9
crates/wasm/src/browser_module.rs
View File

@@ -1,6 +1,4 @@
use rustpython_vm::VirtualMachine;
pub(crate) use _browser::make_module;
pub(crate) use _browser::module_def;
#[pymodule]
mod _browser {
@@ -257,8 +255,3 @@ mod _browser {
Ok(PyPromise::from_future(future).into_pyobject(vm))
}
}
pub fn setup_browser_module(vm: &mut VirtualMachine) {
vm.add_native_module("_browser".to_owned(), Box::new(make_module));
vm.add_frozen(py_freeze!(dir = "Lib"));
}

7
crates/wasm/src/js_module.rs
View File

@@ -1,5 +1,4 @@
pub(crate) use _js::{PyJsValue, PyPromise};
use rustpython_vm::VirtualMachine;
#[pymodule]
mod _js {
@@ -621,8 +620,4 @@ mod _js {
}
}
pub(crate) use _js::make_module;
pub fn setup_js_module(vm: &mut VirtualMachine) {
vm.add_native_module("_js".to_owned(), Box::new(make_module));
}
pub(crate) use _js::module_def;

1
crates/wasm/src/lib.rs
View File

@@ -11,7 +11,6 @@ extern crate rustpython_vm;
use js_sys::{Reflect, WebAssembly::RuntimeError};
use std::panic;
pub use vm_class::add_init_func;
pub(crate) use vm_class::weak_vm;
use wasm_bindgen::prelude::*;

Some files were not shown because too many files have changed in this diff Show More