diff --git a/benchmarks/bench.rs b/benchmarks/bench.rs index 84c91be282..7f27f0a1e4 100644 --- a/benchmarks/bench.rs +++ b/benchmarks/bench.rs @@ -94,15 +94,14 @@ fn bench_rustpy_nbody(b: &mut test::Bencher) { let vm = VirtualMachine::default(); - let code = match vm.compile(source, compile::Mode::Single, "".to_owned()) { - Ok(code) => code, - Err(e) => panic!("{:?}", e), - }; + let code = vm + .compile(source, compile::Mode::Exec, "".to_owned()) + .unwrap(); b.iter(|| { let scope = vm.new_scope_with_builtins(); let res: PyResult = vm.run_code_obj(code.clone(), scope); - assert!(res.is_ok()); + vm.unwrap_pyresult(res); }) } @@ -113,14 +112,13 @@ fn bench_rustpy_mandelbrot(b: &mut test::Bencher) { let vm = VirtualMachine::default(); - let code = match vm.compile(source, compile::Mode::Single, "".to_owned()) { - Ok(code) => code, - Err(e) => panic!("{:?}", e), - }; + let code = vm + .compile(source, compile::Mode::Exec, "".to_owned()) + .unwrap(); b.iter(|| { let scope = vm.new_scope_with_builtins(); let res: PyResult = vm.run_code_obj(code.clone(), scope); - assert!(res.is_ok()); + vm.unwrap_pyresult(res); }) } diff --git a/bytecode/src/bytecode.rs b/bytecode/src/bytecode.rs index 18ae446d90..34e916cc2c 100644 --- a/bytecode/src/bytecode.rs +++ b/bytecode/src/bytecode.rs @@ -50,7 +50,7 @@ pub struct CodeObject { bitflags! { #[derive(Serialize, Deserialize)] - pub struct CodeFlags: u8 { + pub struct CodeFlags: u16 { const HAS_DEFAULTS = 0x01; const HAS_KW_ONLY_DEFAULTS = 0x02; const HAS_ANNOTATIONS = 0x04; diff --git a/compiler/src/compile.rs b/compiler/src/compile.rs index 8f88965b77..413c9c4369 100644 --- a/compiler/src/compile.rs +++ b/compiler/src/compile.rs @@ -29,8 +29,21 @@ struct Compiler { source_path: Option, current_source_location: ast::Location, current_qualified_path: Option, + done_with_future_stmts: bool, ctx: CompileContext, - optimize: u8, + opts: CompileOpts, +} + +#[derive(Debug, Clone)] +pub struct CompileOpts { + /// How optimized the bytecode output should be; any optimize > 0 does + /// not emit assert statements + pub optimize: u8, +} +impl Default for CompileOpts { + fn default() -> Self { + CompileOpts { optimize: 0 } + } } #[derive(Clone, Copy)] @@ -60,20 +73,20 @@ pub fn compile( source: &str, mode: Mode, source_path: String, - optimize: u8, + opts: CompileOpts, ) -> CompileResult { match mode { Mode::Exec => { let ast = parser::parse_program(source)?; - compile_program(ast, source_path, optimize) + compile_program(ast, source_path, opts) } Mode::Eval => { let statement = parser::parse_statement(source)?; - compile_statement_eval(statement, source_path, optimize) + compile_statement_eval(statement, source_path, opts) } Mode::Single => { let ast = parser::parse_program(source)?; - compile_program_single(ast, source_path, optimize) + compile_program_single(ast, source_path, opts) } } } @@ -81,10 +94,10 @@ pub fn compile( /// A helper function for the shared code of the different compile functions fn with_compiler( source_path: String, - optimize: u8, + opts: CompileOpts, f: impl FnOnce(&mut Compiler) -> CompileResult<()>, ) -> CompileResult { - let mut compiler = Compiler::new(optimize); + let mut compiler = Compiler::new(opts); compiler.source_path = Some(source_path); compiler.push_new_code_object("".to_owned()); f(&mut compiler)?; @@ -97,9 +110,9 @@ fn with_compiler( pub fn compile_program( ast: ast::Program, source_path: String, - optimize: u8, + opts: CompileOpts, ) -> CompileResult { - with_compiler(source_path, optimize, |compiler| { + with_compiler(source_path, opts, |compiler| { let symbol_table = make_symbol_table(&ast)?; compiler.compile_program(&ast, symbol_table) }) @@ -109,9 +122,9 @@ pub fn compile_program( pub fn compile_statement_eval( statement: Vec, source_path: String, - optimize: u8, + opts: CompileOpts, ) -> CompileResult { - with_compiler(source_path, optimize, |compiler| { + with_compiler(source_path, opts, |compiler| { let symbol_table = statements_to_symbol_table(&statement)?; compiler.compile_statement_eval(&statement, symbol_table) }) @@ -121,9 +134,9 @@ pub fn compile_statement_eval( pub fn compile_program_single( ast: ast::Program, source_path: String, - optimize: u8, + opts: CompileOpts, ) -> CompileResult { - with_compiler(source_path, optimize, |compiler| { + with_compiler(source_path, opts, |compiler| { let symbol_table = make_symbol_table(&ast)?; compiler.compile_program_single(&ast, symbol_table) }) @@ -134,12 +147,12 @@ where O: OutputStream, { fn default() -> Self { - Compiler::new(0) + Compiler::new(CompileOpts::default()) } } impl Compiler { - fn new(optimize: u8) -> Self { + fn new(opts: CompileOpts) -> Self { Compiler { output_stack: Vec::new(), symbol_table_stack: Vec::new(), @@ -147,11 +160,12 @@ impl Compiler { source_path: None, current_source_location: ast::Location::default(), current_qualified_path: None, + done_with_future_stmts: false, ctx: CompileContext { in_loop: false, func: FunctionContext::NoFunction, }, - optimize, + opts, } } @@ -314,6 +328,16 @@ impl Compiler { self.set_source_location(statement.location); use ast::StatementType::*; + match &statement.node { + // we do this here because `from __future__` still executes that `from` statement at runtime, + // we still need to compile the ImportFrom down below + ImportFrom { module, names, .. } if module.as_deref() == Some("__future__") => { + self.compile_future_features(&names)? + } + // if we find any other statement, stop accepting future statements + _ => self.done_with_future_stmts = true, + } + match &statement.node { Import { names } => { // import a, b, c as d @@ -518,7 +542,7 @@ impl Compiler { } => self.compile_class_def(name, body, bases, keywords, decorator_list)?, Assert { test, msg } => { // if some flag, ignore all assert statements! - if self.optimize == 0 { + if self.opts.optimize == 0 { let end_label = self.new_label(); self.compile_jump_if(test, true, end_label)?; self.emit(Instruction::LoadName { @@ -2112,6 +2136,28 @@ impl Compiler { Ok(()) } + fn compile_future_features( + &mut self, + features: &[ast::ImportSymbol], + ) -> Result<(), CompileError> { + if self.done_with_future_stmts { + return Err(self.error(CompileErrorType::InvalidFuturePlacement)); + } + for feature in features { + match &*feature.symbol { + // Python 3 features; we've already implemented them by default + "nested_scopes" | "generators" | "division" | "absolute_import" + | "with_statement" | "print_function" | "unicode_literals" => {} + // "generator_stop" => {} + // "annotations" => {} + other => { + return Err(self.error(CompileErrorType::InvalidFutureFeature(other.to_owned()))) + } + } + } + Ok(()) + } + // Scope helpers: fn enter_scope(&mut self) { // println!("Enter scope {:?}", self.symbol_table_stack); diff --git a/compiler/src/error.rs b/compiler/src/error.rs index 6f28e3a33a..85bc9f4625 100644 --- a/compiler/src/error.rs +++ b/compiler/src/error.rs @@ -58,6 +58,8 @@ pub enum CompileErrorType { InvalidAwait, AsyncYieldFrom, AsyncReturnValue, + InvalidFuturePlacement, + InvalidFutureFeature(String), } impl CompileError { @@ -106,6 +108,12 @@ impl fmt::Display for CompileError { CompileErrorType::AsyncReturnValue => { "'return' with value inside async generator".to_owned() } + CompileErrorType::InvalidFuturePlacement => { + "from __future__ imports must occur at the beginning of the file".to_owned() + } + CompileErrorType::InvalidFutureFeature(feat) => { + format!("future feature {} is not defined", feat) + } }; if let Some(statement) = &self.statement { diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index 1206469fa4..e3ae7eb38f 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -49,7 +49,7 @@ impl CompilationSource { module_name: String, origin: F, ) -> Result { - compile::compile(source, mode, module_name, 0).map_err(|err| { + compile::compile(source, mode, module_name, Default::default()).map_err(|err| { Diagnostic::spans_error( self.span, format!("Python compile error from {}: {}", origin(), err), diff --git a/examples/dis.rs b/examples/dis.rs index 49fda57601..bd18537e39 100644 --- a/examples/dis.rs +++ b/examples/dis.rs @@ -62,9 +62,14 @@ fn main() { let optimize = matches.occurrences_of("optimize") as u8; let scripts = matches.values_of_os("scripts").unwrap(); + let opts = compile::CompileOpts { + optimize, + ..Default::default() + }; + for script in scripts.map(Path::new) { if script.exists() && script.is_file() { - let res = display_script(script, mode, optimize, expand_codeobjects); + let res = display_script(script, mode, opts.clone(), expand_codeobjects); if let Err(e) = res { error!("Error while compiling {:?}: {}", script, e); } @@ -77,11 +82,11 @@ fn main() { fn display_script( path: &Path, mode: compile::Mode, - optimize: u8, + opts: compile::CompileOpts, expand_codeobjects: bool, ) -> Result<(), Box> { let source = fs::read_to_string(path)?; - let code = compile::compile(&source, mode, path.to_string_lossy().into_owned(), optimize)?; + let code = compile::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?; println!("{}:", path.display()); if expand_codeobjects { println!("{}", code.display_expand_codeobjects()); diff --git a/tests/snippets/function.py b/tests/snippets/function.py index bb9b453292..ebea34fe58 100644 --- a/tests/snippets/function.py +++ b/tests/snippets/function.py @@ -13,6 +13,7 @@ assert foo.__doc__ == "test" assert foo.__name__ == "foo" assert foo.__qualname__ == "foo" assert foo.__module__ == "function" +assert foo.__globals__ is globals() def my_func(a,): return a+2 diff --git a/tests/snippets/invalid_syntax.py b/tests/snippets/invalid_syntax.py index 33428e94bf..37a3805a29 100644 --- a/tests/snippets/invalid_syntax.py +++ b/tests/snippets/invalid_syntax.py @@ -7,12 +7,9 @@ def valid_func(): yield 2 """ -try: +with assert_raises(SyntaxError) as ae: compile(src, 'test.py', 'exec') -except SyntaxError as ex: - assert ex.lineno == 5 -else: - raise AssertionError("Must throw syntax error") +assert ae.exception.lineno == 5 src = """ if True: @@ -60,3 +57,23 @@ src = """ with assert_raises(SyntaxError): compile(src, 'test.py', 'exec') + +src = """ +from __future__ import not_a_real_future_feature +""" + +with assert_raises(SyntaxError): + compile(src, 'test.py', 'exec') + +src = """ +a = 1 +from __future__ import print_function +""" + +with assert_raises(SyntaxError): + compile(src, 'test.py', 'exec') + +src = """ +from __future__ import print_function +""" +compile(src, 'test.py', 'exec') diff --git a/vm/src/import.rs b/vm/src/import.rs index bcfe4ba228..362af5825e 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -74,13 +74,8 @@ pub fn import_file( file_path: String, content: String, ) -> PyResult { - let code_obj = compile::compile( - &content, - compile::Mode::Exec, - file_path, - vm.settings.optimize, - ) - .map_err(|err| vm.new_syntax_error(&err))?; + let code_obj = compile::compile(&content, compile::Mode::Exec, file_path, vm.compile_opts()) + .map_err(|err| vm.new_syntax_error(&err))?; import_codeobj(vm, module_name, code_obj, true) } diff --git a/vm/src/obj/objcode.rs b/vm/src/obj/objcode.rs index 16410e9be7..71053bdb92 100644 --- a/vm/src/obj/objcode.rs +++ b/vm/src/obj/objcode.rs @@ -103,7 +103,7 @@ impl PyCodeRef { } #[pyproperty] - fn co_flags(self) -> u8 { + fn co_flags(self) -> u16 { self.code.flags.bits() } } diff --git a/vm/src/obj/objfunction.rs b/vm/src/obj/objfunction.rs index c363dd5281..a707d4d7a7 100644 --- a/vm/src/obj/objfunction.rs +++ b/vm/src/obj/objfunction.rs @@ -278,6 +278,11 @@ impl PyFunction { fn kwdefaults(&self) -> Option { self.kw_only_defaults.clone() } + + #[pyproperty(magic)] + fn globals(&self) -> PyDictRef { + self.scope.globals.clone() + } } #[pyclass] diff --git a/vm/src/vm.rs b/vm/src/vm.rs index cc13bc92e2..777c5f7e40 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -17,7 +17,10 @@ use num_bigint::BigInt; use num_traits::ToPrimitive; use once_cell::sync::Lazy; #[cfg(feature = "rustpython-compiler")] -use rustpython_compiler::{compile, error::CompileError}; +use rustpython_compiler::{ + compile::{self, CompileOpts}, + error::CompileError, +}; use crate::builtins::{self, to_ascii}; use crate::bytecode; @@ -1000,6 +1003,15 @@ impl VirtualMachine { } } + /// Returns a basic CompileOpts instance with options accurate to the vm. Used + /// as the CompileOpts for `vm.compile()`. + #[cfg(feature = "rustpython-compiler")] + pub fn compile_opts(&self) -> CompileOpts { + CompileOpts { + optimize: self.settings.optimize, + } + } + #[cfg(feature = "rustpython-compiler")] pub fn compile( &self, @@ -1007,7 +1019,18 @@ impl VirtualMachine { mode: compile::Mode, source_path: String, ) -> Result { - compile::compile(source, mode, source_path, self.settings.optimize) + self.compile_with_opts(source, mode, source_path, self.compile_opts()) + } + + #[cfg(feature = "rustpython-compiler")] + pub fn compile_with_opts( + &self, + source: &str, + mode: compile::Mode, + source_path: String, + opts: CompileOpts, + ) -> Result { + compile::compile(source, mode, source_path, opts) .map(|codeobj| PyCode::new(codeobj).into_ref(self)) .map_err(|mut compile_error| { compile_error.update_statement_info(source.trim_end().to_owned()); diff --git a/wasm/demo/src/index.js b/wasm/demo/src/index.js index b0e08549a2..9e03edb615 100644 --- a/wasm/demo/src/index.js +++ b/wasm/demo/src/index.js @@ -127,7 +127,7 @@ async function readPrompts() { try { terminalVM.execSingle(input); } catch (err) { - if (err instanceof SyntaxError && err.message.includes('EOF')) { + if (err.canContinue) { continuing = true; continue; } else if (err instanceof WebAssembly.RuntimeError) { diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 44214ea711..916d36fb98 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -1,7 +1,9 @@ -use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, Uint8Array}; +use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array}; use serde_wasm_bindgen; use wasm_bindgen::{closure::Closure, prelude::*, JsCast}; +use rustpython_compiler::error::{CompileError, CompileErrorType}; +use rustpython_parser::error::ParseErrorType; use rustpython_vm::exceptions::PyBaseExceptionRef; use rustpython_vm::function::PyFuncArgs; use rustpython_vm::obj::{objbyteinner::PyBytesLike, objtype}; @@ -216,3 +218,28 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef { .unwrap_or_else(|_| vm.get_none()) } } + +pub fn syntax_err(err: CompileError) -> SyntaxError { + let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err)); + let _ = Reflect::set(&js_err, &"row".into(), &(err.location.row() as u32).into()); + let _ = Reflect::set( + &js_err, + &"col".into(), + &(err.location.column() as u32).into(), + ); + let can_continue = match &err.error { + CompileErrorType::Parse(ParseErrorType::EOF) => true, + _ => false, + }; + let _ = Reflect::set(&js_err, &"canContinue".into(), &can_continue.into()); + js_err +} + +pub trait PyResultExt { + fn to_js(self, vm: &VirtualMachine) -> Result; +} +impl PyResultExt for PyResult { + fn to_js(self, vm: &VirtualMachine) -> Result { + self.map_err(|err| py_err_to_js_err(vm, &err)) + } +} diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index b7cf90d2ad..a917b2ba23 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -2,19 +2,22 @@ use std::cell::RefCell; use std::collections::HashMap; use std::rc::{Rc, Weak}; -use js_sys::{Object, Reflect, SyntaxError, TypeError}; +use js_sys::{Object, TypeError}; use wasm_bindgen::prelude::*; use rustpython_compiler::compile; use rustpython_vm::function::PyFuncArgs; -use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult, PyValue}; +use rustpython_vm::pyobject::{ + ItemProtocol, PyObject, PyObjectPayload, PyObjectRef, PyResult, PyValue, +}; use rustpython_vm::scope::{NameProtocol, Scope}; use rustpython_vm::{InitParameter, PySettings, VirtualMachine}; use crate::browser_module::setup_browser_module; -use crate::convert; +use crate::convert::{self, PyResultExt}; use crate::js_module; use crate::wasm_builtins; +use rustpython_compiler::mode::Mode; pub(crate) struct StoredVirtualMachine { pub vm: VirtualMachine, @@ -251,7 +254,47 @@ impl WASMVirtualMachine { } #[wasm_bindgen(js_name = injectModule)] - pub fn inject_module(&self, name: String, module: Object) -> Result<(), JsValue> { + pub fn inject_module( + &self, + name: String, + source: &str, + imports: Option, + ) -> Result<(), JsValue> { + self.with(|StoredVirtualMachine { ref vm, .. }| { + let code = vm + .compile(source, Mode::Exec, name.clone()) + .map_err(convert::syntax_err)?; + let attrs = vm.ctx.new_dict(); + attrs + .set_item("__name__", vm.new_str(name.clone()), vm) + .to_js(vm)?; + + if let Some(imports) = imports { + for entry in convert::object_entries(&imports) { + let (key, value) = entry?; + let key: String = Object::from(key).to_string().into(); + attrs + .set_item(&key, convert::js_to_py(vm, value), vm) + .to_js(vm)?; + } + } + + vm.run_code_obj(code, Scope::new(None, attrs.clone(), vm)) + .to_js(vm)?; + + let module = vm.new_module(&name, attrs); + + let sys_modules = vm + .get_attribute(vm.sys_module.clone(), "modules") + .to_js(vm)?; + sys_modules.set_item(&name, module, vm).to_js(vm)?; + + Ok(()) + })? + } + + #[wasm_bindgen(js_name = injectJSModule)] + pub fn inject_js_module(&self, name: String, module: Object) -> Result<(), JsValue> { self.with(|StoredVirtualMachine { ref vm, .. }| { let mut module_items: HashMap = HashMap::new(); for entry in convert::object_entries(&module) { @@ -284,28 +327,17 @@ impl WASMVirtualMachine { mode: compile::Mode, source_path: Option, ) -> Result { - self.assert_valid()?; - self.with_unchecked( + self.with( |StoredVirtualMachine { ref vm, ref scope, .. }| { let source_path = source_path.unwrap_or_else(|| "".to_owned()); let code = vm.compile(source, mode, source_path); - let code = code.map_err(|err| { - let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err)); - let _ = - Reflect::set(&js_err, &"row".into(), &(err.location.row() as u32).into()); - let _ = Reflect::set( - &js_err, - &"col".into(), - &(err.location.column() as u32).into(), - ); - js_err - })?; + let code = code.map_err(convert::syntax_err)?; let result = vm.run_code_obj(code, scope.borrow().clone()); convert::pyresult_to_jsresult(vm, result) }, - ) + )? } pub fn exec(&self, source: &str, source_path: Option) -> Result { diff --git a/wasm/tests/test_demo.py b/wasm/tests/test_demo.py index 9b0dc5103d..2dd4d731d1 100644 --- a/wasm/tests/test_demo.py +++ b/wasm/tests/test_demo.py @@ -1,8 +1,6 @@ import time import sys -from selenium import webdriver -from selenium.webdriver.firefox.options import Options import pytest RUN_CODE_TEMPLATE = """ diff --git a/wasm/tests/test_inject_module.py b/wasm/tests/test_inject_module.py new file mode 100644 index 0000000000..462ed65353 --- /dev/null +++ b/wasm/tests/test_inject_module.py @@ -0,0 +1,22 @@ +def test_inject_module_basic(wdriver): + wdriver.execute_script( + """ +const vm = rp.vmStore.init("vm") +vm.injectModule( +"mod", +` +__all__ = ['get_thing'] +def get_thing(): return __thing() +`, +{ __thing: () => 1 }, +true +) +vm.execSingle( +` +import mod +assert mod.get_thing() == 1 +` +); + """ + ) +