mschoi/RustPython
1
0
Fork 0
forked from Rust-related/RustPython
Code Pull Requests Actions Activity

Merge pull request #1744 from RustPython/coolreader18/wasm-inject-module

Browse Source 
Add a better injectModule method to WASM
This commit is contained in:
Noah
2020-04-03 15:39:39 -05:00
committed by GitHub
parent a104d43d7f a8027ab48a
commit 71af10bc5a
17 changed files with 246 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
benchmarks/bench.rs
View File

@@ -94,15 +94,14 @@ fn bench_rustpy_nbody(b: &mut test::Bencher) {
let vm = VirtualMachine::default();
let code = match vm.compile(source, compile::Mode::Single, "<stdin>".to_owned()) {
Ok(code) => code,
Err(e) => panic!("{:?}", e),
};
let code = vm
.compile(source, compile::Mode::Exec, "<stdin>".to_owned())
.unwrap();
b.iter(|| {
let scope = vm.new_scope_with_builtins();
let res: PyResult = vm.run_code_obj(code.clone(), scope);
assert!(res.is_ok());
vm.unwrap_pyresult(res);
})
}
@@ -113,14 +112,13 @@ fn bench_rustpy_mandelbrot(b: &mut test::Bencher) {
let vm = VirtualMachine::default();
let code = match vm.compile(source, compile::Mode::Single, "<stdin>".to_owned()) {
Ok(code) => code,
Err(e) => panic!("{:?}", e),
};
let code = vm
.compile(source, compile::Mode::Exec, "<stdin>".to_owned())
.unwrap();
b.iter(|| {
let scope = vm.new_scope_with_builtins();
let res: PyResult = vm.run_code_obj(code.clone(), scope);
assert!(res.is_ok());
vm.unwrap_pyresult(res);
})
}

2
bytecode/src/bytecode.rs
View File

@@ -50,7 +50,7 @@ pub struct CodeObject {
bitflags! {
#[derive(Serialize, Deserialize)]
pub struct CodeFlags: u8 {
pub struct CodeFlags: u16 {
const HAS_DEFAULTS = 0x01;
const HAS_KW_ONLY_DEFAULTS = 0x02;
const HAS_ANNOTATIONS = 0x04;

80
compiler/src/compile.rs
View File

@@ -29,8 +29,21 @@ struct Compiler<O: OutputStream = BasicOutputStream> {
source_path: Option<String>,
current_source_location: ast::Location,
current_qualified_path: Option<String>,
done_with_future_stmts: bool,
ctx: CompileContext,
optimize: u8,
opts: CompileOpts,
}
#[derive(Debug, Clone)]
pub struct CompileOpts {
/// How optimized the bytecode output should be; any optimize > 0 does
/// not emit assert statements
pub optimize: u8,
}
impl Default for CompileOpts {
fn default() -> Self {
CompileOpts { optimize: 0 }
}
}
#[derive(Clone, Copy)]
@@ -60,20 +73,20 @@ pub fn compile(
source: &str,
mode: Mode,
source_path: String,
optimize: u8,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
match mode {
Mode::Exec => {
let ast = parser::parse_program(source)?;
compile_program(ast, source_path, optimize)
compile_program(ast, source_path, opts)
}
Mode::Eval => {
let statement = parser::parse_statement(source)?;
compile_statement_eval(statement, source_path, optimize)
compile_statement_eval(statement, source_path, opts)
}
Mode::Single => {
let ast = parser::parse_program(source)?;
compile_program_single(ast, source_path, optimize)
compile_program_single(ast, source_path, opts)
}
}
}
@@ -81,10 +94,10 @@ pub fn compile(
/// A helper function for the shared code of the different compile functions
fn with_compiler(
source_path: String,
optimize: u8,
opts: CompileOpts,
f: impl FnOnce(&mut Compiler) -> CompileResult<()>,
) -> CompileResult<CodeObject> {
let mut compiler = Compiler::new(optimize);
let mut compiler = Compiler::new(opts);
compiler.source_path = Some(source_path);
compiler.push_new_code_object("<module>".to_owned());
f(&mut compiler)?;
@@ -97,9 +110,9 @@ fn with_compiler(
pub fn compile_program(
ast: ast::Program,
source_path: String,
optimize: u8,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
with_compiler(source_path, optimize, |compiler| {
with_compiler(source_path, opts, |compiler| {
let symbol_table = make_symbol_table(&ast)?;
compiler.compile_program(&ast, symbol_table)
})
@@ -109,9 +122,9 @@ pub fn compile_program(
pub fn compile_statement_eval(
statement: Vec<ast::Statement>,
source_path: String,
optimize: u8,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
with_compiler(source_path, optimize, |compiler| {
with_compiler(source_path, opts, |compiler| {
let symbol_table = statements_to_symbol_table(&statement)?;
compiler.compile_statement_eval(&statement, symbol_table)
})
@@ -121,9 +134,9 @@ pub fn compile_statement_eval(
pub fn compile_program_single(
ast: ast::Program,
source_path: String,
optimize: u8,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
with_compiler(source_path, optimize, |compiler| {
with_compiler(source_path, opts, |compiler| {
let symbol_table = make_symbol_table(&ast)?;
compiler.compile_program_single(&ast, symbol_table)
})
@@ -134,12 +147,12 @@ where
O: OutputStream,
{
fn default() -> Self {
Compiler::new(0)
Compiler::new(CompileOpts::default())
}
}
impl<O: OutputStream> Compiler<O> {
fn new(optimize: u8) -> Self {
fn new(opts: CompileOpts) -> Self {
Compiler {
output_stack: Vec::new(),
symbol_table_stack: Vec::new(),
@@ -147,11 +160,12 @@ impl<O: OutputStream> Compiler<O> {
source_path: None,
current_source_location: ast::Location::default(),
current_qualified_path: None,
done_with_future_stmts: false,
ctx: CompileContext {
in_loop: false,
func: FunctionContext::NoFunction,
},
optimize,
opts,
}
}
@@ -314,6 +328,16 @@ impl<O: OutputStream> Compiler<O> {
self.set_source_location(statement.location);
use ast::StatementType::*;
match &statement.node {
// we do this here because `from __future__` still executes that `from` statement at runtime,
// we still need to compile the ImportFrom down below
ImportFrom { module, names, .. } if module.as_deref() == Some("__future__") => {
self.compile_future_features(&names)?
}
// if we find any other statement, stop accepting future statements
_ => self.done_with_future_stmts = true,
}
match &statement.node {
Import { names } => {
// import a, b, c as d
@@ -518,7 +542,7 @@ impl<O: OutputStream> Compiler<O> {
} => self.compile_class_def(name, body, bases, keywords, decorator_list)?,
Assert { test, msg } => {
// if some flag, ignore all assert statements!
if self.optimize == 0 {
if self.opts.optimize == 0 {
let end_label = self.new_label();
self.compile_jump_if(test, true, end_label)?;
self.emit(Instruction::LoadName {
@@ -2112,6 +2136,28 @@ impl<O: OutputStream> Compiler<O> {
Ok(())
}
fn compile_future_features(
&mut self,
features: &[ast::ImportSymbol],
) -> Result<(), CompileError> {
if self.done_with_future_stmts {
return Err(self.error(CompileErrorType::InvalidFuturePlacement));
}
for feature in features {
match &*feature.symbol {
// Python 3 features; we've already implemented them by default
"nested_scopes" | "generators" | "division" | "absolute_import"
| "with_statement" | "print_function" | "unicode_literals" => {}
// "generator_stop" => {}
// "annotations" => {}
other => {
return Err(self.error(CompileErrorType::InvalidFutureFeature(other.to_owned())))
}
}
}
Ok(())
}
// Scope helpers:
fn enter_scope(&mut self) {
// println!("Enter scope {:?}", self.symbol_table_stack);

8
compiler/src/error.rs
View File

@@ -58,6 +58,8 @@ pub enum CompileErrorType {
InvalidAwait,
AsyncYieldFrom,
AsyncReturnValue,
InvalidFuturePlacement,
InvalidFutureFeature(String),
}
impl CompileError {
@@ -106,6 +108,12 @@ impl fmt::Display for CompileError {
CompileErrorType::AsyncReturnValue => {
"'return' with value inside async generator".to_owned()
}
CompileErrorType::InvalidFuturePlacement => {
"from __future__ imports must occur at the beginning of the file".to_owned()
}
CompileErrorType::InvalidFutureFeature(feat) => {
format!("future feature {} is not defined", feat)
}
};
if let Some(statement) = &self.statement {

2
derive/src/compile_bytecode.rs
View File

@@ -49,7 +49,7 @@ impl CompilationSource {
module_name: String,
origin: F,
) -> Result<CodeObject, Diagnostic> {
compile::compile(source, mode, module_name, 0).map_err(|err| {
compile::compile(source, mode, module_name, Default::default()).map_err(|err| {
Diagnostic::spans_error(
self.span,
format!("Python compile error from {}: {}", origin(), err),

11
examples/dis.rs
View File

@@ -62,9 +62,14 @@ fn main() {
let optimize = matches.occurrences_of("optimize") as u8;
let scripts = matches.values_of_os("scripts").unwrap();
let opts = compile::CompileOpts {
optimize,
..Default::default()
};
for script in scripts.map(Path::new) {
if script.exists() && script.is_file() {
let res = display_script(script, mode, optimize, expand_codeobjects);
let res = display_script(script, mode, opts.clone(), expand_codeobjects);
if let Err(e) = res {
error!("Error while compiling {:?}: {}", script, e);
}
@@ -77,11 +82,11 @@ fn main() {
fn display_script(
path: &Path,
mode: compile::Mode,
optimize: u8,
opts: compile::CompileOpts,
expand_codeobjects: bool,
) -> Result<(), Box<dyn Error>> {
let source = fs::read_to_string(path)?;
let code = compile::compile(&source, mode, path.to_string_lossy().into_owned(), optimize)?;
let code = compile::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?;
println!("{}:", path.display());
if expand_codeobjects {
println!("{}", code.display_expand_codeobjects());

1
tests/snippets/function.py
View File

@@ -13,6 +13,7 @@ assert foo.__doc__ == "test"
assert foo.__name__ == "foo"
assert foo.__qualname__ == "foo"
assert foo.__module__ == "function"
assert foo.__globals__ is globals()
def my_func(a,):
return a+2

27
tests/snippets/invalid_syntax.py
View File

@@ -7,12 +7,9 @@ def valid_func():
yield 2
"""
try:
with assert_raises(SyntaxError) as ae:
compile(src, 'test.py', 'exec')
except SyntaxError as ex:
assert ex.lineno == 5
else:
raise AssertionError("Must throw syntax error")
assert ae.exception.lineno == 5
src = """
if True:
@@ -60,3 +57,23 @@ src = """
with assert_raises(SyntaxError):
compile(src, 'test.py', 'exec')
src = """
from __future__ import not_a_real_future_feature
"""
with assert_raises(SyntaxError):
compile(src, 'test.py', 'exec')
src = """
a = 1
from __future__ import print_function
"""
with assert_raises(SyntaxError):
compile(src, 'test.py', 'exec')
src = """
from __future__ import print_function
"""
compile(src, 'test.py', 'exec')

9
vm/src/import.rs
View File

@@ -74,13 +74,8 @@ pub fn import_file(
file_path: String,
content: String,
) -> PyResult {
let code_obj = compile::compile(
&content,
compile::Mode::Exec,
file_path,
vm.settings.optimize,
)
.map_err(|err| vm.new_syntax_error(&err))?;
let code_obj = compile::compile(&content, compile::Mode::Exec, file_path, vm.compile_opts())
.map_err(|err| vm.new_syntax_error(&err))?;
import_codeobj(vm, module_name, code_obj, true)
}

2
vm/src/obj/objcode.rs
View File

@@ -103,7 +103,7 @@ impl PyCodeRef {
}
#[pyproperty]
fn co_flags(self) -> u8 {
fn co_flags(self) -> u16 {
self.code.flags.bits()
}
}

5
vm/src/obj/objfunction.rs
View File

@@ -278,6 +278,11 @@ impl PyFunction {
fn kwdefaults(&self) -> Option<PyDictRef> {
self.kw_only_defaults.clone()
}
#[pyproperty(magic)]
fn globals(&self) -> PyDictRef {
self.scope.globals.clone()
}
}
#[pyclass]

27
vm/src/vm.rs
View File

@@ -17,7 +17,10 @@ use num_bigint::BigInt;
use num_traits::ToPrimitive;
use once_cell::sync::Lazy;
#[cfg(feature = "rustpython-compiler")]
use rustpython_compiler::{compile, error::CompileError};
use rustpython_compiler::{
compile::{self, CompileOpts},
error::CompileError,
};
use crate::builtins::{self, to_ascii};
use crate::bytecode;
@@ -1000,6 +1003,15 @@ impl VirtualMachine {
}
}
/// Returns a basic CompileOpts instance with options accurate to the vm. Used
/// as the CompileOpts for `vm.compile()`.
#[cfg(feature = "rustpython-compiler")]
pub fn compile_opts(&self) -> CompileOpts {
CompileOpts {
optimize: self.settings.optimize,
}
}
#[cfg(feature = "rustpython-compiler")]
pub fn compile(
&self,
@@ -1007,7 +1019,18 @@ impl VirtualMachine {
mode: compile::Mode,
source_path: String,
) -> Result<PyCodeRef, CompileError> {
compile::compile(source, mode, source_path, self.settings.optimize)
self.compile_with_opts(source, mode, source_path, self.compile_opts())
}
#[cfg(feature = "rustpython-compiler")]
pub fn compile_with_opts(
&self,
source: &str,
mode: compile::Mode,
source_path: String,
opts: CompileOpts,
) -> Result<PyCodeRef, CompileError> {
compile::compile(source, mode, source_path, opts)
.map(|codeobj| PyCode::new(codeobj).into_ref(self))
.map_err(|mut compile_error| {
compile_error.update_statement_info(source.trim_end().to_owned());

2
wasm/demo/src/index.js
View File

@@ -127,7 +127,7 @@ async function readPrompts() {
try {
terminalVM.execSingle(input);
} catch (err) {
if (err instanceof SyntaxError && err.message.includes('EOF')) {
if (err.canContinue) {
continuing = true;
continue;
} else if (err instanceof WebAssembly.RuntimeError) {

29
wasm/lib/src/convert.rs
View File

@@ -1,7 +1,9 @@
use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, Uint8Array};
use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Array};
use serde_wasm_bindgen;
use wasm_bindgen::{closure::Closure, prelude::*, JsCast};
use rustpython_compiler::error::{CompileError, CompileErrorType};
use rustpython_parser::error::ParseErrorType;
use rustpython_vm::exceptions::PyBaseExceptionRef;
use rustpython_vm::function::PyFuncArgs;
use rustpython_vm::obj::{objbyteinner::PyBytesLike, objtype};
@@ -216,3 +218,28 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef {
.unwrap_or_else(|_| vm.get_none())
}
}
pub fn syntax_err(err: CompileError) -> SyntaxError {
let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err));
let _ = Reflect::set(&js_err, &"row".into(), &(err.location.row() as u32).into());
let _ = Reflect::set(
&js_err,
&"col".into(),
&(err.location.column() as u32).into(),
);
let can_continue = match &err.error {
CompileErrorType::Parse(ParseErrorType::EOF) => true,
_ => false,
};
let _ = Reflect::set(&js_err, &"canContinue".into(), &can_continue.into());
js_err
}
pub trait PyResultExt<T> {
fn to_js(self, vm: &VirtualMachine) -> Result<T, JsValue>;
}
impl<T> PyResultExt<T> for PyResult<T> {
fn to_js(self, vm: &VirtualMachine) -> Result<T, JsValue> {
self.map_err(|err| py_err_to_js_err(vm, &err))
}
}

68
wasm/lib/src/vm_class.rs
View File

@@ -2,19 +2,22 @@ use std::cell::RefCell;
use std::collections::HashMap;
use std::rc::{Rc, Weak};
use js_sys::{Object, Reflect, SyntaxError, TypeError};
use js_sys::{Object, TypeError};
use wasm_bindgen::prelude::*;
use rustpython_compiler::compile;
use rustpython_vm::function::PyFuncArgs;
use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult, PyValue};
use rustpython_vm::pyobject::{
ItemProtocol, PyObject, PyObjectPayload, PyObjectRef, PyResult, PyValue,
};
use rustpython_vm::scope::{NameProtocol, Scope};
use rustpython_vm::{InitParameter, PySettings, VirtualMachine};
use crate::browser_module::setup_browser_module;
use crate::convert;
use crate::convert::{self, PyResultExt};
use crate::js_module;
use crate::wasm_builtins;
use rustpython_compiler::mode::Mode;
pub(crate) struct StoredVirtualMachine {
pub vm: VirtualMachine,
@@ -251,7 +254,47 @@ impl WASMVirtualMachine {
}
#[wasm_bindgen(js_name = injectModule)]
pub fn inject_module(&self, name: String, module: Object) -> Result<(), JsValue> {
pub fn inject_module(
&self,
name: String,
source: &str,
imports: Option<Object>,
) -> Result<(), JsValue> {
self.with(|StoredVirtualMachine { ref vm, .. }| {
let code = vm
.compile(source, Mode::Exec, name.clone())
.map_err(convert::syntax_err)?;
let attrs = vm.ctx.new_dict();
attrs
.set_item("__name__", vm.new_str(name.clone()), vm)
.to_js(vm)?;
if let Some(imports) = imports {
for entry in convert::object_entries(&imports) {
let (key, value) = entry?;
let key: String = Object::from(key).to_string().into();
attrs
.set_item(&key, convert::js_to_py(vm, value), vm)
.to_js(vm)?;
}
}
vm.run_code_obj(code, Scope::new(None, attrs.clone(), vm))
.to_js(vm)?;
let module = vm.new_module(&name, attrs);
let sys_modules = vm
.get_attribute(vm.sys_module.clone(), "modules")
.to_js(vm)?;
sys_modules.set_item(&name, module, vm).to_js(vm)?;
Ok(())
})?
}
#[wasm_bindgen(js_name = injectJSModule)]
pub fn inject_js_module(&self, name: String, module: Object) -> Result<(), JsValue> {
self.with(|StoredVirtualMachine { ref vm, .. }| {
let mut module_items: HashMap<String, PyObjectRef> = HashMap::new();
for entry in convert::object_entries(&module) {
@@ -284,28 +327,17 @@ impl WASMVirtualMachine {
mode: compile::Mode,
source_path: Option<String>,
) -> Result<JsValue, JsValue> {
self.assert_valid()?;
self.with_unchecked(
self.with(
|StoredVirtualMachine {
ref vm, ref scope, ..
}| {
let source_path = source_path.unwrap_or_else(|| "<wasm>".to_owned());
let code = vm.compile(source, mode, source_path);
let code = code.map_err(|err| {
let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err));
let _ =
Reflect::set(&js_err, &"row".into(), &(err.location.row() as u32).into());
let _ = Reflect::set(
&js_err,
&"col".into(),
&(err.location.column() as u32).into(),
);
js_err
})?;
let code = code.map_err(convert::syntax_err)?;
let result = vm.run_code_obj(code, scope.borrow().clone());
convert::pyresult_to_jsresult(vm, result)
},
)
)?
}
pub fn exec(&self, source: &str, source_path: Option<String>) -> Result<JsValue, JsValue> {

2
wasm/tests/test_demo.py
View File

@@ -1,8 +1,6 @@
import time
import sys
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
import pytest
RUN_CODE_TEMPLATE = """

22
wasm/tests/test_inject_module.py Normal file
View File

@@ -0,0 +1,22 @@
def test_inject_module_basic(wdriver):
wdriver.execute_script(
"""
const vm = rp.vmStore.init("vm")
vm.injectModule(
"mod",
`
__all__ = ['get_thing']
def get_thing(): return __thing()
`,
{ __thing: () => 1 },
true
)
vm.execSingle(
`
import mod
assert mod.get_thing() == 1
`
);
"""
)