diff --git a/Cargo.lock b/Cargo.lock index e844f5343..5ec65d910 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1967,7 +1967,7 @@ dependencies = [ [[package]] name = "rustpython-ast" version = "0.2.0" -source = "git+https://github.com/RustPython/Parser.git?rev=6b60f85cc4ca248af9b787697c41be2adb7a3ec8#6b60f85cc4ca248af9b787697c41be2adb7a3ec8" +source = "git+https://github.com/RustPython/Parser.git?rev=48920a034e93ae7c737129ea427c068dc30e2da5#48920a034e93ae7c737129ea427c068dc30e2da5" dependencies = [ "num-bigint", "rustpython-compiler-core", @@ -2027,7 +2027,7 @@ dependencies = [ [[package]] name = "rustpython-compiler-core" version = "0.2.0" -source = "git+https://github.com/RustPython/Parser.git?rev=6b60f85cc4ca248af9b787697c41be2adb7a3ec8#6b60f85cc4ca248af9b787697c41be2adb7a3ec8" +source = "git+https://github.com/RustPython/Parser.git?rev=48920a034e93ae7c737129ea427c068dc30e2da5#48920a034e93ae7c737129ea427c068dc30e2da5" dependencies = [ "bitflags", "bstr", @@ -2089,7 +2089,7 @@ dependencies = [ [[package]] name = "rustpython-literal" version = "0.2.0" -source = "git+https://github.com/RustPython/Parser.git?rev=6b60f85cc4ca248af9b787697c41be2adb7a3ec8#6b60f85cc4ca248af9b787697c41be2adb7a3ec8" +source = "git+https://github.com/RustPython/Parser.git?rev=48920a034e93ae7c737129ea427c068dc30e2da5#48920a034e93ae7c737129ea427c068dc30e2da5" dependencies = [ "hexf-parse", "lexical-parse-float", @@ -2100,7 +2100,7 @@ dependencies = [ [[package]] name = "rustpython-parser" version = "0.2.0" -source = "git+https://github.com/RustPython/Parser.git?rev=6b60f85cc4ca248af9b787697c41be2adb7a3ec8#6b60f85cc4ca248af9b787697c41be2adb7a3ec8" +source = "git+https://github.com/RustPython/Parser.git?rev=48920a034e93ae7c737129ea427c068dc30e2da5#48920a034e93ae7c737129ea427c068dc30e2da5" dependencies = [ "ahash", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 4428fa44f..98486b1b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,10 +17,10 @@ members = [ ] [workspace.dependencies] -rustpython-literal = { git = "https://github.com/RustPython/Parser.git", rev = "6b60f85cc4ca248af9b787697c41be2adb7a3ec8" } -rustpython-compiler-core = { git = "https://github.com/RustPython/Parser.git", rev = "6b60f85cc4ca248af9b787697c41be2adb7a3ec8" } -rustpython-parser = { git = "https://github.com/RustPython/Parser.git", rev = "6b60f85cc4ca248af9b787697c41be2adb7a3ec8" } -rustpython-ast = { git = "https://github.com/RustPython/Parser.git", rev = "6b60f85cc4ca248af9b787697c41be2adb7a3ec8" } +rustpython-literal = { git = "https://github.com/RustPython/Parser.git", rev = "48920a034e93ae7c737129ea427c068dc30e2da5" } +rustpython-compiler-core = { git = "https://github.com/RustPython/Parser.git", rev = "48920a034e93ae7c737129ea427c068dc30e2da5" } +rustpython-parser = { git = "https://github.com/RustPython/Parser.git", rev = "48920a034e93ae7c737129ea427c068dc30e2da5" } +rustpython-ast = { git = "https://github.com/RustPython/Parser.git", rev = "48920a034e93ae7c737129ea427c068dc30e2da5" } # rustpython-literal = { path = "../RustPython-parser/literal" } # rustpython-compiler-core = { path = "../RustPython-parser/core" } # rustpython-parser = { path = "../RustPython-parser/parser" } diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 5786aa65e..97a5ee874 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -2,7 +2,7 @@ use rustpython_codegen::{compile, symboltable}; use rustpython_parser::ast::{fold::Fold, ConstantOptimizer}; pub use rustpython_codegen::compile::CompileOpts; -pub use rustpython_compiler_core::{BaseError as CompileErrorBody, CodeObject, Mode}; +pub use rustpython_compiler_core::{CodeObject, Mode}; // these modules are out of repository. re-exporting them here for convenience. pub use rustpython_codegen as codegen; @@ -45,12 +45,7 @@ impl From for CompileErrorType { } } -pub type CompileError = rustpython_compiler_core::CompileError; - -fn error_from_parse(error: parser::ParseError, source: &str) -> CompileError { - let error: CompileErrorBody = error.into(); - CompileError::from(error, source) -} +pub type CompileError = rustpython_compiler_core::BaseError; /// Compile a given source code into a bytecode object. pub fn compile( @@ -61,14 +56,14 @@ pub fn compile( ) -> Result { let mut ast = match parser::parse(source, mode.into(), &source_path) { Ok(x) => x, - Err(e) => return Err(error_from_parse(e, source)), + Err(e) => return Err(e.into()), }; if opts.optimize > 0 { ast = ConstantOptimizer::new() .fold_mod(ast) .unwrap_or_else(|e| match e {}); } - compile::compile_top(&ast, source_path, mode, opts).map_err(|e| CompileError::from(e, source)) + compile::compile_top(&ast, source_path, mode, opts).map_err(|e| e.into()) } pub fn compile_symtable( @@ -76,16 +71,15 @@ pub fn compile_symtable( mode: compile::Mode, source_path: &str, ) -> Result { - let parse_err = |e| error_from_parse(e, source); let res = match mode { compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => { - let ast = parser::parse_program(source, source_path).map_err(parse_err)?; + let ast = parser::parse_program(source, source_path).map_err(|e| e.into())?; symboltable::SymbolTable::scan_program(&ast) } compile::Mode::Eval => { - let expr = parser::parse_expression(source, source_path).map_err(parse_err)?; + let expr = parser::parse_expression(source, source_path).map_err(|e| e.into())?; symboltable::SymbolTable::scan_expr(&expr) } }; - res.map_err(|e| CompileError::from(e.into_codegen_error(source_path.to_owned()), source)) + res.map_err(|e| e.into_codegen_error(source_path.to_owned()).into()) } diff --git a/examples/hello_embed.rs b/examples/hello_embed.rs index 82d2cfe67..958fdaaf4 100644 --- a/examples/hello_embed.rs +++ b/examples/hello_embed.rs @@ -3,14 +3,10 @@ use rustpython_vm as vm; fn main() -> vm::PyResult<()> { vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { let scope = vm.new_scope_with_builtins(); - + let source = r#"print("Hello World!")"#; let code_obj = vm - .compile( - r#"print("Hello World!")"#, - vm::compiler::Mode::Exec, - "".to_owned(), - ) - .map_err(|err| vm.new_syntax_error(&err))?; + .compile(source, vm::compiler::Mode::Exec, "".to_owned()) + .map_err(|err| vm.new_syntax_error(&err, Some(source)))?; vm.run_code_obj(code_obj, scope)?; diff --git a/examples/mini_repl.rs b/examples/mini_repl.rs index c5c2114ac..fe3b13dcf 100644 --- a/examples/mini_repl.rs +++ b/examples/mini_repl.rs @@ -65,7 +65,7 @@ def fib(n): // (note that this is only the case when compiler::Mode::Single is passed to vm.compile) match vm .compile(&input, vm::compiler::Mode::Single, "".to_owned()) - .map_err(|err| vm.new_syntax_error(&err)) + .map_err(|err| vm.new_syntax_error(&err, Some(&input))) .and_then(|code_obj| vm.run_code_obj(code_obj, scope.clone())) { Ok(output) => { diff --git a/src/shell.rs b/src/shell.rs index a156bbed5..4b26aff1a 100644 --- a/src/shell.rs +++ b/src/shell.rs @@ -3,7 +3,7 @@ mod helper; use rustpython_parser::{lexer::LexicalErrorType, ParseErrorType, Tok}; use rustpython_vm::{ builtins::PyBaseExceptionRef, - compiler::{self, CompileError, CompileErrorBody, CompileErrorType}, + compiler::{self, CompileError, CompileErrorType}, readline::{Readline, ReadlineResult}, scope::Scope, AsObject, PyResult, VirtualMachine, @@ -36,19 +36,11 @@ fn shell_exec( } } Err(CompileError { - body: - CompileErrorBody { - error: CompileErrorType::Parse(ParseErrorType::Lexical(LexicalErrorType::Eof)), - .. - }, + error: CompileErrorType::Parse(ParseErrorType::Lexical(LexicalErrorType::Eof)), .. }) | Err(CompileError { - body: - CompileErrorBody { - error: CompileErrorType::Parse(ParseErrorType::Eof), - .. - }, + error: CompileErrorType::Parse(ParseErrorType::Eof), .. }) => ShellExecResult::Continue, Err(err) => { @@ -57,13 +49,13 @@ fn shell_exec( // since indentations errors on columns other than 0 should be ignored. // if its an unrecognized token for dedent, set to false - let bad_error = match err.body.error { + let bad_error = match err.error { CompileErrorType::Parse(ref p) => { if matches!( p, ParseErrorType::Lexical(LexicalErrorType::IndentationError) ) { - continuing && err.body.location.column() != 0 + continuing && err.location.column() != 0 } else { !matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _)) } @@ -73,7 +65,7 @@ fn shell_exec( // If we are handling an error on an empty line or an error worthy of throwing if empty_line_given || bad_error { - ShellExecResult::PyErr(vm.new_syntax_error(&err)) + ShellExecResult::PyErr(vm.new_syntax_error(&err, Some(source))) } else { ShellExecResult::Continue } diff --git a/stdlib/src/dis.rs b/stdlib/src/dis.rs index dee808c05..9ac6245a2 100644 --- a/stdlib/src/dis.rs +++ b/stdlib/src/dis.rs @@ -16,7 +16,7 @@ mod decl { } else if let Ok(co_str) = PyStrRef::try_from_object(vm, obj.clone()) { // String: vm.compile(co_str.as_str(), compiler::Mode::Exec, "".to_owned()) - .map_err(|err| vm.new_syntax_error(&err))? + .map_err(|err| vm.new_syntax_error(&err, Some(co_str.as_str())))? } else { PyRef::try_from_object(vm, obj)? }; diff --git a/vm/src/compiler.rs b/vm/src/compiler.rs index d3b3714c8..4acba750d 100644 --- a/vm/src/compiler.rs +++ b/vm/src/compiler.rs @@ -26,8 +26,8 @@ mod error { #[cfg(not(feature = "rustpython-compiler"))] pub use error::{CompileError, CompileErrorType}; -impl ToPyException for CompileError { +impl ToPyException for (CompileError, Option<&str>) { fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef { - vm.new_syntax_error(self) + vm.new_syntax_error(&self.0, self.1) } } diff --git a/vm/src/eval.rs b/vm/src/eval.rs index b503d8e2a..35f27dc9d 100644 --- a/vm/src/eval.rs +++ b/vm/src/eval.rs @@ -6,7 +6,7 @@ pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) debug!("Code object: {:?}", bytecode); vm.run_code_obj(bytecode, scope) } - Err(err) => Err(vm.new_syntax_error(&err)), + Err(err) => Err(vm.new_syntax_error(&err, Some(source))), } } diff --git a/vm/src/import.rs b/vm/src/import.rs index 5d7c0da65..0edc2f77a 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -118,7 +118,7 @@ pub fn import_file( file_path, vm.compile_opts(), ) - .map_err(|err| vm.new_syntax_error(&err))?; + .map_err(|err| vm.new_syntax_error(&err, Some(&content)))?; import_codeobj(vm, module_name, code, true) } diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 2c4254d3e..17485c98b 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -307,8 +307,7 @@ pub(crate) fn parse( source: &str, mode: parser::Mode, ) -> Result { - let top = - parser::parse(source, mode, "").map_err(|err| CompileError::from(err, source))?; + let top = parser::parse(source, mode, "").map_err(CompileError::from)?; Ok(top.ast_to_object(vm)) } @@ -322,7 +321,7 @@ pub(crate) fn compile( let opts = vm.compile_opts(); let ast = Node::ast_from_object(vm, object)?; let code = codegen::compile::compile_top(&ast, filename.to_owned(), mode, opts) - .map_err(|err| CompileError::from(err, "").to_pyexception(vm))?; // FIXME source + .map_err(|err| (CompileError::from(err), None).to_pyexception(vm))?; // FIXME source Ok(vm.ctx.new_code(code).into()) } diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index e14a6847b..6ea472803 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -173,14 +173,14 @@ mod builtins { .map_err(|err| vm.new_value_error(err.to_string()))?; let code = vm .compile(source, mode, args.filename.as_str().to_owned()) - .map_err(|err| err.to_pyexception(vm))?; + .map_err(|err| (err, Some(source)).to_pyexception(vm))?; Ok(code.into()) } } else { let mode = mode_str .parse::() .map_err(|err| vm.new_value_error(err.to_string()))?; - ast::parse(vm, source, mode).map_err(|e| e.to_pyexception(vm)) + ast::parse(vm, source, mode).map_err(|e| (e, Some(source)).to_pyexception(vm)) } } } @@ -300,7 +300,7 @@ mod builtins { #[cfg(feature = "rustpython-compiler")] Either::A(string) => vm .compile(string.as_str(), mode, "".to_owned()) - .map_err(|err| vm.new_syntax_error(&err))?, + .map_err(|err| vm.new_syntax_error(&err, Some(string.as_str())))?, #[cfg(not(feature = "rustpython-compiler"))] Either::A(_) => return Err(vm.new_type_error(CODEGEN_NOT_SUPPORTED.to_owned())), Either::B(code_obj) => code_obj, diff --git a/vm/src/stdlib/symtable.rs b/vm/src/stdlib/symtable.rs index 4a8fb8638..d497a15b7 100644 --- a/vm/src/stdlib/symtable.rs +++ b/vm/src/stdlib/symtable.rs @@ -23,7 +23,7 @@ mod symtable { .map_err(|err| vm.new_value_error(err.to_string()))?; let symtable = compiler::compile_symtable(source.as_str(), mode, filename.as_str()) - .map_err(|err| vm.new_syntax_error(&err))?; + .map_err(|err| vm.new_syntax_error(&err, Some(source.as_str())))?; let py_symbol_table = to_py_symbol_table(symtable); Ok(py_symbol_table.into_ref(&vm.ctx)) diff --git a/vm/src/vm/compile.rs b/vm/src/vm/compile.rs index d2bfb18ab..c44158f20 100644 --- a/vm/src/vm/compile.rs +++ b/vm/src/vm/compile.rs @@ -58,7 +58,7 @@ impl VirtualMachine { pub fn run_code_string(&self, scope: Scope, source: &str, source_path: String) -> PyResult { let code_obj = self .compile(source, compiler::Mode::Exec, source_path.clone()) - .map_err(|err| self.new_syntax_error(&err))?; + .map_err(|err| self.new_syntax_error(&err, Some(source)))?; // trace!("Code object: {:?}", code_obj.borrow()); scope.globals.set_item( identifier!(self, __file__), @@ -71,7 +71,7 @@ impl VirtualMachine { pub fn run_block_expr(&self, scope: Scope, source: &str) -> PyResult { let code_obj = self .compile(source, compiler::Mode::BlockExpr, "".to_owned()) - .map_err(|err| self.new_syntax_error(&err))?; + .map_err(|err| self.new_syntax_error(&err, Some(source)))?; // trace!("Code object: {:?}", code_obj.borrow()); self.run_code_obj(code_obj, scope) } diff --git a/vm/src/vm/interpreter.rs b/vm/src/vm/interpreter.rs index de928636a..aff8ef05c 100644 --- a/vm/src/vm/interpreter.rs +++ b/vm/src/vm/interpreter.rs @@ -14,10 +14,12 @@ use std::sync::atomic::Ordering; /// use rustpython_vm::compiler::Mode; /// Interpreter::without_stdlib(Default::default()).enter(|vm| { /// let scope = vm.new_scope_with_builtins(); -/// let code_obj = vm.compile(r#"print("Hello World!")"#, +/// let source = r#"print("Hello World!")"#; +/// let code_obj = vm.compile( +/// source, /// Mode::Exec, /// "".to_owned(), -/// ).map_err(|err| vm.new_syntax_error(&err)).unwrap(); +/// ).map_err(|err| vm.new_syntax_error(&err, Some(source))).unwrap(); /// vm.run_code_obj(code_obj, scope).unwrap(); /// }); /// ``` diff --git a/vm/src/vm/mod.rs b/vm/src/vm/mod.rs index d993cdb93..c632ab693 100644 --- a/vm/src/vm/mod.rs +++ b/vm/src/vm/mod.rs @@ -848,13 +848,10 @@ fn test_nested_frozen() { .enter(|vm| { let scope = vm.new_scope_with_builtins(); + let source = "from dir_module.dir_module_inner import value2"; let code_obj = vm - .compile( - "from dir_module.dir_module_inner import value2", - vm::compiler::Mode::Exec, - "".to_owned(), - ) - .map_err(|err| vm.new_syntax_error(&err)) + .compile(source, vm::compiler::Mode::Exec, "".to_owned()) + .map_err(|err| vm.new_syntax_error(&err, Some(source))) .unwrap(); if let Err(e) = vm.run_code_obj(code_obj, scope) { diff --git a/vm/src/vm/vm_new.rs b/vm/src/vm/vm_new.rs index 301c488f0..181783480 100644 --- a/vm/src/vm/vm_new.rs +++ b/vm/src/vm/vm_new.rs @@ -248,7 +248,11 @@ impl VirtualMachine { } #[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))] - pub fn new_syntax_error(&self, error: &crate::compiler::CompileError) -> PyBaseExceptionRef { + pub fn new_syntax_error( + &self, + error: &crate::compiler::CompileError, + source: Option<&str>, + ) -> PyBaseExceptionRef { let syntax_error_type = match &error.error { #[cfg(feature = "rustpython-parser")] crate::compiler::CompileErrorType::Parse(p) if p.is_indentation_error() => { @@ -261,7 +265,40 @@ impl VirtualMachine { _ => self.ctx.exceptions.syntax_error, } .to_owned(); - let syntax_error = self.new_exception_msg(syntax_error_type, error.to_string()); + + fn get_statement(source: &str, loc: rustpython_compiler_core::Location) -> Option { + if loc.column() == 0 || loc.row() == 0 { + return None; + } + let line = source.split('\n').nth(loc.row() - 1)?.to_owned(); + Some(line + "\n") + } + + let statement = if let Some(source) = source { + get_statement(source, error.location) + } else { + None + }; + + fn fmt( + error: &crate::compiler::CompileError, + statement: Option<&str>, + f: &mut impl std::fmt::Write, + ) -> std::fmt::Result { + let loc = error.location; + if let Some(ref stmt) = statement { + // visualize the error when location and statement are provided + loc.fmt_with(f, &error.error)?; + write!(f, "\n{stmt}{arrow:>pad$}", pad = loc.column(), arrow = "^") + } else { + loc.fmt_with(f, &error.error) + } + } + + let mut msg = String::new(); + fmt(error, statement.as_deref(), &mut msg).unwrap(); + + let syntax_error = self.new_exception_msg(syntax_error_type, msg); let lineno = self.ctx.new_int(error.location.row()); let offset = self.ctx.new_int(error.location.column()); syntax_error @@ -272,9 +309,10 @@ impl VirtualMachine { .as_object() .set_attr("offset", offset, self) .unwrap(); + syntax_error .as_object() - .set_attr("text", error.statement.clone().to_pyobject(self), self) + .set_attr("text", statement.to_pyobject(self), self) .unwrap(); syntax_error .as_object()