mschoi/RustPython
1
0
Fork 0
forked from Rust-related/RustPython
Code Pull Requests Actions Activity

Merge pull request #4956 from youknowone/remove-core-compiler-error

Browse Source 
Remove compiler_core::CompileError
This commit is contained in:
Jeong, YunWon
2023-05-06 19:14:57 +09:00
committed by GitHub
parent 91b9ed17c2 bac1cdc55f
commit 0897385a30
17 changed files with 86 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
Cargo.lock generated
View File

@@ -1967,7 +1967,7 @@ dependencies = [
[[package]]
name = "rustpython-ast"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=6b60f85cc4ca248af9b787697c41be2adb7a3ec8#6b60f85cc4ca248af9b787697c41be2adb7a3ec8"
source = "git+https://github.com/RustPython/Parser.git?rev=48920a034e93ae7c737129ea427c068dc30e2da5#48920a034e93ae7c737129ea427c068dc30e2da5"
dependencies = [
"num-bigint",
"rustpython-compiler-core",
@@ -2027,7 +2027,7 @@ dependencies = [
[[package]]
name = "rustpython-compiler-core"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=6b60f85cc4ca248af9b787697c41be2adb7a3ec8#6b60f85cc4ca248af9b787697c41be2adb7a3ec8"
source = "git+https://github.com/RustPython/Parser.git?rev=48920a034e93ae7c737129ea427c068dc30e2da5#48920a034e93ae7c737129ea427c068dc30e2da5"
dependencies = [
"bitflags",
"bstr",
@@ -2089,7 +2089,7 @@ dependencies = [
[[package]]
name = "rustpython-literal"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=6b60f85cc4ca248af9b787697c41be2adb7a3ec8#6b60f85cc4ca248af9b787697c41be2adb7a3ec8"
source = "git+https://github.com/RustPython/Parser.git?rev=48920a034e93ae7c737129ea427c068dc30e2da5#48920a034e93ae7c737129ea427c068dc30e2da5"
dependencies = [
"hexf-parse",
"lexical-parse-float",
@@ -2100,7 +2100,7 @@ dependencies = [
[[package]]
name = "rustpython-parser"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=6b60f85cc4ca248af9b787697c41be2adb7a3ec8#6b60f85cc4ca248af9b787697c41be2adb7a3ec8"
source = "git+https://github.com/RustPython/Parser.git?rev=48920a034e93ae7c737129ea427c068dc30e2da5#48920a034e93ae7c737129ea427c068dc30e2da5"
dependencies = [
"ahash",
"anyhow",

8
Cargo.toml
View File

@@ -17,10 +17,10 @@ members = [
]
[workspace.dependencies]
rustpython-literal = { git = "https://github.com/RustPython/Parser.git", rev = "6b60f85cc4ca248af9b787697c41be2adb7a3ec8" }
rustpython-compiler-core = { git = "https://github.com/RustPython/Parser.git", rev = "6b60f85cc4ca248af9b787697c41be2adb7a3ec8" }
rustpython-parser = { git = "https://github.com/RustPython/Parser.git", rev = "6b60f85cc4ca248af9b787697c41be2adb7a3ec8" }
rustpython-ast = { git = "https://github.com/RustPython/Parser.git", rev = "6b60f85cc4ca248af9b787697c41be2adb7a3ec8" }
rustpython-literal = { git = "https://github.com/RustPython/Parser.git", rev = "48920a034e93ae7c737129ea427c068dc30e2da5" }
rustpython-compiler-core = { git = "https://github.com/RustPython/Parser.git", rev = "48920a034e93ae7c737129ea427c068dc30e2da5" }
rustpython-parser = { git = "https://github.com/RustPython/Parser.git", rev = "48920a034e93ae7c737129ea427c068dc30e2da5" }
rustpython-ast = { git = "https://github.com/RustPython/Parser.git", rev = "48920a034e93ae7c737129ea427c068dc30e2da5" }
# rustpython-literal = { path = "../RustPython-parser/literal" }
# rustpython-compiler-core = { path = "../RustPython-parser/core" }
# rustpython-parser = { path = "../RustPython-parser/parser" }

20
compiler/src/lib.rs
View File

@@ -2,7 +2,7 @@ use rustpython_codegen::{compile, symboltable};
use rustpython_parser::ast::{fold::Fold, ConstantOptimizer};
pub use rustpython_codegen::compile::CompileOpts;
pub use rustpython_compiler_core::{BaseError as CompileErrorBody, CodeObject, Mode};
pub use rustpython_compiler_core::{CodeObject, Mode};
// these modules are out of repository. re-exporting them here for convenience.
pub use rustpython_codegen as codegen;
@@ -45,12 +45,7 @@ impl From<parser::ParseErrorType> for CompileErrorType {
}
}
pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;
fn error_from_parse(error: parser::ParseError, source: &str) -> CompileError {
let error: CompileErrorBody<parser::ParseErrorType> = error.into();
CompileError::from(error, source)
}
pub type CompileError = rustpython_compiler_core::BaseError<CompileErrorType>;
/// Compile a given source code into a bytecode object.
pub fn compile(
@@ -61,14 +56,14 @@ pub fn compile(
) -> Result<CodeObject, CompileError> {
let mut ast = match parser::parse(source, mode.into(), &source_path) {
Ok(x) => x,
Err(e) => return Err(error_from_parse(e, source)),
Err(e) => return Err(e.into()),
};
if opts.optimize > 0 {
ast = ConstantOptimizer::new()
.fold_mod(ast)
.unwrap_or_else(|e| match e {});
}
compile::compile_top(&ast, source_path, mode, opts).map_err(|e| CompileError::from(e, source))
compile::compile_top(&ast, source_path, mode, opts).map_err(|e| e.into())
}
pub fn compile_symtable(
@@ -76,16 +71,15 @@ pub fn compile_symtable(
mode: compile::Mode,
source_path: &str,
) -> Result<symboltable::SymbolTable, CompileError> {
let parse_err = |e| error_from_parse(e, source);
let res = match mode {
compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
let ast = parser::parse_program(source, source_path).map_err(parse_err)?;
let ast = parser::parse_program(source, source_path).map_err(|e| e.into())?;
symboltable::SymbolTable::scan_program(&ast)
}
compile::Mode::Eval => {
let expr = parser::parse_expression(source, source_path).map_err(parse_err)?;
let expr = parser::parse_expression(source, source_path).map_err(|e| e.into())?;
symboltable::SymbolTable::scan_expr(&expr)
}
};
res.map_err(|e| CompileError::from(e.into_codegen_error(source_path.to_owned()), source))
res.map_err(|e| e.into_codegen_error(source_path.to_owned()).into())
}

10
examples/hello_embed.rs
View File

@@ -3,14 +3,10 @@ use rustpython_vm as vm;
fn main() -> vm::PyResult<()> {
vm::Interpreter::without_stdlib(Default::default()).enter(|vm| {
let scope = vm.new_scope_with_builtins();
let source = r#"print("Hello World!")"#;
let code_obj = vm
.compile(
r#"print("Hello World!")"#,
vm::compiler::Mode::Exec,
"<embedded>".to_owned(),
)
.map_err(|err| vm.new_syntax_error(&err))?;
.compile(source, vm::compiler::Mode::Exec, "<embedded>".to_owned())
.map_err(|err| vm.new_syntax_error(&err, Some(source)))?;
vm.run_code_obj(code_obj, scope)?;

2
examples/mini_repl.rs
View File

@@ -65,7 +65,7 @@ def fib(n):
// (note that this is only the case when compiler::Mode::Single is passed to vm.compile)
match vm
.compile(&input, vm::compiler::Mode::Single, "<embedded>".to_owned())
.map_err(|err| vm.new_syntax_error(&err))
.map_err(|err| vm.new_syntax_error(&err, Some(&input)))
.and_then(|code_obj| vm.run_code_obj(code_obj, scope.clone()))
{
Ok(output) => {

20
src/shell.rs
View File

@@ -3,7 +3,7 @@ mod helper;
use rustpython_parser::{lexer::LexicalErrorType, ParseErrorType, Tok};
use rustpython_vm::{
builtins::PyBaseExceptionRef,
compiler::{self, CompileError, CompileErrorBody, CompileErrorType},
compiler::{self, CompileError, CompileErrorType},
readline::{Readline, ReadlineResult},
scope::Scope,
AsObject, PyResult, VirtualMachine,
@@ -36,19 +36,11 @@ fn shell_exec(
}
}
Err(CompileError {
body:
CompileErrorBody {
error: CompileErrorType::Parse(ParseErrorType::Lexical(LexicalErrorType::Eof)),
..
},
error: CompileErrorType::Parse(ParseErrorType::Lexical(LexicalErrorType::Eof)),
..
})
| Err(CompileError {
body:
CompileErrorBody {
error: CompileErrorType::Parse(ParseErrorType::Eof),
..
},
error: CompileErrorType::Parse(ParseErrorType::Eof),
..
}) => ShellExecResult::Continue,
Err(err) => {
@@ -57,13 +49,13 @@ fn shell_exec(
// since indentations errors on columns other than 0 should be ignored.
// if its an unrecognized token for dedent, set to false
let bad_error = match err.body.error {
let bad_error = match err.error {
CompileErrorType::Parse(ref p) => {
if matches!(
p,
ParseErrorType::Lexical(LexicalErrorType::IndentationError)
) {
continuing && err.body.location.column() != 0
continuing && err.location.column() != 0
} else {
!matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _))
}
@@ -73,7 +65,7 @@ fn shell_exec(
// If we are handling an error on an empty line or an error worthy of throwing
if empty_line_given || bad_error {
ShellExecResult::PyErr(vm.new_syntax_error(&err))
ShellExecResult::PyErr(vm.new_syntax_error(&err, Some(source)))
} else {
ShellExecResult::Continue
}

2
stdlib/src/dis.rs
View File

@@ -16,7 +16,7 @@ mod decl {
} else if let Ok(co_str) = PyStrRef::try_from_object(vm, obj.clone()) {
// String:
vm.compile(co_str.as_str(), compiler::Mode::Exec, "<dis>".to_owned())
.map_err(|err| vm.new_syntax_error(&err))?
.map_err(|err| vm.new_syntax_error(&err, Some(co_str.as_str())))?
} else {
PyRef::try_from_object(vm, obj)?
};

4
vm/src/compiler.rs
View File

@@ -26,8 +26,8 @@ mod error {
#[cfg(not(feature = "rustpython-compiler"))]
pub use error::{CompileError, CompileErrorType};
impl ToPyException for CompileError {
impl ToPyException for (CompileError, Option<&str>) {
fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef {
vm.new_syntax_error(self)
vm.new_syntax_error(&self.0, self.1)
}
}

2
vm/src/eval.rs
View File

@@ -6,7 +6,7 @@ pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str)
debug!("Code object: {:?}", bytecode);
vm.run_code_obj(bytecode, scope)
}
Err(err) => Err(vm.new_syntax_error(&err)),
Err(err) => Err(vm.new_syntax_error(&err, Some(source))),
}
}

2
vm/src/import.rs
View File

@@ -118,7 +118,7 @@ pub fn import_file(
file_path,
vm.compile_opts(),
)
.map_err(|err| vm.new_syntax_error(&err))?;
.map_err(|err| vm.new_syntax_error(&err, Some(&content)))?;
import_codeobj(vm, module_name, code, true)
}

5
vm/src/stdlib/ast.rs
View File

@@ -307,8 +307,7 @@ pub(crate) fn parse(
source: &str,
mode: parser::Mode,
) -> Result<PyObjectRef, CompileError> {
let top =
parser::parse(source, mode, "<unknown>").map_err(|err| CompileError::from(err, source))?;
let top = parser::parse(source, mode, "<unknown>").map_err(CompileError::from)?;
Ok(top.ast_to_object(vm))
}
@@ -322,7 +321,7 @@ pub(crate) fn compile(
let opts = vm.compile_opts();
let ast = Node::ast_from_object(vm, object)?;
let code = codegen::compile::compile_top(&ast, filename.to_owned(), mode, opts)
.map_err(|err| CompileError::from(err, "<unknown>").to_pyexception(vm))?; // FIXME source
.map_err(|err| (CompileError::from(err), None).to_pyexception(vm))?; // FIXME source
Ok(vm.ctx.new_code(code).into())
}

6
vm/src/stdlib/builtins.rs
View File

@@ -173,14 +173,14 @@ mod builtins {
.map_err(|err| vm.new_value_error(err.to_string()))?;
let code = vm
.compile(source, mode, args.filename.as_str().to_owned())
.map_err(|err| err.to_pyexception(vm))?;
.map_err(|err| (err, Some(source)).to_pyexception(vm))?;
Ok(code.into())
}
} else {
let mode = mode_str
.parse::<parser::Mode>()
.map_err(|err| vm.new_value_error(err.to_string()))?;
ast::parse(vm, source, mode).map_err(|e| e.to_pyexception(vm))
ast::parse(vm, source, mode).map_err(|e| (e, Some(source)).to_pyexception(vm))
}
}
}
@@ -300,7 +300,7 @@ mod builtins {
#[cfg(feature = "rustpython-compiler")]
Either::A(string) => vm
.compile(string.as_str(), mode, "<string>".to_owned())
.map_err(|err| vm.new_syntax_error(&err))?,
.map_err(|err| vm.new_syntax_error(&err, Some(string.as_str())))?,
#[cfg(not(feature = "rustpython-compiler"))]
Either::A(_) => return Err(vm.new_type_error(CODEGEN_NOT_SUPPORTED.to_owned())),
Either::B(code_obj) => code_obj,

2
vm/src/stdlib/symtable.rs
View File

@@ -23,7 +23,7 @@ mod symtable {
.map_err(|err| vm.new_value_error(err.to_string()))?;
let symtable = compiler::compile_symtable(source.as_str(), mode, filename.as_str())
.map_err(|err| vm.new_syntax_error(&err))?;
.map_err(|err| vm.new_syntax_error(&err, Some(source.as_str())))?;
let py_symbol_table = to_py_symbol_table(symtable);
Ok(py_symbol_table.into_ref(&vm.ctx))

4
vm/src/vm/compile.rs
View File

@@ -58,7 +58,7 @@ impl VirtualMachine {
pub fn run_code_string(&self, scope: Scope, source: &str, source_path: String) -> PyResult {
let code_obj = self
.compile(source, compiler::Mode::Exec, source_path.clone())
.map_err(|err| self.new_syntax_error(&err))?;
.map_err(|err| self.new_syntax_error(&err, Some(source)))?;
// trace!("Code object: {:?}", code_obj.borrow());
scope.globals.set_item(
identifier!(self, __file__),
@@ -71,7 +71,7 @@ impl VirtualMachine {
pub fn run_block_expr(&self, scope: Scope, source: &str) -> PyResult {
let code_obj = self
.compile(source, compiler::Mode::BlockExpr, "<embedded>".to_owned())
.map_err(|err| self.new_syntax_error(&err))?;
.map_err(|err| self.new_syntax_error(&err, Some(source)))?;
// trace!("Code object: {:?}", code_obj.borrow());
self.run_code_obj(code_obj, scope)
}

6
vm/src/vm/interpreter.rs
View File

@@ -14,10 +14,12 @@ use std::sync::atomic::Ordering;
/// use rustpython_vm::compiler::Mode;
/// Interpreter::without_stdlib(Default::default()).enter(|vm| {
/// let scope = vm.new_scope_with_builtins();
/// let code_obj = vm.compile(r#"print("Hello World!")"#,
/// let source = r#"print("Hello World!")"#;
/// let code_obj = vm.compile(
/// source,
/// Mode::Exec,
/// "<embedded>".to_owned(),
/// ).map_err(|err| vm.new_syntax_error(&err)).unwrap();
/// ).map_err(|err| vm.new_syntax_error(&err, Some(source))).unwrap();
/// vm.run_code_obj(code_obj, scope).unwrap();
/// });
/// ```

9
vm/src/vm/mod.rs
View File

@@ -848,13 +848,10 @@ fn test_nested_frozen() {
.enter(|vm| {
let scope = vm.new_scope_with_builtins();
let source = "from dir_module.dir_module_inner import value2";
let code_obj = vm
.compile(
"from dir_module.dir_module_inner import value2",
vm::compiler::Mode::Exec,
"<embedded>".to_owned(),
)
.map_err(|err| vm.new_syntax_error(&err))
.compile(source, vm::compiler::Mode::Exec, "<embedded>".to_owned())
.map_err(|err| vm.new_syntax_error(&err, Some(source)))
.unwrap();
if let Err(e) = vm.run_code_obj(code_obj, scope) {

44
vm/src/vm/vm_new.rs
View File

@@ -248,7 +248,11 @@ impl VirtualMachine {
}
#[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))]
pub fn new_syntax_error(&self, error: &crate::compiler::CompileError) -> PyBaseExceptionRef {
pub fn new_syntax_error(
&self,
error: &crate::compiler::CompileError,
source: Option<&str>,
) -> PyBaseExceptionRef {
let syntax_error_type = match &error.error {
#[cfg(feature = "rustpython-parser")]
crate::compiler::CompileErrorType::Parse(p) if p.is_indentation_error() => {
@@ -261,7 +265,40 @@ impl VirtualMachine {
_ => self.ctx.exceptions.syntax_error,
}
.to_owned();
let syntax_error = self.new_exception_msg(syntax_error_type, error.to_string());
fn get_statement(source: &str, loc: rustpython_compiler_core::Location) -> Option<String> {
if loc.column() == 0 || loc.row() == 0 {
return None;
}
let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
Some(line + "\n")
}
let statement = if let Some(source) = source {
get_statement(source, error.location)
} else {
None
};
fn fmt(
error: &crate::compiler::CompileError,
statement: Option<&str>,
f: &mut impl std::fmt::Write,
) -> std::fmt::Result {
let loc = error.location;
if let Some(ref stmt) = statement {
// visualize the error when location and statement are provided
loc.fmt_with(f, &error.error)?;
write!(f, "\n{stmt}{arrow:>pad$}", pad = loc.column(), arrow = "^")
} else {
loc.fmt_with(f, &error.error)
}
}
let mut msg = String::new();
fmt(error, statement.as_deref(), &mut msg).unwrap();
let syntax_error = self.new_exception_msg(syntax_error_type, msg);
let lineno = self.ctx.new_int(error.location.row());
let offset = self.ctx.new_int(error.location.column());
syntax_error
@@ -272,9 +309,10 @@ impl VirtualMachine {
.as_object()
.set_attr("offset", offset, self)
.unwrap();
syntax_error
.as_object()
.set_attr("text", error.statement.clone().to_pyobject(self), self)
.set_attr("text", statement.to_pyobject(self), self)
.unwrap();
syntax_error
.as_object()