Merge pull request #4121 from youknowone/compiler

more parser/codegen separation

.github/workflows/ci.yaml

@@ -15,9 +15,10 @@ concurrency:
 env:
   CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
   NON_WASM_PACKAGES: >-
-    -p rustpython-bytecode
     -p rustpython-common
+    -p rustpython-compiler-core
     -p rustpython-compiler
+    -p rustpython-codegen
     -p rustpython-parser
     -p rustpython-vm
     -p rustpython-stdlib

Cargo.lock

@@ -1735,23 +1735,8 @@ name = "rustpython-ast"
 version = "0.1.0"
 dependencies = [
  "num-bigint",
- "rustpython-bytecode",
  "rustpython-common",
-]
-
-[[package]]
-name = "rustpython-bytecode"
-version = "0.1.2"
-dependencies = [
- "bincode",
- "bitflags",
- "bstr",
- "itertools",
- "lz4_flex",
- "num-bigint",
- "num-complex",
- "serde",
- "static_assertions",
+ "rustpython-compiler-core",
 ]
 
 [[package]]
@@ -1766,8 +1751,9 @@ dependencies = [
  "num-complex",
  "num-traits",
  "rustpython-ast",
- "rustpython-bytecode",
+ "rustpython-compiler-core",
  "rustpython-parser",
+ "thiserror",
 ]
 
 [[package]]
@@ -1797,12 +1783,28 @@ dependencies = [
 name = "rustpython-compiler"
 version = "0.1.2"
 dependencies = [
- "rustpython-bytecode",
  "rustpython-codegen",
+ "rustpython-compiler-core",
  "rustpython-parser",
  "thiserror",
 ]
 
+[[package]]
+name = "rustpython-compiler-core"
+version = "0.1.2"
+dependencies = [
+ "bincode",
+ "bitflags",
+ "bstr",
+ "itertools",
+ "lz4_flex",
+ "num-bigint",
+ "num-complex",
+ "serde",
+ "static_assertions",
+ "thiserror",
+]
+
 [[package]]
 name = "rustpython-derive"
 version = "0.1.2"
@@ -1813,8 +1815,9 @@ dependencies = [
  "once_cell",
  "proc-macro2",
  "quote",
- "rustpython-bytecode",
+ "rustpython-codegen",
  "rustpython-compiler",
+ "rustpython-compiler-core",
  "rustpython-doc",
  "syn",
  "syn-ext",
@@ -1839,7 +1842,7 @@ dependencies = [
  "cranelift-module",
  "libffi",
  "num-traits",
- "rustpython-bytecode",
+ "rustpython-compiler-core",
  "rustpython-derive",
  "thiserror",
 ]
@@ -1860,6 +1863,8 @@ dependencies = [
  "phf",
  "phf_codegen",
  "rustpython-ast",
+ "rustpython-compiler-core",
+ "thiserror",
  "tiny-keccak",
  "unic-emoji-char",
  "unic-ucd-ident",
@@ -1871,7 +1876,7 @@ name = "rustpython-pylib"
 version = "0.1.0"
 dependencies = [
  "glob",
- "rustpython-bytecode",
+ "rustpython-compiler-core",
  "rustpython-derive",
 ]
 
@@ -1990,10 +1995,10 @@ dependencies = [
  "result-like",
  "rustc_version",
  "rustpython-ast",
- "rustpython-bytecode",
  "rustpython-codegen",
  "rustpython-common",
  "rustpython-compiler",
+ "rustpython-compiler-core",
  "rustpython-derive",
  "rustpython-jit",
  "rustpython-parser",

Cargo.toml

@@ -14,7 +14,7 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 [workspace]
 resolver = "2"
 members = [
-    "compiler", "compiler/ast", "compiler/bytecode", "compiler/codegen", "compiler/parser",
+    "compiler", "compiler/ast", "compiler/core", "compiler/codegen", "compiler/parser",
     ".", "common", "derive", "jit", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
 ]
 
@@ -33,28 +33,28 @@ ssl = ["rustpython-stdlib/ssl"]
 ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
 
 [dependencies]
-log = "0.4.16"
-env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
-clap = "2.34"
 rustpython-compiler = { path = "compiler", version = "0.1.1" }
 rustpython-parser = { path = "compiler/parser", version = "0.1.1" }
-rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
 rustpython-stdlib = {path = "stdlib", optional = true, default-features = false}
-dirs = { package = "dirs-next", version = "2.0.0" }
-num-traits = "0.2.14"
-cfg-if = "1.0.0"
-libc = "0.2.126"
+rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
 
+cfg-if = "1.0.0"
+clap = "2.34"
+dirs = { package = "dirs-next", version = "2.0.0" }
+env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
 flame = { version = "0.2.2", optional = true }
 flamescope = { version = "0.1.2", optional = true }
+libc = "0.2.126"
+log = "0.4.16"
+num-traits = "0.2.14"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 rustyline = "10.0.0"
 
 [dev-dependencies]
 cpython = "0.7.0"
-python3-sys = "0.7.0"
 criterion = "0.3.5"
+python3-sys = "0.7.0"
 
 [[bench]]
 name = "execution"

Lib/test/test_ast.py

@@ -760,8 +760,6 @@ class ASTHelpers_Test(unittest.TestCase):
         b = compile('foo(1 + 1)', '<unknown>', 'exec', ast.PyCF_ONLY_AST)
         self.assertEqual(ast.dump(a), ast.dump(b))
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_parse_in_error(self):
         try:
             1/0
@@ -1101,8 +1099,6 @@ Module(
         malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)])
         self.assertRaises(ValueError, ast.literal_eval, malformed)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_literal_eval_trailing_ws(self):
         self.assertEqual(ast.literal_eval("    -1"), -1)
         self.assertEqual(ast.literal_eval("\t\t-1"), -1)
@@ -1121,8 +1117,6 @@ Module(
         with self.assertRaisesRegex(ValueError, msg):
             ast.literal_eval(node)
 
-    # TODO: RUSTPYTHON
-    @unittest.expectedFailure
     def test_literal_eval_syntax_errors(self):
         with self.assertRaisesRegex(SyntaxError, "unexpected indent"):
             ast.literal_eval(r'''
@@ -1767,6 +1761,8 @@ class ASTValidatorTests(unittest.TestCase):
         ast.MatchMapping([], [], rest="_"),
     ]
 
+    # TODO: RUSTPYTHON
+    @unittest.expectedFailure
     def test_match_validation_pattern(self):
         name_x = ast.Name('x', ast.Load())
         for pattern in self._MATCH_PATTERNS:

benches/execution.rs

@@ -51,7 +51,7 @@ pub fn benchmark_file_execution(
 pub fn benchmark_file_parsing(group: &mut BenchmarkGroup<WallTime>, name: &str, contents: &str) {
     group.throughput(Throughput::Bytes(contents.len() as u64));
     group.bench_function(BenchmarkId::new("rustpython", name), |b| {
-        b.iter(|| parse_program(contents).unwrap())
+        b.iter(|| parse_program(contents, name).unwrap())
     });
     group.bench_function(BenchmarkId::new("cpython", name), |b| {
         let gil = cpython::Python::acquire_gil();

common/Cargo.toml

@@ -8,22 +8,22 @@ edition = "2021"
 threading = ["parking_lot"]
 
 [dependencies]
-lock_api = "0.4"
-parking_lot = { version = "0.12.0", optional = true }
-num-traits = "0.2"
-num-complex = "0.4.0"
-num-bigint = "0.4.2"
-lexical-parse-float = { version = "0.8.0", features = ["format"] }
-hexf-parse = "0.2.1"
-cfg-if = "1.0"
-once_cell = "1.4.1"
-siphasher = "0.3"
-rand = "0.8"
-volatile = "0.3"
-radium = "0.7"
-libc = "0.2.126"
 ascii = "1.0"
+cfg-if = "1.0"
+hexf-parse = "0.2.1"
+lexical-parse-float = { version = "0.8.0", features = ["format"] }
+libc = "0.2.126"
+lock_api = "0.4"
+num-bigint = "0.4.2"
+num-complex = "0.4.0"
+num-traits = "0.2"
+once_cell = "1.4.1"
+parking_lot = { version = "0.12.0", optional = true }
+radium = "0.7"
+rand = "0.8"
+siphasher = "0.3"
 unic-ucd-category = "0.9"
+volatile = "0.3"
 
 [target.'cfg(windows)'.dependencies]
 widestring = "0.5.1"

compiler/Cargo.toml

@@ -6,7 +6,8 @@ authors = ["RustPython Team"]
 edition = "2021"
 
 [dependencies]
-thiserror = "1.0"
+rustpython-compiler-core = { path = "core" }
 rustpython-codegen = { path = "codegen" }
 rustpython-parser = { path = "parser" }
-rustpython-bytecode = { path = "bytecode" }
+
+thiserror = "1.0"

compiler/ast/Cargo.toml

@@ -12,5 +12,5 @@ unparse = ["rustpython-common"]
 
 [dependencies]
 num-bigint = "0.4.3"
+rustpython-compiler-core = { path = "../core" }
 rustpython-common = { path = "../../common", optional = true }
-rustpython-bytecode = { path = "../bytecode"}

compiler/ast/asdl_rs.py

@@ -645,7 +645,7 @@ def write_ast_def(mod, typeinfo, f):
         #![allow(clippy::derive_partial_eq_without_eq)]
         
         pub use crate::constant::*;
-        pub use crate::location::Location;
+        pub use crate::Location;
 
         type Ident = String;
         \n

compiler/ast/src/ast_gen.rs

@@ -3,7 +3,7 @@
 #![allow(clippy::derive_partial_eq_without_eq)]
 
 pub use crate::constant::*;
-pub use crate::location::Location;
+pub use crate::Location;
 
 type Ident = String;
 

compiler/ast/src/constant.rs

@@ -1,5 +1,5 @@
 use num_bigint::BigInt;
-pub use rustpython_bytecode::ConversionFlag;
+pub use rustpython_compiler_core::ConversionFlag;
 
 #[derive(Debug, PartialEq)]
 pub enum Constant {

compiler/ast/src/fold_helpers.rs

@@ -1,5 +1,4 @@
-use crate::constant;
-use crate::fold::Fold;
+use crate::{constant, fold::Fold};
 
 pub(crate) trait Foldable<T, U> {
     type Mapped;

compiler/ast/src/lib.rs

@@ -3,11 +3,10 @@ mod constant;
 #[cfg(feature = "fold")]
 mod fold_helpers;
 mod impls;
-mod location;
 #[cfg(feature = "unparse")]
 mod unparse;
 
 pub use ast_gen::*;
-pub use location::Location;
+pub use rustpython_compiler_core::Location;
 
 pub type Suite<U = ()> = Vec<Stmt<U>>;

compiler/ast/src/location.rs

@@ -1,79 +0,0 @@
-//! Datatypes to support source location information.
-
-use std::fmt;
-
-/// A location somewhere in the sourcecode.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
-pub struct Location {
-    row: usize,
-    column: usize,
-}
-
-impl fmt::Display for Location {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "line {} column {}", self.row, self.column)
-    }
-}
-
-impl Location {
-    pub fn visualize<'a>(
-        &self,
-        line: &'a str,
-        desc: impl fmt::Display + 'a,
-    ) -> impl fmt::Display + 'a {
-        struct Visualize<'a, D: fmt::Display> {
-            loc: Location,
-            line: &'a str,
-            desc: D,
-        }
-        impl<D: fmt::Display> fmt::Display for Visualize<'_, D> {
-            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-                write!(
-                    f,
-                    "{}\n{}{arrow:>pad$}",
-                    self.desc,
-                    self.line,
-                    pad = self.loc.column,
-                    arrow = "^",
-                )
-            }
-        }
-        Visualize {
-            loc: *self,
-            line,
-            desc,
-        }
-    }
-}
-
-impl Location {
-    pub fn new(row: usize, column: usize) -> Self {
-        Location { row, column }
-    }
-
-    pub fn row(&self) -> usize {
-        self.row
-    }
-
-    pub fn column(&self) -> usize {
-        self.column
-    }
-
-    pub fn reset(&mut self) {
-        self.row = 1;
-        self.column = 1;
-    }
-
-    pub fn go_right(&mut self) {
-        self.column += 1;
-    }
-
-    pub fn go_left(&mut self) {
-        self.column -= 1;
-    }
-
-    pub fn newline(&mut self) {
-        self.row += 1;
-        self.column = 1;
-    }
-}

compiler/codegen/Cargo.toml

@@ -8,15 +8,18 @@ license = "MIT"
 edition = "2021"
 
 [dependencies]
+rustpython-ast = { path = "../ast", features = ["unparse"] }
+rustpython-compiler-core = { path = "../core", version = "0.1.1" }
+
+ahash = "0.7.6"
 indexmap = "1.8.1"
 itertools = "0.10.3"
-rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
-rustpython-ast = { path = "../ast", features = ["unparse"] }
+log = "0.4.16"
 num-complex = { version = "0.4.0", features = ["serde"] }
 num-traits = "0.2.14"
-log = "0.4.16"
-ahash = "0.7.6"
+thiserror = "1.0"
 
 [dev-dependencies]
 rustpython-parser = { path = "../parser" }
+
 insta = "1.14.0"

compiler/codegen/src/compile.rs

@@ -6,21 +6,21 @@
 //!   <https://github.com/micropython/micropython/blob/master/py/compile.c>
 
 use crate::{
-    error::{CompileError, CompileErrorType},
+    error::{CodegenError, CodegenErrorType},
     ir,
-    symboltable::{self, make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable},
+    symboltable::{self, SymbolScope, SymbolTable},
     IndexSet,
 };
 use itertools::Itertools;
 use num_complex::Complex64;
 use num_traits::ToPrimitive;
 use rustpython_ast as ast;
-use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction};
+use rustpython_compiler_core::{self as bytecode, CodeObject, ConstantData, Instruction, Location};
 use std::borrow::Cow;
 
-pub use crate::mode::Mode;
+pub use rustpython_compiler_core::Mode;
 
-type CompileResult<T> = Result<T, CompileError>;
+type CompileResult<T> = Result<T, CodegenError>;
 
 #[derive(PartialEq, Eq, Clone, Copy)]
 enum NameUsage {
@@ -64,7 +64,7 @@ struct Compiler {
     code_stack: Vec<ir::CodeInfo>,
     symbol_table_stack: Vec<SymbolTable>,
     source_path: String,
-    current_source_location: ast::Location,
+    current_source_location: Location,
     qualified_path: Vec<String>,
     done_with_future_stmts: bool,
     future_annotations: bool,
@@ -129,7 +129,7 @@ fn compile_impl<Ast: ?Sized>(
 ) -> CompileResult<CodeObject> {
     let symbol_table = match make_symbol_table(ast) {
         Ok(x) => x,
-        Err(e) => return Err(e.into_compile_error(source_path)),
+        Err(e) => return Err(e.into_codegen_error(source_path)),
     };
 
     let mut compiler = Compiler::new(opts, source_path, "<module>".to_owned());
@@ -149,7 +149,7 @@ pub fn compile_program(
         ast,
         source_path,
         opts,
-        make_symbol_table,
+        SymbolTable::scan_program,
         Compiler::compile_program,
     )
 }
@@ -164,7 +164,7 @@ pub fn compile_program_single(
         ast,
         source_path,
         opts,
-        make_symbol_table,
+        SymbolTable::scan_program,
         Compiler::compile_program_single,
     )
 }
@@ -178,7 +178,7 @@ pub fn compile_block_expression(
         ast,
         source_path,
         opts,
-        make_symbol_table,
+        SymbolTable::scan_program,
         Compiler::compile_block_expr,
     )
 }
@@ -192,7 +192,7 @@ pub fn compile_expression(
         ast,
         source_path,
         opts,
-        make_symbol_table_expr,
+        SymbolTable::scan_expr,
         Compiler::compile_eval,
     )
 }
@@ -220,7 +220,7 @@ impl Compiler {
             code_stack: vec![module_code],
             symbol_table_stack: Vec::new(),
             source_path,
-            current_source_location: ast::Location::default(),
+            current_source_location: Location::default(),
             qualified_path: Vec::new(),
             done_with_future_stmts: false,
             future_annotations: false,
@@ -234,11 +234,11 @@ impl Compiler {
         }
     }
 
-    fn error(&self, error: CompileErrorType) -> CompileError {
+    fn error(&self, error: CodegenErrorType) -> CodegenError {
         self.error_loc(error, self.current_source_location)
     }
-    fn error_loc(&self, error: CompileErrorType, location: ast::Location) -> CompileError {
-        CompileError {
+    fn error_loc(&self, error: CodegenErrorType, location: Location) -> CodegenError {
+        CodegenError {
             error,
             location,
             source_path: self.source_path.clone(),
@@ -455,7 +455,7 @@ impl Compiler {
             NameUsage::Delete if is_forbidden_name(name) => "cannot delete",
             _ => return Ok(()),
         };
-        Err(self.error(CompileErrorType::SyntaxError(format!("{} {}", msg, name))))
+        Err(self.error(CodegenErrorType::SyntaxError(format!("{} {}", msg, name))))
     }
 
     fn compile_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> {
@@ -580,7 +580,7 @@ impl Compiler {
                 let from_list = if import_star {
                     if self.ctx.in_func() {
                         return Err(self
-                            .error_loc(CompileErrorType::FunctionImportStar, statement.location));
+                            .error_loc(CodegenErrorType::FunctionImportStar, statement.location));
                     }
                     vec![ConstantData::Str {
                         value: "*".to_owned(),
@@ -769,7 +769,7 @@ impl Compiler {
                     self.emit(Instruction::Break { target: end });
                 }
                 None => {
-                    return Err(self.error_loc(CompileErrorType::InvalidBreak, statement.location));
+                    return Err(self.error_loc(CodegenErrorType::InvalidBreak, statement.location));
                 }
             },
             Continue => match self.ctx.loop_data {
@@ -778,13 +778,13 @@ impl Compiler {
                 }
                 None => {
                     return Err(
-                        self.error_loc(CompileErrorType::InvalidContinue, statement.location)
+                        self.error_loc(CodegenErrorType::InvalidContinue, statement.location)
                     );
                 }
             },
             Return { value } => {
                 if !self.ctx.in_func() {
-                    return Err(self.error_loc(CompileErrorType::InvalidReturn, statement.location));
+                    return Err(self.error_loc(CodegenErrorType::InvalidReturn, statement.location));
                 }
                 match value {
                     Some(v) => {
@@ -795,7 +795,7 @@ impl Compiler {
                                 .contains(bytecode::CodeFlags::IS_GENERATOR)
                         {
                             return Err(self.error_loc(
-                                CompileErrorType::AsyncReturnValue,
+                                CodegenErrorType::AsyncReturnValue,
                                 statement.location,
                             ));
                         }
@@ -857,9 +857,9 @@ impl Compiler {
                 }
             }
             ast::ExprKind::BinOp { .. } | ast::ExprKind::UnaryOp { .. } => {
-                return Err(self.error(CompileErrorType::Delete("expression")))
+                return Err(self.error(CodegenErrorType::Delete("expression")))
             }
-            _ => return Err(self.error(CompileErrorType::Delete(expression.node.name()))),
+            _ => return Err(self.error(CodegenErrorType::Delete(expression.node.name()))),
         }
         Ok(())
     }
@@ -922,7 +922,7 @@ impl Compiler {
             .chain(&args.kwonlyargs);
         for name in args_iter {
             if Compiler::is_forbidden_arg_name(&name.node.arg) {
-                return Err(self.error(CompileErrorType::SyntaxError(format!(
+                return Err(self.error(CodegenErrorType::SyntaxError(format!(
                     "cannot assign to {}",
                     &name.node.arg
                 ))));
@@ -1401,7 +1401,7 @@ impl Compiler {
         let (item, items) = if let Some(parts) = items.split_first() {
             parts
         } else {
-            return Err(self.error(CompileErrorType::EmptyWithItems));
+            return Err(self.error(CodegenErrorType::EmptyWithItems));
         };
 
         let final_block = {
@@ -1433,7 +1433,7 @@ impl Compiler {
 
         if items.is_empty() {
             if body.is_empty() {
-                return Err(self.error(CompileErrorType::EmptyWithBody));
+                return Err(self.error(CodegenErrorType::EmptyWithBody));
             }
             self.compile_statements(body)?;
         } else {
@@ -1529,7 +1529,7 @@ impl Compiler {
     ) -> CompileResult<()> {
         eprintln!("match subject: {subject:?}");
         eprintln!("match cases: {cases:?}");
-        Err(self.error(CompileErrorType::NotImplementedYet))
+        Err(self.error(CodegenErrorType::NotImplementedYet))
     }
 
     fn compile_chained_comparison(
@@ -1699,7 +1699,7 @@ impl Compiler {
                 for (i, element) in elts.iter().enumerate() {
                     if let ast::ExprKind::Starred { .. } = &element.node {
                         if seen_star {
-                            return Err(self.error(CompileErrorType::MultipleStarArgs));
+                            return Err(self.error(CodegenErrorType::MultipleStarArgs));
                         } else {
                             seen_star = true;
                             let before = i;
@@ -1707,7 +1707,7 @@ impl Compiler {
                             let (before, after) = (|| Some((before.to_u8()?, after.to_u8()?)))()
                                 .ok_or_else(|| {
                                     self.error_loc(
-                                        CompileErrorType::TooManyStarUnpack,
+                                        CodegenErrorType::TooManyStarUnpack,
                                         target.location,
                                     )
                                 })?;
@@ -1732,10 +1732,10 @@ impl Compiler {
             }
             _ => {
                 return Err(self.error(match target.node {
-                    ast::ExprKind::Starred { .. } => CompileErrorType::SyntaxError(
+                    ast::ExprKind::Starred { .. } => CodegenErrorType::SyntaxError(
                         "starred assignment target must be in a list or tuple".to_owned(),
                     ),
-                    _ => CompileErrorType::Assign(target.node.name()),
+                    _ => CodegenErrorType::Assign(target.node.name()),
                 }));
             }
         }
@@ -1776,7 +1776,7 @@ impl Compiler {
                 AugAssignKind::Attr { idx }
             }
             _ => {
-                return Err(self.error(CompileErrorType::Assign(target.node.name())));
+                return Err(self.error(CodegenErrorType::Assign(target.node.name())));
             }
         };
 
@@ -2049,7 +2049,7 @@ impl Compiler {
             }
             Yield { value } => {
                 if !self.ctx.in_func() {
-                    return Err(self.error(CompileErrorType::InvalidYield));
+                    return Err(self.error(CodegenErrorType::InvalidYield));
                 }
                 self.mark_generator();
                 match value {
@@ -2060,7 +2060,7 @@ impl Compiler {
             }
             Await { value } => {
                 if self.ctx.func != FunctionContext::AsyncFunction {
-                    return Err(self.error(CompileErrorType::InvalidAwait));
+                    return Err(self.error(CodegenErrorType::InvalidAwait));
                 }
                 self.compile_expression(value)?;
                 self.emit(Instruction::GetAwaitable);
@@ -2070,10 +2070,10 @@ impl Compiler {
             YieldFrom { value } => {
                 match self.ctx.func {
                     FunctionContext::NoFunction => {
-                        return Err(self.error(CompileErrorType::InvalidYieldFrom));
+                        return Err(self.error(CodegenErrorType::InvalidYieldFrom));
                     }
                     FunctionContext::AsyncFunction => {
-                        return Err(self.error(CompileErrorType::AsyncYieldFrom));
+                        return Err(self.error(CodegenErrorType::AsyncYieldFrom));
                     }
                     FunctionContext::Function => {}
                 }
@@ -2208,7 +2208,7 @@ impl Compiler {
                 })?;
             }
             Starred { .. } => {
-                return Err(self.error(CompileErrorType::InvalidStarExpr));
+                return Err(self.error(CodegenErrorType::InvalidStarExpr));
             }
             IfExp { test, body, orelse } => {
                 let else_block = self.new_block();
@@ -2417,8 +2417,8 @@ impl Compiler {
 
     fn compile_comprehension_element(&mut self, element: &ast::Expr) -> CompileResult<()> {
         self.compile_expression(element).map_err(|e| {
-            if let CompileErrorType::InvalidStarExpr = e.error {
-                self.error(CompileErrorType::SyntaxError(
+            if let CodegenErrorType::InvalidStarExpr = e.error {
+                self.error(CodegenErrorType::SyntaxError(
                     "iterable unpacking cannot be used in comprehension".to_owned(),
                 ))
             } else {
@@ -2547,9 +2547,9 @@ impl Compiler {
         Ok(())
     }
 
-    fn compile_future_features(&mut self, features: &[ast::Alias]) -> Result<(), CompileError> {
+    fn compile_future_features(&mut self, features: &[ast::Alias]) -> Result<(), CodegenError> {
         if self.done_with_future_stmts {
-            return Err(self.error(CompileErrorType::InvalidFuturePlacement));
+            return Err(self.error(CodegenErrorType::InvalidFuturePlacement));
         }
         for feature in features {
             match &*feature.node.name {
@@ -2559,7 +2559,7 @@ impl Compiler {
                 // "generator_stop" => {}
                 "annotations" => self.future_annotations = true,
                 other => {
-                    return Err(self.error(CompileErrorType::InvalidFutureFeature(other.to_owned())))
+                    return Err(self.error(CodegenErrorType::InvalidFutureFeature(other.to_owned())))
                 }
             }
         }
@@ -2617,7 +2617,7 @@ impl Compiler {
         code.current_block = block;
     }
 
-    fn set_source_location(&mut self, location: ast::Location) {
+    fn set_source_location(&mut self, location: Location) {
         self.current_source_location = location;
     }
 
@@ -2672,7 +2672,7 @@ fn try_get_constant_string(values: &[ast::Expr]) -> Option<String> {
     }
 }
 
-fn compile_location(location: &ast::Location) -> bytecode::Location {
+fn compile_location(location: &Location) -> bytecode::Location {
     bytecode::Location::new(location.row(), location.column())
 }
 
@@ -2697,8 +2697,8 @@ fn compile_constant(value: &ast::Constant) -> ConstantData {
 #[cfg(test)]
 mod tests {
     use super::{CompileOpts, Compiler};
-    use crate::symboltable::make_symbol_table;
-    use rustpython_bytecode::CodeObject;
+    use crate::symboltable::SymbolTable;
+    use rustpython_compiler_core::CodeObject;
     use rustpython_parser::parser;
 
     fn compile_exec(source: &str) -> CodeObject {
@@ -2707,8 +2707,8 @@ mod tests {
             "source_path".to_owned(),
             "<module>".to_owned(),
         );
-        let ast = parser::parse_program(source).unwrap();
-        let symbol_scope = make_symbol_table(&ast).unwrap();
+        let ast = parser::parse_program(source, "<test>").unwrap();
+        let symbol_scope = SymbolTable::scan_program(&ast).unwrap();
         compiler.compile_program(&ast, symbol_scope).unwrap();
         compiler.pop_code_object()
     }

compiler/codegen/src/error.rs

@@ -1,17 +1,10 @@
-use rustpython_ast::Location;
+use std::fmt;
 
-use std::{error::Error, fmt};
+pub type CodegenError = rustpython_compiler_core::BaseError<CodegenErrorType>;
 
-#[derive(Debug)]
-pub struct CompileError {
-    pub error: CompileErrorType,
-    pub location: Location,
-    pub source_path: String,
-}
-
-#[derive(Debug)]
+#[derive(Debug, thiserror::Error)]
 #[non_exhaustive]
-pub enum CompileErrorType {
+pub enum CodegenErrorType {
     /// Invalid assignment, cannot store value in target.
     Assign(&'static str),
     /// Invalid delete
@@ -40,62 +33,49 @@ pub enum CompileErrorType {
     NotImplementedYet, // RustPython marker for unimplemented features
 }
 
-impl fmt::Display for CompileErrorType {
+impl fmt::Display for CodegenErrorType {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        use CodegenErrorType::*;
         match self {
-            CompileErrorType::Assign(target) => write!(f, "cannot assign to {}", target),
-            CompileErrorType::Delete(target) => write!(f, "cannot delete {}", target),
-            CompileErrorType::SyntaxError(err) => write!(f, "{}", err.as_str()),
-            CompileErrorType::MultipleStarArgs => {
+            Assign(target) => write!(f, "cannot assign to {}", target),
+            Delete(target) => write!(f, "cannot delete {}", target),
+            SyntaxError(err) => write!(f, "{}", err.as_str()),
+            MultipleStarArgs => {
                 write!(f, "two starred expressions in assignment")
             }
-            CompileErrorType::InvalidStarExpr => write!(f, "cannot use starred expression here"),
-            CompileErrorType::InvalidBreak => write!(f, "'break' outside loop"),
-            CompileErrorType::InvalidContinue => write!(f, "'continue' outside loop"),
-            CompileErrorType::InvalidReturn => write!(f, "'return' outside function"),
-            CompileErrorType::InvalidYield => write!(f, "'yield' outside function"),
-            CompileErrorType::InvalidYieldFrom => write!(f, "'yield from' outside function"),
-            CompileErrorType::InvalidAwait => write!(f, "'await' outside async function"),
-            CompileErrorType::AsyncYieldFrom => write!(f, "'yield from' inside async function"),
-            CompileErrorType::AsyncReturnValue => {
+            InvalidStarExpr => write!(f, "cannot use starred expression here"),
+            InvalidBreak => write!(f, "'break' outside loop"),
+            InvalidContinue => write!(f, "'continue' outside loop"),
+            InvalidReturn => write!(f, "'return' outside function"),
+            InvalidYield => write!(f, "'yield' outside function"),
+            InvalidYieldFrom => write!(f, "'yield from' outside function"),
+            InvalidAwait => write!(f, "'await' outside async function"),
+            AsyncYieldFrom => write!(f, "'yield from' inside async function"),
+            AsyncReturnValue => {
                 write!(f, "'return' with value inside async generator")
             }
-            CompileErrorType::InvalidFuturePlacement => write!(
+            InvalidFuturePlacement => write!(
                 f,
                 "from __future__ imports must occur at the beginning of the file"
             ),
-            CompileErrorType::InvalidFutureFeature(feat) => {
+            InvalidFutureFeature(feat) => {
                 write!(f, "future feature {} is not defined", feat)
             }
-            CompileErrorType::FunctionImportStar => {
+            FunctionImportStar => {
                 write!(f, "import * only allowed at module level")
             }
-            CompileErrorType::TooManyStarUnpack => {
+            TooManyStarUnpack => {
                 write!(f, "too many expressions in star-unpacking assignment")
             }
-            CompileErrorType::EmptyWithItems => {
+            EmptyWithItems => {
                 write!(f, "empty items on With")
             }
-            CompileErrorType::EmptyWithBody => {
+            EmptyWithBody => {
                 write!(f, "empty body on With")
             }
-            CompileErrorType::NotImplementedYet => {
+            NotImplementedYet => {
                 write!(f, "RustPython does not implement this feature yet")
             }
         }
     }
 }
-
-impl Error for CompileErrorType {}
-
-impl fmt::Display for CompileError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{} at {}", self.error, self.location)
-    }
-}
-
-impl Error for CompileError {
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        None
-    }
-}

compiler/codegen/src/ir.rs

@@ -1,5 +1,5 @@
 use crate::IndexSet;
-use rustpython_bytecode::{CodeFlags, CodeObject, ConstantData, Instruction, Label, Location};
+use rustpython_compiler_core::{CodeFlags, CodeObject, ConstantData, Instruction, Label, Location};
 
 pub type BlockIdx = Label;
 

compiler/codegen/src/lib.rs

@@ -11,5 +11,6 @@ type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>;
 pub mod compile;
 pub mod error;
 pub mod ir;
-pub mod mode;
 pub mod symboltable;
+
+pub use compile::CompileOpts;

compiler/codegen/src/symboltable.rs

@@ -8,24 +8,13 @@ Inspirational file: https://github.com/python/cpython/blob/main/Python/symtable.
 */
 
 use crate::{
-    error::{CompileError, CompileErrorType},
+    error::{CodegenError, CodegenErrorType},
     IndexMap,
 };
-use rustpython_ast::{self as ast, Location};
+use rustpython_ast as ast;
+use rustpython_compiler_core::Location;
 use std::{borrow::Cow, fmt};
 
-pub fn make_symbol_table(program: &[ast::Stmt]) -> SymbolTableResult<SymbolTable> {
-    let mut builder = SymbolTableBuilder::new();
-    builder.scan_statements(program)?;
-    builder.finish()
-}
-
-pub fn make_symbol_table_expr(expr: &ast::Expr) -> SymbolTableResult<SymbolTable> {
-    let mut builder = SymbolTableBuilder::new();
-    builder.scan_expression(expr, ExpressionContext::Load)?;
-    builder.finish()
-}
-
 /// Captures all symbols in the current scope, and has a list of subscopes in this scope.
 #[derive(Clone)]
 pub struct SymbolTable {
@@ -60,6 +49,18 @@ impl SymbolTable {
             sub_tables: vec![],
         }
     }
+
+    pub fn scan_program(program: &[ast::Stmt]) -> SymbolTableResult<Self> {
+        let mut builder = SymbolTableBuilder::new();
+        builder.scan_statements(program)?;
+        builder.finish()
+    }
+
+    pub fn scan_expr(expr: &ast::Expr) -> SymbolTableResult<Self> {
+        let mut builder = SymbolTableBuilder::new();
+        builder.scan_expression(expr, ExpressionContext::Load)?;
+        builder.finish()
+    }
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -168,9 +169,9 @@ pub struct SymbolTableError {
 }
 
 impl SymbolTableError {
-    pub fn into_compile_error(self, source_path: String) -> CompileError {
-        CompileError {
-            error: CompileErrorType::SyntaxError(self.error),
+    pub fn into_codegen_error(self, source_path: String) -> CodegenError {
+        CodegenError {
+            error: CodegenErrorType::SyntaxError(self.error),
             location: self.location,
             source_path,
         }

compiler/core/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "rustpython-bytecode"
+name = "rustpython-compiler-core"
 description = "RustPython specific bytecode."
 version = "0.1.2"
 authors = ["RustPython Team"]
@@ -7,14 +7,14 @@ edition = "2021"
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
-
 [dependencies]
 bincode = "1.3.3"
 bitflags = "1.3.2"
+bstr = "0.2.17"
+itertools = "0.10.3"
 lz4_flex = "0.9.2"
 num-bigint = { version = "0.4.3", features = ["serde"] }
 num-complex = { version = "0.4.0", features = ["serde"] }
 serde = { version = "1.0.136", features = ["derive"] }
-itertools = "0.10.3"
-bstr = "0.2.17"
 static_assertions = "1.1.0"
+thiserror = "1.0"

compiler/core/src/bytecode.rs

@@ -1,9 +1,7 @@
 //! Implement python as a virtual machine with bytecodes. This module
 //! implements bytecode structure.
 
-#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
-#![doc(html_root_url = "https://docs.rs/rustpython-bytecode/")]
-
+use crate::Location;
 use bitflags::bitflags;
 use bstr::ByteSlice;
 use itertools::Itertools;
@@ -12,38 +10,6 @@ use num_complex::Complex64;
 use serde::{Deserialize, Serialize};
 use std::{collections::BTreeSet, fmt, hash};
 
-/// Sourcecode location.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
-pub struct Location {
-    row: u32,
-    column: u32,
-}
-
-impl Location {
-    /// Creates a new Location object at the given row and column.
-    ///
-    /// # Example
-    /// ```
-    /// use rustpython_bytecode::Location;
-    /// let loc = Location::new(10, 10);
-    /// ```
-    pub fn new(row: usize, column: usize) -> Self {
-        let row = row.try_into().expect("Location::row over u32");
-        let column = column.try_into().expect("Location::column over u32");
-        Location { row, column }
-    }
-
-    /// Current row
-    pub fn row(&self) -> usize {
-        self.row as usize
-    }
-
-    /// Current column
-    pub fn column(&self) -> usize {
-        self.column as usize
-    }
-}
-
 pub trait Constant: Sized {
     type Name: AsRef<str>;
 
@@ -430,7 +396,7 @@ bitflags! {
 ///
 /// # Examples
 /// ```
-/// use rustpython_bytecode::ConstantData;
+/// use rustpython_compiler_core::ConstantData;
 /// let a = ConstantData::Float {value: 120f64};
 /// let b = ConstantData::Boolean {value: false};
 /// assert_ne!(a, b);
@@ -596,8 +562,8 @@ pub enum TestOperator {
 /// # Examples
 ///
 /// ```
-/// use rustpython_bytecode::Instruction::BinaryOperation;
-/// use rustpython_bytecode::BinaryOperator::Add;
+/// use rustpython_compiler_core::Instruction::BinaryOperation;
+/// use rustpython_compiler_core::BinaryOperator::Add;
 /// let op = BinaryOperation {op: Add};
 /// ```
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -940,7 +906,7 @@ impl Instruction {
     /// # Examples
     ///
     /// ```
-    /// use rustpython_bytecode::{Instruction, Label};
+    /// use rustpython_compiler_core::{Instruction, Label};
     /// let label = Label(0xF);
     /// let jump_inst = Instruction::Jump {target: label};
     /// assert!(jump_inst.unconditional_branch())
@@ -957,7 +923,7 @@ impl Instruction {
     /// # Examples
     ///
     /// ```
-    /// use rustpython_bytecode::{Instruction, Label, UnaryOperator};
+    /// use rustpython_compiler_core::{Instruction, Label, UnaryOperator};
     /// let jump_instruction = Instruction::Jump {target: Label(0xF)};
     /// let invert_instruction = Instruction::UnaryOperation {op: UnaryOperator::Invert};
     /// assert_eq!(jump_instruction.stack_effect(true), 0);

compiler/core/src/error.rs

@@ -0,0 +1,102 @@
+use crate::Location;
+use std::fmt::Display;
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct BaseError<T> {
+    pub error: T,
+    pub location: Location,
+    pub source_path: String,
+}
+
+impl<T> std::ops::Deref for BaseError<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.error
+    }
+}
+
+impl<T> std::error::Error for BaseError<T> where T: std::fmt::Display + std::fmt::Debug {}
+
+impl<T> Display for BaseError<T>
+where
+    T: std::fmt::Display,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        self.location.fmt_with(f, &self.error)
+    }
+}
+
+impl<T> BaseError<T> {
+    pub fn error(self) -> T {
+        self.error
+    }
+
+    pub fn from<U>(obj: BaseError<U>) -> Self
+    where
+        U: Into<T>,
+    {
+        Self {
+            error: obj.error.into(),
+            location: obj.location,
+            source_path: obj.source_path,
+        }
+    }
+
+    pub fn into<U>(self) -> BaseError<U>
+    where
+        T: Into<U>,
+    {
+        BaseError::from(self)
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub struct CompileError<T> {
+    pub body: BaseError<T>,
+    pub statement: Option<String>,
+}
+
+impl<T> std::ops::Deref for CompileError<T> {
+    type Target = BaseError<T>;
+    fn deref(&self) -> &Self::Target {
+        &self.body
+    }
+}
+
+impl<T> std::fmt::Display for CompileError<T>
+where
+    T: std::fmt::Display,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        let loc = self.location;
+        if let Some(ref stmt) = self.statement {
+            // visualize the error when location and statement are provided
+            loc.fmt_with(f, &self.error)?;
+            write!(f, "\n{stmt}{arrow:>pad$}", pad = loc.column(), arrow = "^")
+        } else {
+            loc.fmt_with(f, &self.error)
+        }
+    }
+}
+
+impl<T> CompileError<T> {
+    pub fn from<U>(error: BaseError<U>, source: &str) -> Self
+    where
+        T: From<U>,
+    {
+        let statement = get_statement(source, error.location);
+        CompileError {
+            body: error.into(),
+            statement,
+        }
+    }
+}
+
+fn get_statement(source: &str, loc: Location) -> Option<String> {
+    if loc.column() == 0 || loc.row() == 0 {
+        return None;
+    }
+    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
+    Some(line + "\n")
+}

compiler/core/src/lib.rs

@@ -0,0 +1,12 @@
+#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
+#![doc(html_root_url = "https://docs.rs/rustpython-compiler-core/")]
+
+mod bytecode;
+mod error;
+mod location;
+mod mode;
+
+pub use bytecode::*;
+pub use error::{BaseError, CompileError};
+pub use location::Location;
+pub use mode::Mode;

compiler/core/src/location.rs

@@ -0,0 +1,61 @@
+use serde::{Deserialize, Serialize};
+
+/// Sourcecode location.
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Location {
+    pub(super) row: u32,
+    pub(super) column: u32,
+}
+
+impl Location {
+    pub fn fmt_with(
+        &self,
+        f: &mut std::fmt::Formatter,
+        e: &impl std::fmt::Display,
+    ) -> std::fmt::Result {
+        write!(f, "{} at line {} column {}", e, self.row(), self.column())
+    }
+}
+
+impl Location {
+    /// Creates a new Location object at the given row and column.
+    ///
+    /// # Example
+    /// ```
+    /// use rustpython_compiler_core::Location;
+    /// let loc = Location::new(10, 10);
+    /// ```
+    pub fn new(row: usize, column: usize) -> Self {
+        let row = row.try_into().expect("Location::row over u32");
+        let column = column.try_into().expect("Location::column over u32");
+        Location { row, column }
+    }
+
+    /// Current row
+    pub fn row(&self) -> usize {
+        self.row as usize
+    }
+
+    /// Current column
+    pub fn column(&self) -> usize {
+        self.column as usize
+    }
+
+    pub fn reset(&mut self) {
+        self.row = 1;
+        self.column = 1;
+    }
+
+    pub fn go_right(&mut self) {
+        self.column += 1;
+    }
+
+    pub fn go_left(&mut self) {
+        self.column -= 1;
+    }
+
+    pub fn newline(&mut self) {
+        self.row += 1;
+        self.column = 1;
+    }
+}

compiler/parser/Cargo.toml

@@ -2,7 +2,7 @@
 name = "rustpython-parser"
 version = "0.1.2"
 description = "Parser for python code."
-authors = [ "RustPython Team" ]
+authors = ["RustPython Team"]
 build = "build.rs"
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
@@ -15,6 +15,9 @@ phf_codegen = "0.10"
 tiny-keccak = { version = "2", features = ["sha3"] }
 
 [dependencies]
+rustpython-ast = { path = "../ast" }
+rustpython-compiler-core = { path = "../core" }
+
 ahash = "0.7.6"
 itertools = "0.10.3"
 lalrpop-util = "0.19.8"
@@ -22,10 +25,10 @@ log = "0.4.16"
 num-bigint = "0.4.3"
 num-traits = "0.2.14"
 phf = "0.10.1"
-rustpython-ast = { path = "../ast" }
 unic-emoji-char = "0.9.0"
 unic-ucd-ident  = "0.9.0"
 unicode_names2 = "0.5.0"
+thiserror = "1.0"
 
 [dev-dependencies]
 insta = "1.14.0"

compiler/parser/src/error.rs

@@ -1,11 +1,8 @@
 //! Define internal parse error types
 //! The goal is to provide a matching and a safe error API, maksing errors from LALR
+
+use crate::{ast::Location, token::Tok};
 use lalrpop_util::ParseError as LalrpopError;
-
-use crate::ast::Location;
-use crate::token::Tok;
-
-use std::error::Error;
 use std::fmt;
 
 /// Represents an error during lexical scanning.
@@ -120,13 +117,9 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
 }
 
 /// Represents an error during parsing
-#[derive(Debug, PartialEq)]
-pub struct ParseError {
-    pub error: ParseErrorType,
-    pub location: Location,
-}
+pub type ParseError = rustpython_compiler_core::BaseError<ParseErrorType>;
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, thiserror::Error)]
 pub enum ParseErrorType {
     /// Parser encountered an unexpected end of input
     Eof,
@@ -141,42 +134,43 @@ pub enum ParseErrorType {
 }
 
 /// Convert `lalrpop_util::ParseError` to our internal type
-impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
-    fn from(err: LalrpopError<Location, Tok, LexicalError>) -> Self {
-        match err {
-            // TODO: Are there cases where this isn't an EOF?
-            LalrpopError::InvalidToken { location } => ParseError {
-                error: ParseErrorType::Eof,
-                location,
-            },
-            LalrpopError::ExtraToken { token } => ParseError {
-                error: ParseErrorType::ExtraToken(token.1),
+pub(crate) fn parse_error_from_lalrpop(
+    err: LalrpopError<Location, Tok, LexicalError>,
+    source_path: &str,
+) -> ParseError {
+    let source_path = source_path.to_owned();
+    match err {
+        // TODO: Are there cases where this isn't an EOF?
+        LalrpopError::InvalidToken { location } => ParseError {
+            error: ParseErrorType::Eof,
+            location,
+            source_path,
+        },
+        LalrpopError::ExtraToken { token } => ParseError {
+            error: ParseErrorType::ExtraToken(token.1),
+            location: token.0,
+            source_path,
+        },
+        LalrpopError::User { error } => ParseError {
+            error: ParseErrorType::Lexical(error.error),
+            location: error.location,
+            source_path,
+        },
+        LalrpopError::UnrecognizedToken { token, expected } => {
+            // Hacky, but it's how CPython does it. See PyParser_AddToken,
+            // in particular "Only one possible expected token" comment.
+            let expected = (expected.len() == 1).then(|| expected[0].clone());
+            ParseError {
+                error: ParseErrorType::UnrecognizedToken(token.1, expected),
                 location: token.0,
-            },
-            LalrpopError::User { error } => ParseError {
-                error: ParseErrorType::Lexical(error.error),
-                location: error.location,
-            },
-            LalrpopError::UnrecognizedToken { token, expected } => {
-                // Hacky, but it's how CPython does it. See PyParser_AddToken,
-                // in particular "Only one possible expected token" comment.
-                let expected = (expected.len() == 1).then(|| expected[0].clone());
-                ParseError {
-                    error: ParseErrorType::UnrecognizedToken(token.1, expected),
-                    location: token.0,
-                }
+                source_path,
             }
-            LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
-                error: ParseErrorType::Eof,
-                location,
-            },
         }
-    }
-}
-
-impl fmt::Display for ParseError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{} at {}", self.error, self.location)
+        LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
+            error: ParseErrorType::Eof,
+            location,
+            source_path,
+        },
     }
 }
 
@@ -200,8 +194,6 @@ impl fmt::Display for ParseErrorType {
     }
 }
 
-impl Error for ParseErrorType {}
-
 impl ParseErrorType {
     pub fn is_indentation_error(&self) -> bool {
         match self {
@@ -220,16 +212,3 @@ impl ParseErrorType {
         )
     }
 }
-
-impl std::ops::Deref for ParseError {
-    type Target = ParseErrorType;
-    fn deref(&self) -> &Self::Target {
-        &self.error
-    }
-}
-
-impl Error for ParseError {
-    fn source(&self) -> Option<&(dyn Error + 'static)> {
-        None
-    }
-}

compiler/parser/src/fstring.rs

@@ -293,7 +293,7 @@ impl<'a> FStringParser<'a> {
 
 fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
     let fstring_body = format!("({})", source);
-    parse_expression(&fstring_body)
+    parse_expression(&fstring_body, "<fstring>")
 }
 
 /// Parse an fstring from a string, located at a certain position in the sourcecode.

compiler/parser/src/function.rs

@@ -1,8 +1,7 @@
-use ahash::RandomState;
-use std::collections::HashSet;
-
 use crate::ast;
 use crate::error::{LexicalError, LexicalErrorType};
+use ahash::RandomState;
+use std::collections::HashSet;
 
 pub struct ArgumentList {
     pub args: Vec<ast::Expr>,

compiler/parser/src/lib.rs

@@ -11,7 +11,7 @@
 //! use rustpython_parser::{parser, ast};
 //!
 //! let python_source = "print('Hello world')";
-//! let python_ast = parser::parse_expression(python_source).unwrap();
+//! let python_ast = parser::parse_expression(python_source, "<embedded>").unwrap();
 //!
 //! ```
 

compiler/parser/src/mode.rs

@@ -17,6 +17,17 @@ impl Mode {
     }
 }
 
+impl From<rustpython_compiler_core::Mode> for Mode {
+    fn from(mode: rustpython_compiler_core::Mode) -> Self {
+        use rustpython_compiler_core::Mode as CompileMode;
+        match mode {
+            CompileMode::Exec => Self::Module,
+            CompileMode::Eval => Self::Expression,
+            CompileMode::Single | CompileMode::BlockExpr => Self::Interactive,
+        }
+    }
+}
+
 impl std::str::FromStr for Mode {
     type Err = ModeParseError;
     fn from_str(s: &str) -> Result<Self, ModeParseError> {

compiler/parser/src/parser.rs

@@ -5,13 +5,9 @@
 //! parse a whole program, a single statement, or a single
 //! expression.
 
-use std::iter;
-
-use crate::ast;
-use crate::error::ParseError;
-use crate::lexer;
 pub use crate::mode::Mode;
-use crate::python;
+use crate::{ast, error::ParseError, lexer, python};
+use std::iter;
 
 /*
  * Parse python code.
@@ -20,8 +16,8 @@ use crate::python;
  */
 
 /// Parse a full python program, containing usually multiple lines.
-pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
-    parse(source, Mode::Module).map(|top| match top {
+pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> {
+    parse(source, Mode::Module, source_path).map(|top| match top {
         ast::Mod::Module { body, .. } => body,
         _ => unreachable!(),
     })
@@ -33,7 +29,7 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
 /// ```
 /// extern crate num_bigint;
 /// use rustpython_parser::{parser, ast};
-/// let expr = parser::parse_expression("1 + 2").unwrap();
+/// let expr = parser::parse_expression("1 + 2", "<embedded>").unwrap();
 ///
 /// assert_eq!(
 ///     expr,
@@ -63,22 +59,22 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
 /// );
 ///
 /// ```
-pub fn parse_expression(source: &str) -> Result<ast::Expr, ParseError> {
-    parse(source, Mode::Expression).map(|top| match top {
+pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
+    parse(source, Mode::Expression, path).map(|top| match top {
         ast::Mod::Expression { body } => *body,
         _ => unreachable!(),
     })
 }
 
 // Parse a given source code
-pub fn parse(source: &str, mode: Mode) -> Result<ast::Mod, ParseError> {
+pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
     let lxr = lexer::make_tokenizer(source);
     let marker_token = (Default::default(), mode.to_marker(), Default::default());
     let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
 
     python::TopParser::new()
         .parse(tokenizer)
-        .map_err(ParseError::from)
+        .map_err(|e| crate::error::parse_error_from_lalrpop(e, source_path))
 }
 
 #[cfg(test)]
@@ -87,56 +83,56 @@ mod tests {
 
     #[test]
     fn test_parse_empty() {
-        let parse_ast = parse_program("").unwrap();
+        let parse_ast = parse_program("", "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_string() {
         let source = String::from("'Hello world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string() {
         let source = String::from("f'Hello world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_print_hello() {
         let source = String::from("print('Hello world')");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_print_2() {
         let source = String::from("print('Hello world', 2)");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_kwargs() {
         let source = String::from("my_func('positional', keyword=2)");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_if_elif_else() {
         let source = String::from("if 1: 10\nelif 2: 20\nelse: 30");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_lambda() {
         let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
-        let parse_ast = parse_program(source).unwrap();
+        let parse_ast = parse_program(source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
@@ -144,7 +140,7 @@ mod tests {
     fn test_parse_tuples() {
         let source = "a, b = 4, 5";
 
-        insta::assert_debug_snapshot!(parse_program(source).unwrap());
+        insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
     }
 
     #[test]
@@ -155,48 +151,48 @@ class Foo(A, B):
   pass
  def method_with_default(self, arg='default'):
   pass";
-        insta::assert_debug_snapshot!(parse_program(source).unwrap());
+        insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
     }
 
     #[test]
     fn test_parse_dict_comprehension() {
         let source = String::from("{x1: x2 for y in z}");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_list_comprehension() {
         let source = String::from("[x for y in z]");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_double_list_comprehension() {
         let source = String::from("[x for y, y2 in z for a in b if a < 5 if a > 10]");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_generator_comprehension() {
         let source = String::from("(x for y in z)");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_named_expression_generator_comprehension() {
         let source = String::from("(x := y + 1 for y in z)");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_if_else_generator_comprehension() {
         let source = String::from("(x if y else y for y in z)");
-        let parse_ast = parse_expression(&source).unwrap();
+        let parse_ast = parse_expression(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 }

compiler/parser/src/string.rs

@@ -88,56 +88,56 @@ mod tests {
     #[test]
     fn test_parse_string_concat() {
         let source = String::from("'Hello ' 'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_string_concat_1() {
         let source = String::from("'Hello ' u'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_string_concat_2() {
         let source = String::from("u'Hello ' 'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_1() {
         let source = String::from("'Hello ' f'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_2() {
         let source = String::from("'Hello ' f'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_f_string_concat_3() {
         let source = String::from("'Hello ' f'world{\"!\"}'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_f_string_concat_1() {
         let source = String::from("u'Hello ' f'world'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 
     #[test]
     fn test_parse_u_f_string_concat_2() {
         let source = String::from("u'Hello ' f'world' '!'");
-        let parse_ast = parse_program(&source).unwrap();
+        let parse_ast = parse_program(&source, "<test>").unwrap();
         insta::assert_debug_snapshot!(parse_ast);
     }
 }

compiler/src/lib.rs

@@ -1,74 +1,27 @@
-use rustpython_bytecode::CodeObject;
 use rustpython_codegen::{compile, symboltable};
+use rustpython_compiler_core::CodeObject;
 use rustpython_parser::{
-    ast::{fold::Fold, ConstantOptimizer, Location},
+    ast::{fold::Fold, ConstantOptimizer},
+    error::ParseErrorType,
     parser,
 };
-use std::fmt;
 
-pub use compile::{CompileOpts, Mode};
-pub use symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType};
+pub use rustpython_codegen::compile::CompileOpts;
+pub use rustpython_compiler_core::{BaseError as CompileErrorBody, Mode};
 
 #[derive(Debug, thiserror::Error)]
 pub enum CompileErrorType {
     #[error(transparent)]
-    Compile(#[from] rustpython_codegen::error::CompileErrorType),
+    Codegen(#[from] rustpython_codegen::error::CodegenErrorType),
     #[error(transparent)]
     Parse(#[from] rustpython_parser::error::ParseErrorType),
 }
 
-#[derive(Debug, thiserror::Error)]
-pub struct CompileError {
-    pub error: CompileErrorType,
-    pub statement: Option<String>,
-    pub source_path: String,
-    pub location: Location,
-}
+pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;
 
-impl fmt::Display for CompileError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let loc = self.location;
-        if let Some(ref stmt) = self.statement {
-            // visualize the error when location and statement are provided
-            write!(
-                f,
-                "{}",
-                loc.visualize(stmt, &format_args!("{} at {}", self.error, loc))
-            )
-        } else {
-            write!(f, "{} at {}", self.error, loc)
-        }
-    }
-}
-
-impl CompileError {
-    fn from_compile(error: rustpython_codegen::error::CompileError, source: &str) -> Self {
-        CompileError {
-            error: error.error.into(),
-            location: error.location,
-            source_path: error.source_path,
-            statement: get_statement(source, error.location),
-        }
-    }
-    fn from_parse(
-        error: rustpython_parser::error::ParseError,
-        source: &str,
-        source_path: String,
-    ) -> Self {
-        CompileError {
-            error: error.error.into(),
-            location: error.location,
-            source_path,
-            statement: get_statement(source, error.location),
-        }
-    }
-    fn from_symtable(
-        error: symboltable::SymbolTableError,
-        source: &str,
-        source_path: String,
-    ) -> Self {
-        Self::from_compile(error.into_compile_error(source_path), source)
-    }
+fn error_from_parse(error: rustpython_parser::error::ParseError, source: &str) -> CompileError {
+    let error: CompileErrorBody<ParseErrorType> = error.into();
+    CompileError::from(error, source)
 }
 
 /// Compile a given sourcecode into a bytecode object.
@@ -76,24 +29,18 @@ pub fn compile(
     source: &str,
     mode: compile::Mode,
     source_path: String,
-    opts: CompileOpts,
+    opts: compile::CompileOpts,
 ) -> Result<CodeObject, CompileError> {
-    let parser_mode = match mode {
-        compile::Mode::Exec => parser::Mode::Module,
-        compile::Mode::Eval => parser::Mode::Expression,
-        compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
-    };
-    let mut ast = match parser::parse(source, parser_mode) {
+    let mut ast = match parser::parse(source, mode.into(), &source_path) {
         Ok(x) => x,
-        Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
+        Err(e) => return Err(error_from_parse(e, source)),
     };
     if opts.optimize > 0 {
         ast = ConstantOptimizer::new()
             .fold_mod(ast)
             .unwrap_or_else(|e| match e {});
     }
-    compile::compile_top(&ast, source_path, mode, opts)
-        .map_err(|e| CompileError::from_compile(e, source))
+    compile::compile_top(&ast, source_path, mode, opts).map_err(|e| CompileError::from(e, source))
 }
 
 pub fn compile_symtable(
@@ -101,24 +48,16 @@ pub fn compile_symtable(
     mode: compile::Mode,
     source_path: &str,
 ) -> Result<symboltable::SymbolTable, CompileError> {
-    let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned());
+    let parse_err = |e| error_from_parse(e, source);
     let res = match mode {
         compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
-            let ast = parser::parse_program(source).map_err(parse_err)?;
-            symboltable::make_symbol_table(&ast)
+            let ast = parser::parse_program(source, source_path).map_err(parse_err)?;
+            symboltable::SymbolTable::scan_program(&ast)
         }
         compile::Mode::Eval => {
-            let expr = parser::parse_expression(source).map_err(parse_err)?;
-            symboltable::make_symbol_table_expr(&expr)
+            let expr = parser::parse_expression(source, source_path).map_err(parse_err)?;
+            symboltable::SymbolTable::scan_expr(&expr)
         }
     };
-    res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned()))
-}
-
-fn get_statement(source: &str, loc: Location) -> Option<String> {
-    if loc.column() == 0 || loc.row() == 0 {
-        return None;
-    }
-    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
-    Some(line + "\n")
+    res.map_err(|e| CompileError::from(e.into_codegen_error(source_path.to_owned()), source))
 }

derive/Cargo.toml

@@ -11,15 +11,17 @@ edition = "2021"
 proc-macro = true
 
 [dependencies]
-syn = { version = "1.0.91", features = ["full", "extra-traits"] }
-syn-ext = { version = "0.4.0", features = ["full"] }
-quote = "1.0.18"
-proc-macro2 = "1.0.37"
-rustpython-bytecode = { path = "../compiler/bytecode", version = "0.1.1" }
+rustpython-codegen = { path = "../compiler/codegen", version = "0.1.1" }
 rustpython-compiler = { path = "../compiler", version = "0.1.1" }
-maplit = "1.0.2"
-once_cell = "1.10.0"
-textwrap = { version = "0.15.0", default-features = false }
+rustpython-compiler-core = { path = "../compiler/core", version = "0.1.1" }
+rustpython-doc = { git = "https://github.com/RustPython/__doc__", branch = "main" }
+
 indexmap = "1.8.1"
 itertools = "0.10.3"
-rustpython-doc = { git = "https://github.com/RustPython/__doc__", branch = "main" }
+maplit = "1.0.2"
+once_cell = "1.10.0"
+proc-macro2 = "1.0.37"
+quote = "1.0.18"
+syn = { version = "1.0.91", features = ["full", "extra-traits"] }
+syn-ext = { version = "0.4.0", features = ["full"] }
+textwrap = { version = "0.15.0", default-features = false }

derive/src/compile_bytecode.rs

@@ -17,8 +17,9 @@ use crate::{extract_spans, Diagnostic};
 use once_cell::sync::Lazy;
 use proc_macro2::{Span, TokenStream};
 use quote::quote;
-use rustpython_bytecode::{CodeObject, FrozenModule};
-use rustpython_compiler as compile;
+use rustpython_codegen as codegen;
+use rustpython_compiler::compile;
+use rustpython_compiler_core::{CodeObject, FrozenModule, Mode};
 use std::{
     collections::HashMap,
     env, fs,
@@ -54,23 +55,21 @@ impl CompilationSource {
     fn compile_string<D: std::fmt::Display, F: FnOnce() -> D>(
         &self,
         source: &str,
-        mode: compile::Mode,
+        mode: Mode,
         module_name: String,
         origin: F,
     ) -> Result<CodeObject, Diagnostic> {
-        compile::compile(source, mode, module_name, compile::CompileOpts::default()).map_err(
-            |err| {
-                Diagnostic::spans_error(
-                    self.span,
-                    format!("Python compile error from {}: {}", origin(), err),
-                )
-            },
-        )
+        compile(source, mode, module_name, codegen::CompileOpts::default()).map_err(|err| {
+            Diagnostic::spans_error(
+                self.span,
+                format!("Python compile error from {}: {}", origin(), err),
+            )
+        })
     }
 
     fn compile(
         &self,
-        mode: compile::Mode,
+        mode: Mode,
         module_name: String,
     ) -> Result<HashMap<String, FrozenModule>, Diagnostic> {
         match &self.kind {
@@ -86,11 +85,7 @@ impl CompilationSource {
         }
     }
 
-    fn compile_single(
-        &self,
-        mode: compile::Mode,
-        module_name: String,
-    ) -> Result<CodeObject, Diagnostic> {
+    fn compile_single(&self, mode: Mode, module_name: String) -> Result<CodeObject, Diagnostic> {
         match &self.kind {
             CompilationSourceKind::File(rel_path) => {
                 let path = CARGO_MANIFEST_DIR.join(rel_path);
@@ -117,7 +112,7 @@ impl CompilationSource {
         &self,
         path: &Path,
         parent: String,
-        mode: compile::Mode,
+        mode: Mode,
     ) -> Result<HashMap<String, FrozenModule>, Diagnostic> {
         let mut code_map = HashMap::new();
         let paths = fs::read_dir(path)
@@ -310,7 +305,7 @@ impl PyCompileInput {
 
         Ok(PyCompileArgs {
             source,
-            mode: mode.unwrap_or(compile::Mode::Exec),
+            mode: mode.unwrap_or(Mode::Exec),
             module_name: module_name.unwrap_or_else(|| "frozen".to_owned()),
             crate_name: crate_name.unwrap_or_else(|| syn::parse_quote!(::rustpython_vm::bytecode)),
         })
@@ -351,7 +346,7 @@ impl Parse for PyCompileInput {
 
 struct PyCompileArgs {
     source: CompilationSource,
-    mode: compile::Mode,
+    mode: Mode,
     module_name: String,
     crate_name: syn::Path,
 }
@@ -381,7 +376,8 @@ pub fn impl_py_freeze(input: TokenStream) -> Result<TokenStream, Diagnostic> {
     let crate_name = args.crate_name;
     let code_map = args.source.compile(args.mode, args.module_name)?;
 
-    let data = rustpython_bytecode::frozen_lib::encode_lib(code_map.iter().map(|(k, v)| (&**k, v)));
+    let data =
+        rustpython_compiler_core::frozen_lib::encode_lib(code_map.iter().map(|(k, v)| (&**k, v)));
     let bytes = LitByteStr::new(&data, Span::call_site());
 
     let output = quote! {

examples/dis.rs

@@ -13,8 +13,7 @@ extern crate env_logger;
 extern crate log;
 
 use clap::{App, Arg};
-
-use rustpython_compiler as compile;
+use rustpython_compiler as compiler;
 use std::error::Error;
 use std::fs;
 use std::path::Path;
@@ -62,7 +61,7 @@ fn main() {
     let optimize = matches.occurrences_of("optimize") as u8;
     let scripts = matches.values_of_os("scripts").unwrap();
 
-    let opts = compile::CompileOpts {
+    let opts = compiler::CompileOpts {
         optimize,
         ..Default::default()
     };
@@ -81,12 +80,12 @@ fn main() {
 
 fn display_script(
     path: &Path,
-    mode: compile::Mode,
-    opts: compile::CompileOpts,
+    mode: compiler::Mode,
+    opts: compiler::CompileOpts,
     expand_codeobjects: bool,
 ) -> Result<(), Box<dyn Error>> {
     let source = fs::read_to_string(path)?;
-    let code = compile::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?;
+    let code = compiler::compile(&source, mode, path.to_string_lossy().into_owned(), opts)?;
     println!("{}:", path.display());
     if expand_codeobjects {
         println!("{}", code.display_expand_codeobjects());

examples/hello_embed.rs

@@ -7,7 +7,7 @@ fn main() -> vm::PyResult<()> {
         let code_obj = vm
             .compile(
                 r#"print("Hello World!")"#,
-                vm::compile::Mode::Exec,
+                vm::compiler::Mode::Exec,
                 "<embedded>".to_owned(),
             )
             .map_err(|err| vm.new_syntax_error(&err))?;

examples/mini_repl.rs

@@ -14,7 +14,7 @@ macro_rules! add_python_function {
     ( $scope:ident, $vm:ident, $src:literal $(,)? ) => {{
         // compile the code to bytecode
         let code = vm::py_compile!(source = $src);
-        // convert the rustpython_bytecode::CodeObject to a PyRef<PyCode>
+        // convert the rustpython_compiler_core::CodeObject to a PyRef<PyCode>
         let code = $vm.ctx.new_code(code);
 
         // run the python code in the scope to store the function
@@ -62,9 +62,9 @@ def fib(n):
             .expect("Failed to read line of input");
 
         // this line also automatically prints the output
-        // (note that this is only the case when compile::Mode::Single is passed to vm.compile)
+        // (note that this is only the case when compiler::Mode::Single is passed to vm.compile)
         match vm
-            .compile(&input, vm::compile::Mode::Single, "<embedded>".to_owned())
+            .compile(&input, vm::compiler::Mode::Single, "<embedded>".to_owned())
             .map_err(|err| vm.new_syntax_error(&err))
             .and_then(|code_obj| vm.run_code_obj(code_obj, scope.clone()))
         {

examples/parse_folder.rs

@@ -84,7 +84,8 @@ fn parse_python_file(filename: &Path) -> ParsedFile {
         },
         Ok(source) => {
             let num_lines = source.to_string().lines().count();
-            let result = parser::parse_program(&source).map_err(|e| e.to_string());
+            let result = parser::parse_program(&source, &filename.to_string_lossy())
+                .map_err(|e| e.to_string());
             ParsedFile {
                 // filename: Box::new(filename.to_path_buf()),
                 // code: source.to_string(),

jit/Cargo.toml

@@ -2,7 +2,7 @@
 name = "rustpython-jit"
 version = "0.1.2"
 description = "Experimental JIT(just in time) compiler for python code."
-authors = [ "RustPython Team" ]
+authors = ["RustPython Team"]
 repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 edition = "2021"
@@ -10,18 +10,20 @@ edition = "2021"
 autotests = false
 
 [dependencies]
+rustpython-compiler-core = { path = "../compiler/core", version = "0.1.2" }
+
 cranelift = "0.76.0"
-cranelift-module = "0.76.0"
 cranelift-jit = "0.76.0"
-num-traits = "0.2"
+cranelift-module = "0.76.0"
 libffi = "2.0.0"
-rustpython-bytecode = { path = "../compiler/bytecode", version = "0.1.2" }
+num-traits = "0.2"
 thiserror = "1.0"
 
 [dev-dependencies]
-approx = "0.5.1"
 rustpython-derive = { path = "../derive", version = "0.1.2" }
 
+approx = "0.5.1"
+
 [[test]]
 name = "integration"
 path = "tests/lib.rs"

jit/src/instructions.rs

@@ -1,6 +1,6 @@
 use cranelift::prelude::*;
 use num_traits::cast::ToPrimitive;
-use rustpython_bytecode::{
+use rustpython_compiler_core::{
     self as bytecode, BinaryOperator, BorrowedConstant, CodeObject, ComparisonOperator,
     Instruction, Label, UnaryOperator,
 };

jit/src/lib.rs

@@ -4,7 +4,7 @@ use cranelift::prelude::*;
 use cranelift_jit::{JITBuilder, JITModule};
 use cranelift_module::{FuncId, Linkage, Module, ModuleError};
 use instructions::FunctionCompiler;
-use rustpython_bytecode as bytecode;
+use rustpython_compiler_core as bytecode;
 use std::{fmt, mem::ManuallyDrop};
 
 #[derive(Debug, thiserror::Error)]

jit/tests/common.rs

@@ -1,4 +1,4 @@
-use rustpython_bytecode::{CodeObject, ConstantData, Instruction};
+use rustpython_compiler_core::{CodeObject, ConstantData, Instruction};
 use rustpython_jit::{CompiledCode, JitType};
 use std::collections::HashMap;
 
@@ -157,7 +157,7 @@ macro_rules! jit_function {
     ($func_name:ident => $($t:tt)*) => {
         {
             let code = rustpython_derive::py_compile!(
-                crate_name = "rustpython_bytecode",
+                crate_name = "rustpython_compiler_core",
                 source = $($t)*
             );
             let mut machine = $crate::common::StackMachine::new();

src/shell.rs

@@ -1,10 +1,10 @@
 mod helper;
 
 use rustpython_parser::error::{LexicalErrorType, ParseErrorType};
-use rustpython_vm::readline::{Readline, ReadlineResult};
 use rustpython_vm::{
     builtins::PyBaseExceptionRef,
-    compile::{self, CompileError, CompileErrorType},
+    compiler::{self, CompileError, CompileErrorBody, CompileErrorType},
+    readline::{Readline, ReadlineResult},
     scope::Scope,
     AsObject, PyResult, VirtualMachine,
 };
@@ -16,17 +16,25 @@ enum ShellExecResult {
 }
 
 fn shell_exec(vm: &VirtualMachine, source: &str, scope: Scope) -> ShellExecResult {
-    match vm.compile(source, compile::Mode::Single, "<stdin>".to_owned()) {
+    match vm.compile(source, compiler::Mode::Single, "<stdin>".to_owned()) {
         Ok(code) => match vm.run_code_obj(code, scope) {
             Ok(_val) => ShellExecResult::Ok,
             Err(err) => ShellExecResult::PyErr(err),
         },
         Err(CompileError {
-            error: CompileErrorType::Parse(ParseErrorType::Lexical(LexicalErrorType::Eof)),
+            body:
+                CompileErrorBody {
+                    error: CompileErrorType::Parse(ParseErrorType::Lexical(LexicalErrorType::Eof)),
+                    ..
+                },
             ..
         })
         | Err(CompileError {
-            error: CompileErrorType::Parse(ParseErrorType::Eof),
+            body:
+                CompileErrorBody {
+                    error: CompileErrorType::Parse(ParseErrorType::Eof),
+                    ..
+                },
             ..
         }) => ShellExecResult::Continue,
         Err(err) => ShellExecResult::PyErr(vm.new_syntax_error(&err)),

stdlib/Cargo.toml

@@ -5,7 +5,15 @@ edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+threading = ["rustpython-common/threading", "rustpython-vm/threading"]
+zlib = ["libz-sys", "flate2/zlib"]
+bz2 = ["bzip2"]
+ssl = ["openssl", "openssl-sys", "foreign-types-shared"]
+ssl-vendor = ["ssl", "openssl/vendored", "openssl-probe"]
+
 [dependencies]
+# rustpython crates
 rustpython-derive = { path = "../derive" }
 rustpython-vm = { path = "../vm" }
 rustpython-common = { path = "../common" }
@@ -79,14 +87,6 @@ page_size = "0.4"
 [target.'cfg(all(unix, not(target_os = "redox"), not(target_os = "ios")))'.dependencies]
 termios = "0.3.3"
 
-[features]
-threading = ["rustpython-common/threading", "rustpython-vm/threading"]
-
-zlib = ["libz-sys", "flate2/zlib"]
-bz2 = ["bzip2"]
-ssl = ["openssl", "openssl-sys", "foreign-types-shared"]
-ssl-vendor = ["ssl", "openssl/vendored", "openssl-probe"]
-
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 gethostname = "0.2.3"
 socket2 = { version = "0.4.4", features = ["all"] }

stdlib/src/dis.rs

@@ -5,7 +5,7 @@ mod decl {
     use crate::vm::{
         builtins::{PyCode, PyDictRef, PyStrRef},
         bytecode::CodeFlags,
-        compile, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine,
+        compiler, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine,
     };
 
     #[pyfunction]
@@ -15,7 +15,7 @@ mod decl {
             PyRef::try_from_object(vm, co)?
         } else if let Ok(co_str) = PyStrRef::try_from_object(vm, obj.clone()) {
             // String:
-            vm.compile(co_str.as_str(), compile::Mode::Exec, "<dis>".to_owned())
+            vm.compile(co_str.as_str(), compiler::Mode::Exec, "<dis>".to_owned())
                 .map_err(|err| vm.new_syntax_error(&err))?
         } else {
             PyRef::try_from_object(vm, obj)?

vm/Cargo.toml

@@ -17,9 +17,9 @@ flame-it = ["flame", "flamer"]
 freeze-stdlib = ["rustpython-pylib/freeze-stdlib"]
 jit = ["rustpython-jit"]
 threading = ["rustpython-common/threading"]
-compiler = ["parser", "codegen"]
+compiler = ["parser", "codegen", "rustpython-compiler"]
 ast = ["rustpython-ast"]
-codegen = ["rustpython-compiler", "rustpython-codegen", "ast"]
+codegen = ["rustpython-codegen", "ast"]
 parser = ["rustpython-parser", "ast"]
 
 [dependencies]
@@ -27,7 +27,7 @@ rustpython-compiler = { path = "../compiler", optional = true, version = "0.1.2"
 rustpython-ast = { path = "../compiler/ast", optional = true, version = "0.1" }
 rustpython-parser = { path = "../compiler/parser", optional = true, version = "0.1.2" }
 rustpython-codegen = { path = "../compiler/codegen", optional = true, version = "0.1.2" }
-rustpython-bytecode = { path = "../compiler/bytecode", version = "0.1.2" }
+rustpython-compiler-core = { path = "../compiler/core", version = "0.1.2" }
 rustpython-common = { path = "../common" }
 rustpython-derive = { path = "../derive", version = "0.1.2" }
 rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" }

vm/pylib-crate/Cargo.toml

@@ -12,8 +12,8 @@ include = ["Cargo.toml", "src/**/*.rs", "Lib/", "!Lib/**/test/", "!Lib/**/*.pyc"
 freeze-stdlib = []
 
 [dependencies]
+rustpython-compiler-core = { version = "0.1.2", path = "../../compiler/core" }
 rustpython-derive = { version = "0.1.2", path = "../../derive" }
-rustpython-bytecode = { version = "0.1.2", path = "../../compiler/bytecode" }
 
 [build-dependencies]
 glob = "0.3"

vm/pylib-crate/src/lib.rs

@@ -9,12 +9,12 @@ pub const LIB_PATH: &str = match option_env!("win_lib_path") {
     None => concat!(env!("CARGO_MANIFEST_DIR"), "/../../Lib"),
 };
 
-use rustpython_bytecode::FrozenModule;
+use rustpython_compiler_core::FrozenModule;
 
 pub fn frozen_builtins() -> impl Iterator<Item = (String, FrozenModule)> {
     rustpython_derive::py_freeze!(
         dir = "../Lib/python_builtins",
-        crate_name = "rustpython_bytecode"
+        crate_name = "rustpython_compiler_core"
     )
 }
 
@@ -22,11 +22,11 @@ pub fn frozen_builtins() -> impl Iterator<Item = (String, FrozenModule)> {
 pub fn frozen_core() -> impl Iterator<Item = (String, FrozenModule)> {
     rustpython_derive::py_freeze!(
         dir = "../Lib/core_modules",
-        crate_name = "rustpython_bytecode"
+        crate_name = "rustpython_compiler_core"
     )
 }
 
 #[cfg(feature = "freeze-stdlib")]
 pub fn frozen_stdlib() -> impl Iterator<Item = (String, FrozenModule)> {
-    rustpython_derive::py_freeze!(dir = "../../Lib", crate_name = "rustpython_bytecode")
+    rustpython_derive::py_freeze!(dir = "../../Lib", crate_name = "rustpython_compiler_core")
 }

vm/src/compiler.rs

@@ -0,0 +1,33 @@
+use crate::{builtins::PyBaseExceptionRef, convert::ToPyException, VirtualMachine};
+
+#[cfg(feature = "rustpython-codegen")]
+pub use rustpython_codegen::CompileOpts;
+#[cfg(feature = "rustpython-compiler")]
+pub use rustpython_compiler::*;
+pub use rustpython_compiler_core::Mode;
+
+#[cfg(not(feature = "rustpython-compiler"))]
+mod error {
+    #[cfg(all(feature = "rustpython-parser", feature = "rustpython-codegen"))]
+    panic!("Use --features=compiler to enable both parser and codegen");
+
+    #[derive(Debug, thiserror::Error)]
+    pub enum CompileErrorType {
+        #[cfg(feature = "rustpython-codegen")]
+        #[error(transparent)]
+        Codegen(#[from] rustpython_codegen::error::CodegenErrorType),
+        #[cfg(feature = "rustpython-parser")]
+        #[error(transparent)]
+        Parse(#[from] rustpython_parser::error::ParseErrorType),
+    }
+
+    pub type CompileError = rustpython_compiler_core::CompileError<CompileErrorType>;
+}
+#[cfg(not(feature = "rustpython-compiler"))]
+pub use error::{CompileError, CompileErrorType};
+
+impl ToPyException for CompileError {
+    fn to_pyexception(&self, vm: &VirtualMachine) -> PyBaseExceptionRef {
+        vm.new_syntax_error(self)
+    }
+}

vm/src/eval.rs

@@ -1,10 +1,7 @@
-use crate::compile;
-use crate::scope::Scope;
-use crate::PyResult;
-use crate::VirtualMachine;
+use crate::{compiler, scope::Scope, PyResult, VirtualMachine};
 
 pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) -> PyResult {
-    match vm.compile(source, compile::Mode::Eval, source_path.to_owned()) {
+    match vm.compile(source, compiler::Mode::Eval, source_path.to_owned()) {
         Ok(bytecode) => {
             debug!("Code object: {:?}", bytecode);
             vm.run_code_obj(bytecode, scope)

vm/src/import.rs

@@ -1,8 +1,6 @@
 /*
  * Import mechanics
  */
-#[cfg(feature = "rustpython-compiler")]
-use crate::compile;
 use crate::{
     builtins::{list, traceback::PyTraceback, PyBaseExceptionRef, PyCode},
     scope::Scope,
@@ -111,7 +109,12 @@ pub fn import_file(
     content: String,
 ) -> PyResult {
     let code = vm
-        .compile_with_opts(&content, compile::Mode::Exec, file_path, vm.compile_opts())
+        .compile_with_opts(
+            &content,
+            crate::compiler::Mode::Exec,
+            file_path,
+            vm.compile_opts(),
+        )
         .map_err(|err| vm.new_syntax_error(&err))?;
     import_codeobj(vm, module_name, code, true)
 }

vm/src/lib.rs

@@ -46,6 +46,7 @@ mod bytesinner;
 pub mod cformat;
 pub mod class;
 mod codecs;
+pub mod compiler;
 pub mod convert;
 mod coroutine;
 mod dictdatatype;
@@ -84,10 +85,8 @@ pub use self::object::{
 };
 pub use self::vm::{Context, Interpreter, Settings, VirtualMachine};
 
-pub use rustpython_bytecode as bytecode;
 pub use rustpython_common as common;
-#[cfg(feature = "rustpython-compiler")]
-pub use rustpython_compiler as compile;
+pub use rustpython_compiler_core as bytecode;
 
 #[doc(hidden)]
 pub mod __exports {

vm/src/stdlib/ast.rs

@@ -8,14 +8,16 @@ mod gen;
 use crate::{
     builtins::{self, PyStrRef, PyType},
     class::{PyClassImpl, StaticType},
+    compiler::CompileError,
+    convert::ToPyException,
     AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, TryFromObject,
     VirtualMachine,
 };
 use num_complex::Complex64;
 use num_traits::{ToPrimitive, Zero};
 use rustpython_ast as ast;
-#[cfg(feature = "rustpython-compiler")]
-use rustpython_compiler as compile;
+#[cfg(feature = "rustpython-codegen")]
+use rustpython_codegen as codegen;
 #[cfg(feature = "rustpython-parser")]
 use rustpython_parser::parser;
 
@@ -263,24 +265,27 @@ impl Node for ast::ConversionFlag {
 }
 
 #[cfg(feature = "rustpython-parser")]
-pub(crate) fn parse(vm: &VirtualMachine, source: &str, mode: parser::Mode) -> PyResult {
-    // TODO: use vm.new_syntax_error()
-    let top = parser::parse(source, mode).map_err(|err| vm.new_value_error(format!("{}", err)))?;
+pub(crate) fn parse(
+    vm: &VirtualMachine,
+    source: &str,
+    mode: parser::Mode,
+) -> Result<PyObjectRef, CompileError> {
+    let top =
+        parser::parse(source, mode, "<unknown>").map_err(|err| CompileError::from(err, source))?;
     Ok(top.ast_to_object(vm))
 }
 
-#[cfg(feature = "rustpython-compiler")]
+#[cfg(feature = "rustpython-codegen")]
 pub(crate) fn compile(
     vm: &VirtualMachine,
     object: PyObjectRef,
     filename: &str,
-    mode: compile::Mode,
+    mode: codegen::compile::Mode,
 ) -> PyResult {
     let opts = vm.compile_opts();
     let ast = Node::ast_from_object(vm, object)?;
-    let code = rustpython_codegen::compile::compile_top(&ast, filename.to_owned(), mode, opts)
-        // TODO: use vm.new_syntax_error()
-        .map_err(|err| vm.new_value_error(err.to_string()))?;
+    let code = codegen::compile::compile_top(&ast, filename.to_owned(), mode, opts)
+        .map_err(|err| CompileError::from(err, "<unknown>").to_pyexception(vm))?; // FIXME source
     Ok(vm.ctx.new_code(code).into())
 }
 

vm/src/stdlib/builtins.rs

@@ -8,8 +8,6 @@ use crate::{class::PyClassImpl, PyObjectRef, VirtualMachine};
 /// Noteworthy: None is the `nil' object; Ellipsis represents `...' in slices.
 #[pymodule]
 mod builtins {
-    #[cfg(feature = "rustpython-compiler")]
-    use crate::compile;
     use crate::{
         builtins::{
             asyncgenerator::PyAsyncGen,
@@ -21,11 +19,11 @@ mod builtins {
             PyByteArray, PyBytes, PyDictRef, PyStr, PyStrRef, PyTuple, PyTupleRef, PyType,
         },
         common::{hash::PyHash, str::to_ascii},
+        convert::ToPyException,
         format::call_object_format,
-        function::Either,
         function::{
             ArgBytesLike, ArgCallable, ArgIntoBool, ArgIterable, ArgMapping, ArgStrOrBytesLike,
-            FuncArgs, KwArgs, OptionalArg, OptionalOption, PosArgs, PyArithmeticValue,
+            Either, FuncArgs, KwArgs, OptionalArg, OptionalOption, PosArgs, PyArithmeticValue,
         },
         protocol::{PyIter, PyIterReturn},
         py_io,
@@ -36,6 +34,10 @@ mod builtins {
     };
     use num_traits::{Signed, ToPrimitive};
 
+    #[cfg(not(feature = "rustpython-compiler"))]
+    const CODEGEN_NOT_SUPPORTED: &str =
+        "can't compile() to bytecode when the `codegen` feature of rustpython is disabled";
+
     #[pyfunction]
     fn abs(x: PyObjectRef, vm: &VirtualMachine) -> PyResult {
         vm._abs(&x)
@@ -113,13 +115,9 @@ mod builtins {
         _feature_version: OptionalArg<i32>,
     }
 
-    #[cfg(feature = "rustpython-compiler")]
+    #[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))]
     #[pyfunction]
     fn compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult {
-        #[cfg(not(feature = "rustpython-ast"))]
-        {
-            Err(vm.new_value_error("can't use compile() when the `compiler` and `parser` features of rustpython are disabled".to_owned()))
-        }
         #[cfg(feature = "rustpython-ast")]
         {
             use crate::{class::PyClassImpl, stdlib::ast};
@@ -134,14 +132,14 @@ mod builtins {
                 .source
                 .fast_isinstance(&ast::AstNode::make_class(&vm.ctx))
             {
-                #[cfg(not(feature = "rustpython-compiler"))]
+                #[cfg(not(feature = "rustpython-codegen"))]
                 {
-                    return Err(vm.new_value_error("can't compile ast nodes when the `compiler` feature of rustpython is disabled"));
+                    return Err(vm.new_type_error(CODEGEN_NOT_SUPPORTED.to_owned()));
                 }
-                #[cfg(feature = "rustpython-compiler")]
+                #[cfg(feature = "rustpython-codegen")]
                 {
                     let mode = mode_str
-                        .parse::<compile::Mode>()
+                        .parse::<crate::compiler::Mode>()
                         .map_err(|err| vm.new_value_error(err.to_string()))?;
                     return ast::compile(vm, args.source, args.filename.as_str(), mode);
                 }
@@ -149,9 +147,9 @@ mod builtins {
 
             #[cfg(not(feature = "rustpython-parser"))]
             {
-                Err(vm.new_value_error(
-                    "can't compile() a string when the `parser` feature of rustpython is disabled",
-                ))
+                const PARSER_NOT_SUPPORTED: &str =
+        "can't compile() source code when the `parser` feature of rustpython is disabled";
+                Err(vm.new_type_error(PARSER_NOT_SUPPORTED.to_owned()))
             }
             #[cfg(feature = "rustpython-parser")]
             {
@@ -160,7 +158,7 @@ mod builtins {
                 use rustpython_parser::parser;
 
                 let source = Either::<PyStrRef, PyBytesRef>::try_from_object(vm, args.source)?;
-                // TODO: compile::compile should probably get bytes
+                // TODO: compiler::compile should probably get bytes
                 let source = match &source {
                     Either::A(string) => string.as_str(),
                     Either::B(bytes) => std::str::from_utf8(bytes)
@@ -172,23 +170,23 @@ mod builtins {
                 if (flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() {
                     #[cfg(not(feature = "rustpython-compiler"))]
                     {
-                        Err(vm.new_value_error("can't compile() a string to bytecode when the `compiler` feature of rustpython is disabled".to_owned()))
+                        Err(vm.new_value_error(CODEGEN_NOT_SUPPORTED.to_owned()))
                     }
                     #[cfg(feature = "rustpython-compiler")]
                     {
                         let mode = mode_str
-                            .parse::<compile::Mode>()
+                            .parse::<crate::compiler::Mode>()
                             .map_err(|err| vm.new_value_error(err.to_string()))?;
                         let code = vm
                             .compile(source, mode, args.filename.as_str().to_owned())
-                            .map_err(|err| vm.new_syntax_error(&err))?;
+                            .map_err(|err| err.to_pyexception(vm))?;
                         Ok(code.into())
                     }
                 } else {
                     let mode = mode_str
                         .parse::<parser::Mode>()
                         .map_err(|err| vm.new_value_error(err.to_string()))?;
-                    ast::parse(vm, source, mode)
+                    ast::parse(vm, source, mode).map_err(|e| e.to_pyexception(vm))
                 }
             }
         }
@@ -209,7 +207,6 @@ mod builtins {
         vm._divmod(&a, &b)
     }
 
-    #[cfg(feature = "rustpython-compiler")]
     #[derive(FromArgs)]
     struct ScopeArgs {
         #[pyarg(any, default)]
@@ -218,7 +215,6 @@ mod builtins {
         locals: Option<ArgMapping>,
     }
 
-    #[cfg(feature = "rustpython-compiler")]
     impl ScopeArgs {
         fn make_scope(self, vm: &VirtualMachine) -> PyResult<crate::scope::Scope> {
             let (globals, locals) = match self.globals {
@@ -251,7 +247,6 @@ mod builtins {
 
     /// Implements `eval`.
     /// See also: https://docs.python.org/3/library/functions.html#eval
-    #[cfg(feature = "rustpython-compiler")]
     #[pyfunction]
     fn eval(
         source: Either<ArgStrOrBytesLike, PyRef<crate::builtins::PyCode>>,
@@ -281,36 +276,37 @@ mod builtins {
             }
             Either::B(code) => Ok(Either::B(code)),
         }?;
-        run_code(vm, code, scope, compile::Mode::Eval, "eval")
+        run_code(vm, code, scope, crate::compiler::Mode::Eval, "eval")
     }
 
     /// Implements `exec`
     /// https://docs.python.org/3/library/functions.html#exec
-    #[cfg(feature = "rustpython-compiler")]
     #[pyfunction]
     fn exec(
         source: Either<PyStrRef, PyRef<crate::builtins::PyCode>>,
         scope: ScopeArgs,
         vm: &VirtualMachine,
     ) -> PyResult {
-        run_code(vm, source, scope, compile::Mode::Exec, "exec")
+        run_code(vm, source, scope, crate::compiler::Mode::Exec, "exec")
     }
 
-    #[cfg(feature = "rustpython-compiler")]
     fn run_code(
         vm: &VirtualMachine,
         source: Either<PyStrRef, PyRef<crate::builtins::PyCode>>,
         scope: ScopeArgs,
-        mode: compile::Mode,
+        #[allow(unused_variables)] mode: crate::compiler::Mode,
         func: &str,
     ) -> PyResult {
         let scope = scope.make_scope(vm)?;
 
         // Determine code object:
         let code_obj = match source {
+            #[cfg(feature = "rustpython-compiler")]
             Either::A(string) => vm
                 .compile(string.as_str(), mode, "<string>".to_owned())
                 .map_err(|err| vm.new_syntax_error(&err))?,
+            #[cfg(not(feature = "rustpython-compiler"))]
+            Either::A(_) => return Err(vm.new_type_error(CODEGEN_NOT_SUPPORTED.to_owned())),
             Either::B(code_obj) => code_obj,
         };
 

vm/src/stdlib/symtable.rs

@@ -3,10 +3,9 @@ pub(crate) use symtable::make_module;
 #[pymodule]
 mod symtable {
     use crate::{
-        builtins::PyStrRef,
-        compile::{self, Symbol, SymbolScope, SymbolTable, SymbolTableType},
-        PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
+        builtins::PyStrRef, compiler, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
     };
+    use rustpython_codegen::symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType};
     use std::fmt;
 
     /// symtable. Return top level SymbolTable.
@@ -20,10 +19,10 @@ mod symtable {
     ) -> PyResult<PySymbolTableRef> {
         let mode = mode
             .as_str()
-            .parse::<compile::Mode>()
+            .parse::<compiler::Mode>()
             .map_err(|err| vm.new_value_error(err.to_string()))?;
 
-        let symtable = compile::compile_symtable(source.as_str(), mode, filename.as_str())
+        let symtable = compiler::compile_symtable(source.as_str(), mode, filename.as_str())
             .map_err(|err| vm.new_syntax_error(&err))?;
 
         let py_symbol_table = to_py_symbol_table(symtable);

vm/src/vm/compile.rs

@@ -1,24 +1,16 @@
 use crate::{
     builtins::{PyCode, PyDictRef},
-    compile::{self, CompileError, CompileOpts},
+    compiler::{self, CompileError, CompileOpts},
     convert::TryFromObject,
     scope::Scope,
     AsObject, PyObjectRef, PyRef, PyResult, VirtualMachine,
 };
 
 impl VirtualMachine {
-    /// Returns a basic CompileOpts instance with options accurate to the vm. Used
-    /// as the CompileOpts for `vm.compile()`.
-    pub fn compile_opts(&self) -> CompileOpts {
-        CompileOpts {
-            optimize: self.state.settings.optimize,
-        }
-    }
-
     pub fn compile(
         &self,
         source: &str,
-        mode: compile::Mode,
+        mode: compiler::Mode,
         source_path: String,
     ) -> Result<PyRef<PyCode>, CompileError> {
         self.compile_with_opts(source, mode, source_path, self.compile_opts())
@@ -27,11 +19,11 @@ impl VirtualMachine {
     pub fn compile_with_opts(
         &self,
         source: &str,
-        mode: compile::Mode,
+        mode: compiler::Mode,
         source_path: String,
         opts: CompileOpts,
     ) -> Result<PyRef<PyCode>, CompileError> {
-        compile::compile(source, mode, source_path, opts).map(|code| self.ctx.new_code(code))
+        compiler::compile(source, mode, source_path, opts).map(|code| self.ctx.new_code(code))
     }
 
     pub fn run_script(&self, scope: Scope, path: &str) -> PyResult<()> {
@@ -68,7 +60,7 @@ impl VirtualMachine {
 
     pub fn run_code_string(&self, scope: Scope, source: &str, source_path: String) -> PyResult {
         let code_obj = self
-            .compile(source, crate::compile::Mode::Exec, source_path.clone())
+            .compile(source, compiler::Mode::Exec, source_path.clone())
             .map_err(|err| self.new_syntax_error(&err))?;
         // trace!("Code object: {:?}", code_obj.borrow());
         scope.globals.set_item(
@@ -81,11 +73,7 @@ impl VirtualMachine {
 
     pub fn run_block_expr(&self, scope: Scope, source: &str) -> PyResult {
         let code_obj = self
-            .compile(
-                source,
-                crate::compile::Mode::BlockExpr,
-                "<embedded>".to_owned(),
-            )
+            .compile(source, compiler::Mode::BlockExpr, "<embedded>".to_owned())
             .map_err(|err| self.new_syntax_error(&err))?;
         // trace!("Code object: {:?}", code_obj.borrow());
         self.run_code_obj(code_obj, scope)

vm/src/vm/interpreter.rs

@@ -11,7 +11,7 @@ use std::sync::atomic::Ordering;
 /// Runs a simple embedded hello world program.
 /// ```
 /// use rustpython_vm::Interpreter;
-/// use rustpython_vm::compile::Mode;
+/// use rustpython_vm::compiler::Mode;
 /// Interpreter::without_stdlib(Default::default()).enter(|vm| {
 ///     let scope = vm.new_scope_with_builtins();
 ///     let code_obj = vm.compile(r#"print("Hello World!")"#,

vm/src/vm/mod.rs

@@ -384,6 +384,15 @@ impl VirtualMachine {
         })
     }
 
+    /// Returns a basic CompileOpts instance with options accurate to the vm. Used
+    /// as the CompileOpts for `vm.compile()`.
+    #[cfg(feature = "rustpython-codegen")]
+    pub fn compile_opts(&self) -> crate::compiler::CompileOpts {
+        crate::compiler::CompileOpts {
+            optimize: self.state.settings.optimize,
+        }
+    }
+
     // To be called right before raising the recursion depth.
     fn check_recursive_call(&self, _where: &str) -> PyResult<()> {
         if self.recursion_depth.get() >= self.recursion_limit.get() {
@@ -794,7 +803,7 @@ fn test_nested_frozen() {
         let code_obj = vm
             .compile(
                 "from dir_module.dir_module_inner import value2",
-                vm::compile::Mode::Exec,
+                vm::compiler::Mode::Exec,
                 "<embedded>".to_owned(),
             )
             .map_err(|err| vm.new_syntax_error(&err))

vm/src/vm/vm_new.rs

@@ -1,5 +1,3 @@
-#[cfg(feature = "rustpython-compiler")]
-use crate::compile::{CompileError, CompileErrorType};
 use crate::{
     builtins::{
         pystr::IntoPyStrRef,
@@ -209,13 +207,17 @@ impl VirtualMachine {
         self.new_exception_msg(overflow_error, msg)
     }
 
-    #[cfg(feature = "rustpython-compiler")]
-    pub fn new_syntax_error(&self, error: &CompileError) -> PyBaseExceptionRef {
+    #[cfg(any(feature = "rustpython-parser", feature = "rustpython-codegen"))]
+    pub fn new_syntax_error(&self, error: &crate::compiler::CompileError) -> PyBaseExceptionRef {
         let syntax_error_type = match &error.error {
-            CompileErrorType::Parse(p) if p.is_indentation_error() => {
+            #[cfg(feature = "rustpython-parser")]
+            crate::compiler::CompileErrorType::Parse(p) if p.is_indentation_error() => {
                 self.ctx.exceptions.indentation_error
             }
-            CompileErrorType::Parse(p) if p.is_tab_error() => self.ctx.exceptions.tab_error,
+            #[cfg(feature = "rustpython-parser")]
+            crate::compiler::CompileErrorType::Parse(p) if p.is_tab_error() => {
+                self.ctx.exceptions.tab_error
+            }
             _ => self.ctx.exceptions.syntax_error,
         }
         .to_owned();

wasm/lib/Cargo.toml

@@ -16,23 +16,21 @@ freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
 no-start-func = []
 
 [dependencies]
-rustpython-parser = { path = "../../compiler/parser" }
 rustpython-common = { path = "../../common" }
+rustpython-parser = { path = "../../compiler/parser" }
+rustpython-stdlib = { path = "../../stdlib", default-features = false, optional = true }
 # make sure no threading! otherwise wasm build will fail
 rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings"] }
-rustpython-stdlib = { path = "../../stdlib", default-features = false, optional = true }
-wasm-bindgen = "0.2"
-wasm-bindgen-futures = "0.4"
-serde-wasm-bindgen = "0.3.1"
-serde = "1.0"
-js-sys = "0.3"
+
 console_error_panic_hook = "0.1"
+js-sys = "0.3"
 # make parking_lot use wasm-bingden for instant
 parking_lot = { version = "0.12.0" }
-
-[dependencies.web-sys]
-version = "0.3"
-features = [
+serde = "1.0"
+serde-wasm-bindgen = "0.3.1"
+wasm-bindgen = "0.2"
+wasm-bindgen-futures = "0.4"
+web-sys = { version = "0.3", features = [
     "console",
     "Document",
     "Element",
@@ -41,7 +39,7 @@ features = [
     "Request",
     "RequestInit",
     "Response"
-]
+] }
 
 [package.metadata.wasm-pack.profile.release]
 wasm-opt = false#["-O1"]

wasm/lib/src/convert.rs

@@ -4,7 +4,7 @@ use js_sys::{Array, ArrayBuffer, Object, Promise, Reflect, SyntaxError, Uint8Arr
 use rustpython_parser::error::ParseErrorType;
 use rustpython_vm::{
     builtins::PyBaseExceptionRef,
-    compile::{CompileError, CompileErrorType},
+    compiler::{CompileError, CompileErrorType},
     exceptions,
     function::{ArgBytesLike, FuncArgs},
     py_serde, AsObject, PyObjectRef, PyPayload, PyResult, TryFromBorrowedObject, VirtualMachine,

wasm/lib/src/lib.rs

@@ -45,7 +45,7 @@ pub fn _setup_console_error() {
 pub mod eval {
     use crate::vm_class::VMStore;
     use js_sys::{Object, Reflect, TypeError};
-    use rustpython_vm::compile::Mode;
+    use rustpython_vm::compiler::Mode;
     use wasm_bindgen::prelude::*;
 
     const PY_EVAL_VM_ID: &str = "__py_eval_vm";

wasm/lib/src/vm_class.rs

@@ -5,10 +5,8 @@ use crate::{
 };
 use js_sys::{Object, TypeError};
 use rustpython_vm::{
-    builtins::PyWeak,
-    compile::{self, Mode},
-    scope::Scope,
-    Interpreter, PyObjectRef, PyPayload, PyRef, PyResult, Settings, VirtualMachine,
+    builtins::PyWeak, compiler::Mode, scope::Scope, Interpreter, PyObjectRef, PyPayload, PyRef,
+    PyResult, Settings, VirtualMachine,
 };
 use std::{
     cell::RefCell,
@@ -324,7 +322,7 @@ impl WASMVirtualMachine {
     pub(crate) fn run(
         &self,
         source: &str,
-        mode: compile::Mode,
+        mode: Mode,
         source_path: Option<String>,
     ) -> Result<JsValue, JsValue> {
         self.with_vm(|vm, StoredVirtualMachine { ref scope, .. }| {
@@ -337,11 +335,11 @@ impl WASMVirtualMachine {
     }
 
     pub fn exec(&self, source: &str, source_path: Option<String>) -> Result<JsValue, JsValue> {
-        self.run(source, compile::Mode::Exec, source_path)
+        self.run(source, Mode::Exec, source_path)
     }
 
     pub fn eval(&self, source: &str, source_path: Option<String>) -> Result<JsValue, JsValue> {
-        self.run(source, compile::Mode::Eval, source_path)
+        self.run(source, Mode::Eval, source_path)
     }
 
     #[wasm_bindgen(js_name = execSingle)]
@@ -350,6 +348,6 @@ impl WASMVirtualMachine {
         source: &str,
         source_path: Option<String>,
     ) -> Result<JsValue, JsValue> {
-        self.run(source, compile::Mode::Single, source_path)
+        self.run(source, Mode::Single, source_path)
     }
 }