Merge pull request #4119 from youknowone/compiler

Reorganize compiler crates

.gitattributes

@@ -1,7 +1,8 @@
 Lib/**      linguist-vendored
 Cargo.lock  linguist-generated -merge
+*.snap      linguist-generated -merge
 ast/src/ast_gen.rs          linguist-generated -merge
 vm/src/stdlib/ast/gen.rs    linguist-generated -merge
-parser/python.lalrpop       text eol=LF
+compiler/parser/python.lalrpop       text eol=LF
 Lib/*.py    text working-tree-encoding=UTF-8 eol=LF
 **/*.rs     text working-tree-encoding=UTF-8 eol=LF

.github/workflows/ci.yaml

@@ -112,8 +112,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
       - name: Set up the Windows environment
         shell: bash
@@ -156,8 +156,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
 
       - uses: dtolnay/rust-toolchain@stable
         with:
@@ -226,8 +226,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
 
       - uses: dtolnay/rust-toolchain@stable
       - uses: actions/setup-python@v2
@@ -288,17 +288,17 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - name: Check if cached generated parser exists
         id: generated_parser
         uses: andstor/file-existence-action@v1
         with:
-          files: "parser/python.rs"
+          files: "compiler/parser/python.rs"
       - if: runner.os == 'Windows'
         name: Force python.lalrpop to be lf  # actions@checkout ignore .gitattributes
         run: |
-          set file parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
+          set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
       - name: Install lalrpop
         if: steps.generated_parser.outputs.files_exists == 'false'
         uses: baptiste0928/cargo-install@v1
@@ -307,7 +307,7 @@ jobs:
           version: "0.19.8"
       - name: Run lalrpop
         if: steps.generated_parser.outputs.files_exists == 'false'
-        run: lalrpop parser/python.lalrpop
+        run: lalrpop compiler/parser/python.lalrpop
 
   lint:
     name: Check Rust code with rustfmt and clippy
@@ -318,8 +318,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
         with:
             components: rustfmt, clippy
@@ -351,8 +351,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@master
         with:
             toolchain: nightly
@@ -372,8 +372,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v1
       - name: install wasm-pack
@@ -420,8 +420,8 @@ jobs:
       - name: Cache generated parser
         uses: actions/cache@v2
         with:
-          path: parser/python.rs
-          key: lalrpop-${{ hashFiles('parser/python.lalrpop') }}
+          path: compiler/parser/python.rs
+          key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
       - uses: dtolnay/rust-toolchain@stable
         with:
           target: wasm32-wasi

.gitignore

@@ -20,4 +20,4 @@ flamescope.json
 extra_tests/snippets/resources
 extra_tests/not_impl.py
 
-parser/python.rs
+compiler/parser/python.rs

Cargo.lock

@@ -1754,6 +1754,22 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "rustpython-codegen"
+version = "0.1.2"
+dependencies = [
+ "ahash",
+ "indexmap",
+ "insta",
+ "itertools",
+ "log",
+ "num-complex",
+ "num-traits",
+ "rustpython-ast",
+ "rustpython-bytecode",
+ "rustpython-parser",
+]
+
 [[package]]
 name = "rustpython-common"
 version = "0.0.0"
@@ -1782,27 +1798,11 @@ name = "rustpython-compiler"
 version = "0.1.2"
 dependencies = [
  "rustpython-bytecode",
- "rustpython-compiler-core",
+ "rustpython-codegen",
  "rustpython-parser",
  "thiserror",
 ]
 
-[[package]]
-name = "rustpython-compiler-core"
-version = "0.1.2"
-dependencies = [
- "ahash",
- "indexmap",
- "insta",
- "itertools",
- "log",
- "num-complex",
- "num-traits",
- "rustpython-ast",
- "rustpython-bytecode",
- "rustpython-parser",
-]
-
 [[package]]
 name = "rustpython-derive"
 version = "0.1.2"
@@ -1922,7 +1922,6 @@ dependencies = [
  "rand_core",
  "rustpython-common",
  "rustpython-derive",
- "rustpython-parser",
  "rustpython-vm",
  "schannel",
  "sha-1",
@@ -1992,9 +1991,9 @@ dependencies = [
  "rustc_version",
  "rustpython-ast",
  "rustpython-bytecode",
+ "rustpython-codegen",
  "rustpython-common",
  "rustpython-compiler",
- "rustpython-compiler-core",
  "rustpython-derive",
  "rustpython-jit",
  "rustpython-parser",

Cargo.toml

@@ -14,8 +14,8 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 [workspace]
 resolver = "2"
 members = [
-    ".", "ast", "bytecode", "common", "compiler", "compiler/porcelain",
-    "derive", "jit", "parser", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
+    "compiler", "compiler/ast", "compiler/bytecode", "compiler/codegen", "compiler/parser",
+    ".", "common", "derive", "jit", "vm", "vm/pylib-crate", "stdlib", "wasm/lib",
 ]
 
 [features]
@@ -36,10 +36,10 @@ ssl-vendor = ["rustpython-stdlib/ssl-vendor"]
 log = "0.4.16"
 env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] }
 clap = "2.34"
-rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" }
-rustpython-parser = { path = "parser", version = "0.1.1" }
-rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] }
-rustpython-stdlib = {path = "stdlib", optional = true, default-features = false, features = ["compile-parse"]}
+rustpython-compiler = { path = "compiler", version = "0.1.1" }
+rustpython-parser = { path = "compiler/parser", version = "0.1.1" }
+rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] }
+rustpython-stdlib = {path = "stdlib", optional = true, default-features = false}
 dirs = { package = "dirs-next", version = "2.0.0" }
 num-traits = "0.2.14"
 cfg-if = "1.0.0"

DEVELOPMENT.md

@@ -115,10 +115,10 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i
 Understanding a new codebase takes time. Here's a brief view of the
 repository's structure:
 
-- `bytecode/src`: python bytecode representation in rust structures
 - `compiler/src`: python compilation to bytecode
+  - `bytecode/src`: python bytecode representation in rust structures
+  - `parser/src`: python lexing, parsing and ast
 - `derive/src`: Rust language extensions and macros specific to rustpython
-- `parser/src`: python lexing, parsing and ast
 - `Lib`: Carefully selected / copied files from CPython sourcecode. This is
    the python side of the standard library.
   - `test`: CPython test suite
@@ -137,7 +137,7 @@ implementation is found in the `src` directory (specifically, `src/lib.rs`).
 
 The top-level `rustpython` binary depends on several lower-level crates including:
 
-- `rustpython-parser` (implementation in `parser/src`)
+- `rustpython-parser` (implementation in `compiler/parser/src`)
 - `rustpython-compiler` (implementation in `compiler/src`)
 - `rustpython-vm` (implementation in `vm/src`)
 
@@ -155,15 +155,15 @@ enable a line of code to go through a series of steps:
 This crate contains the lexer and parser to convert a line of code to
 an Abstract Syntax Tree (AST):
 
-- Lexer: `parser/src/lexer.rs` converts Python source code into tokens
-- Parser: `parser/src/parser.rs` takes the tokens generated by the lexer and parses
+- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens
+- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses
   the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax
   tree are Rust structs and enums.
   - The Parser relies on `LALRPOP`, a Rust parser generator framework. The
-    LALRPOP definition of Python's grammar is in `parser/src/python.lalrpop`.
+    LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`.
   - More information on parsers and a tutorial can be found in the 
     [LALRPOP book](https://lalrpop.github.io/lalrpop/).
-- AST: `ast/` implements in Rust the Python types and expressions
+- AST: `compiler/ast/` implements in Rust the Python types and expressions
   represented by the AST nodes.
 
 ### rustpython-compiler

compiler/Cargo.toml

@@ -1,22 +1,12 @@
 [package]
-name = "rustpython-compiler-core"
+name = "rustpython-compiler"
 version = "0.1.2"
-description = "Compiler for python code into bytecode for the rustpython VM."
+description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
 authors = ["RustPython Team"]
-repository = "https://github.com/RustPython/RustPython"
-license = "MIT"
 edition = "2021"
 
 [dependencies]
-indexmap = "1.8.1"
-itertools = "0.10.3"
-rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
-rustpython-ast = { path = "../ast", features = ["unparse"] }
-num-complex = { version = "0.4.0", features = ["serde"] }
-num-traits = "0.2.14"
-log = "0.4.16"
-ahash = "0.7.6"
-
-[dev-dependencies]
-rustpython-parser = { path = "../parser" }
-insta = "1.14.0"
+thiserror = "1.0"
+rustpython-codegen = { path = "codegen" }
+rustpython-parser = { path = "parser" }
+rustpython-bytecode = { path = "bytecode" }

compiler/ast/Cargo.toml

@@ -12,5 +12,5 @@ unparse = ["rustpython-common"]
 
 [dependencies]
 num-bigint = "0.4.3"
-rustpython-common = { path = "../common", optional = true }
+rustpython-common = { path = "../../common", optional = true }
 rustpython-bytecode = { path = "../bytecode"}

compiler/codegen/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "rustpython-codegen"
+version = "0.1.2"
+description = "Compiler for python code into bytecode for the rustpython VM."
+authors = ["RustPython Team"]
+repository = "https://github.com/RustPython/RustPython"
+license = "MIT"
+edition = "2021"
+
+[dependencies]
+indexmap = "1.8.1"
+itertools = "0.10.3"
+rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
+rustpython-ast = { path = "../ast", features = ["unparse"] }
+num-complex = { version = "0.4.0", features = ["serde"] }
+num-traits = "0.2.14"
+log = "0.4.16"
+ahash = "0.7.6"
+
+[dev-dependencies]
+rustpython-parser = { path = "../parser" }
+insta = "1.14.0"

compiler/codegen/src/compile.rs

@@ -5,13 +5,11 @@
 //!   <https://github.com/python/cpython/blob/main/Python/compile.c>
 //!   <https://github.com/micropython/micropython/blob/master/py/compile.c>
 
-use crate::ir::{self, CodeInfo};
-pub use crate::mode::Mode;
-use crate::symboltable::{make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable};
-use crate::IndexSet;
 use crate::{
     error::{CompileError, CompileErrorType},
-    symboltable,
+    ir,
+    symboltable::{self, make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable},
+    IndexSet,
 };
 use itertools::Itertools;
 use num_complex::Complex64;
@@ -20,6 +18,8 @@ use rustpython_ast as ast;
 use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction};
 use std::borrow::Cow;
 
+pub use crate::mode::Mode;
+
 type CompileResult<T> = Result<T, CompileError>;
 
 #[derive(PartialEq, Eq, Clone, Copy)]
@@ -61,7 +61,7 @@ fn is_forbidden_name(name: &str) -> bool {
 
 /// Main structure holding the state of compilation.
 struct Compiler {
-    code_stack: Vec<CodeInfo>,
+    code_stack: Vec<ir::CodeInfo>,
     symbol_table_stack: Vec<SymbolTable>,
     source_path: String,
     current_source_location: ast::Location,
@@ -199,7 +199,7 @@ pub fn compile_expression(
 
 impl Compiler {
     fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self {
-        let module_code = CodeInfo {
+        let module_code = ir::CodeInfo {
             flags: bytecode::CodeFlags::NEW_LOCALS,
             posonlyarg_count: 0,
             arg_count: 0,
@@ -278,7 +278,7 @@ impl Compiler {
 
         self.symbol_table_stack.push(table);
 
-        let info = CodeInfo {
+        let info = ir::CodeInfo {
             flags,
             posonlyarg_count,
             arg_count,
@@ -318,7 +318,7 @@ impl Compiler {
     fn _name_inner(
         &mut self,
         name: &str,
-        cache: impl FnOnce(&mut CodeInfo) -> &mut IndexSet<String>,
+        cache: impl FnOnce(&mut ir::CodeInfo) -> &mut IndexSet<String>,
     ) -> bytecode::NameIdx {
         let name = self.mangle(name);
         let cache = cache(self.current_codeinfo());
@@ -2583,7 +2583,7 @@ impl Compiler {
         self.emit(Instruction::LoadConst { idx })
     }
 
-    fn current_codeinfo(&mut self) -> &mut CodeInfo {
+    fn current_codeinfo(&mut self) -> &mut ir::CodeInfo {
         self.code_stack.last_mut().expect("no code on stack")
     }
 

compiler/codegen/src/lib.rs

@@ -0,0 +1,15 @@
+//! Compile a Python AST or source code into bytecode consumable by RustPython.
+#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
+#![doc(html_root_url = "https://docs.rs/rustpython-compiler/")]
+
+#[macro_use]
+extern crate log;
+
+type IndexMap<K, V> = indexmap::IndexMap<K, V, ahash::RandomState>;
+type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>;
+
+pub mod compile;
+pub mod error;
+pub mod ir;
+pub mod mode;
+pub mod symboltable;

compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap

@@ -0,0 +1,14 @@
+---
+source: compiler/codegen/src/compile.rs
+expression: "compile_exec(\"\\\nif True and False and False:\n    pass\n\")"
+---
+  1           0 LoadConst            (True)
+              1 JumpIfFalse          (6)
+              2 LoadConst            (False)
+              3 JumpIfFalse          (6)
+              4 LoadConst            (False)
+              5 JumpIfFalse          (6)
+
+  2     >>    6 LoadConst            (None)
+              7 ReturnValue
+

compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap

@@ -0,0 +1,16 @@
+---
+source: compiler/codegen/src/compile.rs
+expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n    pass\n\")"
+---
+  1           0 LoadConst            (True)
+              1 JumpIfFalse          (4)
+              2 LoadConst            (False)
+              3 JumpIfTrue           (8)
+        >>    4 LoadConst            (False)
+              5 JumpIfFalse          (8)
+              6 LoadConst            (True)
+              7 JumpIfFalse          (8)
+
+  2     >>    8 LoadConst            (None)
+              9 ReturnValue
+

compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap

@@ -0,0 +1,14 @@
+---
+source: compiler/codegen/src/compile.rs
+expression: "compile_exec(\"\\\nif True or False or False:\n    pass\n\")"
+---
+  1           0 LoadConst            (True)
+              1 JumpIfTrue           (6)
+              2 LoadConst            (False)
+              3 JumpIfTrue           (6)
+              4 LoadConst            (False)
+              5 JumpIfFalse          (6)
+
+  2     >>    6 LoadConst            (None)
+              7 ReturnValue
+

compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap

@@ -0,0 +1,87 @@
+---
+source: compiler/codegen/src/compile.rs
+expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n    with self.subTest(type=type(stop_exc)):\n        try:\n            async with woohoo():\n                raise stop_exc\n        except Exception as ex:\n            self.assertIs(ex, stop_exc)\n        else:\n            self.fail(f'{stop_exc} was suppressed')\n\")"
+---
+  1           0 SetupLoop            (69)
+              1 LoadNameAny          (0, StopIteration)
+              2 LoadConst            ("spam")
+              3 CallFunctionPositional (1)
+              4 LoadNameAny          (1, StopAsyncIteration)
+              5 LoadConst            ("ham")
+              6 CallFunctionPositional (1)
+              7 BuildTuple           (2, false)
+              8 GetIter
+        >>    9 ForIter              (68)
+             10 StoreLocal           (2, stop_exc)
+
+  2          11 LoadNameAny          (3, self)
+             12 LoadMethod           (subTest)
+             13 LoadNameAny          (5, type)
+             14 LoadNameAny          (2, stop_exc)
+             15 CallFunctionPositional (1)
+             16 LoadConst            (("type"))
+             17 CallMethodKeyword    (1)
+             18 SetupWith            (65)
+             19 Pop
+
+  3          20 SetupExcept          (40)
+
+  4          21 LoadNameAny          (6, woohoo)
+             22 CallFunctionPositional (0)
+             23 BeforeAsyncWith
+             24 GetAwaitable
+             25 LoadConst            (None)
+             26 YieldFrom
+             27 SetupAsyncWith       (33)
+             28 Pop
+
+  5          29 LoadNameAny          (2, stop_exc)
+             30 Raise                (Raise)
+
+  4          31 PopBlock
+             32 EnterFinally
+        >>   33 WithCleanupStart
+             34 GetAwaitable
+             35 LoadConst            (None)
+             36 YieldFrom
+             37 WithCleanupFinish
+             38 PopBlock
+             39 Jump                 (54)
+        >>   40 Duplicate
+
+  6          41 LoadNameAny          (7, Exception)
+             42 TestOperation        (ExceptionMatch)
+             43 JumpIfFalse          (53)
+             44 StoreLocal           (8, ex)
+
+  7          45 LoadNameAny          (3, self)
+             46 LoadMethod           (assertIs)
+             47 LoadNameAny          (8, ex)
+             48 LoadNameAny          (2, stop_exc)
+             49 CallMethodPositional (2)
+             50 Pop
+             51 PopException
+             52 Jump                 (63)
+        >>   53 Raise                (Reraise)
+
+  9     >>   54 LoadNameAny          (3, self)
+             55 LoadMethod           (fail)
+             56 LoadConst            ("")
+
+  1          57 LoadNameAny          (2, stop_exc)
+             58 FormatValue          (None)
+
+  9          59 LoadConst            (" was suppressed")
+             60 BuildString          (2)
+             61 CallMethodPositional (1)
+             62 Pop
+
+  2     >>   63 PopBlock
+             64 EnterFinally
+        >>   65 WithCleanupStart
+             66 WithCleanupFinish
+             67 Jump                 (9)
+        >>   68 PopBlock
+        >>   69 LoadConst            (None)
+             70 ReturnValue
+

compiler/parser/build.rs

@@ -77,7 +77,7 @@ fn try_lalrpop(source: &str, target: &str) -> anyhow::Result<()> {
     }
 
     #[cfg(not(feature = "lalrpop"))]
-    panic!("try: cargo build --manifest-path=parser/Cargo.toml --features=lalrpop");
+    panic!("try: cargo build --manifest-path=compiler/parser/Cargo.toml --features=lalrpop");
 }
 
 fn sha_equal(expected_sha3_str: &str, actual_sha3: &[u8; 32]) -> bool {

compiler/porcelain/Cargo.toml

@@ -1,12 +0,0 @@
-[package]
-name = "rustpython-compiler"
-version = "0.1.2"
-description = "A usability wrapper around rustpython-parser and rustpython-compiler-core"
-authors = ["RustPython Team"]
-edition = "2021"
-
-[dependencies]
-thiserror = "1.0"
-rustpython-compiler-core = { path = ".." }
-rustpython-parser = { path = "../../parser" }
-rustpython-bytecode = { path = "../../bytecode" }

compiler/porcelain/src/lib.rs

@@ -1,122 +0,0 @@
-use rustpython_bytecode::CodeObject;
-use rustpython_compiler_core::{compile, symboltable};
-use rustpython_parser::ast::{fold::Fold, ConstantOptimizer, Location};
-use rustpython_parser::parser;
-use std::fmt;
-
-pub use compile::{CompileOpts, Mode};
-pub use symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType};
-
-#[derive(Debug, thiserror::Error)]
-pub enum CompileErrorType {
-    #[error(transparent)]
-    Compile(#[from] rustpython_compiler_core::error::CompileErrorType),
-    #[error(transparent)]
-    Parse(#[from] rustpython_parser::error::ParseErrorType),
-}
-
-#[derive(Debug, thiserror::Error)]
-pub struct CompileError {
-    pub error: CompileErrorType,
-    pub statement: Option<String>,
-    pub source_path: String,
-    pub location: Location,
-}
-
-impl fmt::Display for CompileError {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        let loc = self.location;
-        if let Some(ref stmt) = self.statement {
-            // visualize the error when location and statement are provided
-            write!(
-                f,
-                "{}",
-                loc.visualize(stmt, &format_args!("{} at {}", self.error, loc))
-            )
-        } else {
-            write!(f, "{} at {}", self.error, loc)
-        }
-    }
-}
-
-impl CompileError {
-    fn from_compile(error: rustpython_compiler_core::error::CompileError, source: &str) -> Self {
-        CompileError {
-            error: error.error.into(),
-            location: error.location,
-            source_path: error.source_path,
-            statement: get_statement(source, error.location),
-        }
-    }
-    fn from_parse(
-        error: rustpython_parser::error::ParseError,
-        source: &str,
-        source_path: String,
-    ) -> Self {
-        CompileError {
-            error: error.error.into(),
-            location: error.location,
-            source_path,
-            statement: get_statement(source, error.location),
-        }
-    }
-    fn from_symtable(
-        error: symboltable::SymbolTableError,
-        source: &str,
-        source_path: String,
-    ) -> Self {
-        Self::from_compile(error.into_compile_error(source_path), source)
-    }
-}
-
-/// Compile a given sourcecode into a bytecode object.
-pub fn compile(
-    source: &str,
-    mode: compile::Mode,
-    source_path: String,
-    opts: CompileOpts,
-) -> Result<CodeObject, CompileError> {
-    let parser_mode = match mode {
-        compile::Mode::Exec => parser::Mode::Module,
-        compile::Mode::Eval => parser::Mode::Expression,
-        compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
-    };
-    let mut ast = match parser::parse(source, parser_mode) {
-        Ok(x) => x,
-        Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
-    };
-    if opts.optimize > 0 {
-        ast = ConstantOptimizer::new()
-            .fold_mod(ast)
-            .unwrap_or_else(|e| match e {});
-    }
-    compile::compile_top(&ast, source_path, mode, opts)
-        .map_err(|e| CompileError::from_compile(e, source))
-}
-
-pub fn compile_symtable(
-    source: &str,
-    mode: compile::Mode,
-    source_path: &str,
-) -> Result<symboltable::SymbolTable, CompileError> {
-    let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned());
-    let res = match mode {
-        compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
-            let ast = parser::parse_program(source).map_err(parse_err)?;
-            symboltable::make_symbol_table(&ast)
-        }
-        compile::Mode::Eval => {
-            let expr = parser::parse_expression(source).map_err(parse_err)?;
-            symboltable::make_symbol_table_expr(&expr)
-        }
-    };
-    res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned()))
-}
-
-fn get_statement(source: &str, loc: Location) -> Option<String> {
-    if loc.column() == 0 || loc.row() == 0 {
-        return None;
-    }
-    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
-    Some(line + "\n")
-}

compiler/src/lib.rs

@@ -1,15 +1,124 @@
-//! Compile a Python AST or source code into bytecode consumable by RustPython.
-#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
-#![doc(html_root_url = "https://docs.rs/rustpython-compiler/")]
+use rustpython_bytecode::CodeObject;
+use rustpython_codegen::{compile, symboltable};
+use rustpython_parser::{
+    ast::{fold::Fold, ConstantOptimizer, Location},
+    parser,
+};
+use std::fmt;
 
-#[macro_use]
-extern crate log;
+pub use compile::{CompileOpts, Mode};
+pub use symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType};
 
-type IndexMap<K, V> = indexmap::IndexMap<K, V, ahash::RandomState>;
-type IndexSet<T> = indexmap::IndexSet<T, ahash::RandomState>;
+#[derive(Debug, thiserror::Error)]
+pub enum CompileErrorType {
+    #[error(transparent)]
+    Compile(#[from] rustpython_codegen::error::CompileErrorType),
+    #[error(transparent)]
+    Parse(#[from] rustpython_parser::error::ParseErrorType),
+}
 
-pub mod compile;
-pub mod error;
-pub mod ir;
-pub mod mode;
-pub mod symboltable;
+#[derive(Debug, thiserror::Error)]
+pub struct CompileError {
+    pub error: CompileErrorType,
+    pub statement: Option<String>,
+    pub source_path: String,
+    pub location: Location,
+}
+
+impl fmt::Display for CompileError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let loc = self.location;
+        if let Some(ref stmt) = self.statement {
+            // visualize the error when location and statement are provided
+            write!(
+                f,
+                "{}",
+                loc.visualize(stmt, &format_args!("{} at {}", self.error, loc))
+            )
+        } else {
+            write!(f, "{} at {}", self.error, loc)
+        }
+    }
+}
+
+impl CompileError {
+    fn from_compile(error: rustpython_codegen::error::CompileError, source: &str) -> Self {
+        CompileError {
+            error: error.error.into(),
+            location: error.location,
+            source_path: error.source_path,
+            statement: get_statement(source, error.location),
+        }
+    }
+    fn from_parse(
+        error: rustpython_parser::error::ParseError,
+        source: &str,
+        source_path: String,
+    ) -> Self {
+        CompileError {
+            error: error.error.into(),
+            location: error.location,
+            source_path,
+            statement: get_statement(source, error.location),
+        }
+    }
+    fn from_symtable(
+        error: symboltable::SymbolTableError,
+        source: &str,
+        source_path: String,
+    ) -> Self {
+        Self::from_compile(error.into_compile_error(source_path), source)
+    }
+}
+
+/// Compile a given sourcecode into a bytecode object.
+pub fn compile(
+    source: &str,
+    mode: compile::Mode,
+    source_path: String,
+    opts: CompileOpts,
+) -> Result<CodeObject, CompileError> {
+    let parser_mode = match mode {
+        compile::Mode::Exec => parser::Mode::Module,
+        compile::Mode::Eval => parser::Mode::Expression,
+        compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
+    };
+    let mut ast = match parser::parse(source, parser_mode) {
+        Ok(x) => x,
+        Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
+    };
+    if opts.optimize > 0 {
+        ast = ConstantOptimizer::new()
+            .fold_mod(ast)
+            .unwrap_or_else(|e| match e {});
+    }
+    compile::compile_top(&ast, source_path, mode, opts)
+        .map_err(|e| CompileError::from_compile(e, source))
+}
+
+pub fn compile_symtable(
+    source: &str,
+    mode: compile::Mode,
+    source_path: &str,
+) -> Result<symboltable::SymbolTable, CompileError> {
+    let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned());
+    let res = match mode {
+        compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
+            let ast = parser::parse_program(source).map_err(parse_err)?;
+            symboltable::make_symbol_table(&ast)
+        }
+        compile::Mode::Eval => {
+            let expr = parser::parse_expression(source).map_err(parse_err)?;
+            symboltable::make_symbol_table_expr(&expr)
+        }
+    };
+    res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned()))
+}
+
+fn get_statement(source: &str, loc: Location) -> Option<String> {
+    if loc.column() == 0 || loc.row() == 0 {
+        return None;
+    }
+    let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
+    Some(line + "\n")
+}

derive/Cargo.toml

@@ -15,8 +15,8 @@ syn = { version = "1.0.91", features = ["full", "extra-traits"] }
 syn-ext = { version = "0.4.0", features = ["full"] }
 quote = "1.0.18"
 proc-macro2 = "1.0.37"
-rustpython-compiler = { path = "../compiler/porcelain", version = "0.1.1" }
-rustpython-bytecode = { path = "../bytecode", version = "0.1.1" }
+rustpython-bytecode = { path = "../compiler/bytecode", version = "0.1.1" }
+rustpython-compiler = { path = "../compiler", version = "0.1.1" }
 maplit = "1.0.2"
 once_cell = "1.10.0"
 textwrap = { version = "0.15.0", default-features = false }

jit/Cargo.toml

@@ -15,7 +15,7 @@ cranelift-module = "0.76.0"
 cranelift-jit = "0.76.0"
 num-traits = "0.2"
 libffi = "2.0.0"
-rustpython-bytecode = { path = "../bytecode", version = "0.1.2" }
+rustpython-bytecode = { path = "../compiler/bytecode", version = "0.1.2" }
 thiserror = "1.0"
 
 [dev-dependencies]

scripts/update_asdl.sh

@@ -3,5 +3,5 @@ set -e
 
 cd "$(dirname "$(dirname "$0")")"
 
-python ast/asdl_rs.py -D ast/src/ast_gen.rs -M vm/src/stdlib/ast/gen.rs ast/Python.asdl
-rustfmt ast/src/ast_gen.rs vm/src/stdlib/ast/gen.rs
+python compiler/ast/asdl_rs.py -D compiler/ast/src/ast_gen.rs -M vm/src/stdlib/ast/gen.rs compiler/ast/Python.asdl
+rustfmt compiler/ast/src/ast_gen.rs vm/src/stdlib/ast/gen.rs

stdlib/Cargo.toml

@@ -6,7 +6,6 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-rustpython-parser = { path = "../parser", optional = true }
 rustpython-derive = { path = "../derive" }
 rustpython-vm = { path = "../vm" }
 rustpython-common = { path = "../common" }
@@ -81,14 +80,7 @@ page_size = "0.4"
 termios = "0.3.3"
 
 [features]
-default = ["compile-parse"]
-compile-parse = ["rustpython-vm/compile-parse", "rustpython-parser"]
 threading = ["rustpython-common/threading", "rustpython-vm/threading"]
-parser = ["rustpython-vm/parser"]
-ast = ["rustpython-vm/ast"]
-compiler = ["rustpython-vm/compiler"]
-# compiler = ["rustpython-compiler", "rustpython-compiler-core", "ast"]
-# parser = ["rustpython-parser", "ast"]
 
 zlib = ["libz-sys", "flate2/zlib"]
 bz2 = ["bzip2"]

stdlib/src/lib.rs

@@ -109,11 +109,6 @@ pub fn get_module_inits() -> impl Iterator<Item = (Cow<'static, str>, StdlibInit
             "_statistics" => statistics::make_module,
             // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module,
         }
-        // parser related modules:
-        #[cfg(feature = "rustpython-ast")]
-        {
-            "_ast" => ast::make_module,
-        }
         #[cfg(any(unix, target_os = "wasi"))]
         {
             "fcntl" => fcntl::make_module,

vm/Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 include = ["src/**/*.rs", "Cargo.toml", "build.rs", "Lib/**/*.py"]
 
 [features]
-default = ["compile-parse"]
+default = ["compiler"]
 importlib = []
 encodings = ["importlib"]
 vm-tracing-logging = []
@@ -17,26 +17,27 @@ flame-it = ["flame", "flamer"]
 freeze-stdlib = ["rustpython-pylib/freeze-stdlib"]
 jit = ["rustpython-jit"]
 threading = ["rustpython-common/threading"]
-compile-parse = ["parser", "compiler"]
+compiler = ["parser", "codegen"]
 ast = ["rustpython-ast"]
-compiler = ["rustpython-compiler", "rustpython-compiler-core", "ast"]
+codegen = ["rustpython-compiler", "rustpython-codegen", "ast"]
 parser = ["rustpython-parser", "ast"]
 
 [dependencies]
+rustpython-compiler = { path = "../compiler", optional = true, version = "0.1.2" }
+rustpython-ast = { path = "../compiler/ast", optional = true, version = "0.1" }
+rustpython-parser = { path = "../compiler/parser", optional = true, version = "0.1.2" }
+rustpython-codegen = { path = "../compiler/codegen", optional = true, version = "0.1.2" }
+rustpython-bytecode = { path = "../compiler/bytecode", version = "0.1.2" }
+rustpython-common = { path = "../common" }
+rustpython-derive = { path = "../derive", version = "0.1.2" }
+rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" }
+rustpython-pylib = { path = "pylib-crate", version = "0.1.0" }
+
 num-complex = { version = "0.4.0", features = ["serde"] }
 num-bigint = { version = "0.4.3", features = ["serde"] }
 num-traits = "0.2.14"
 num-integer = "0.1.44"
 num-rational = "0.4.0"
-rustpython-common = { path = "../common" }
-rustpython-derive = { path = "../derive", version = "0.1.2" }
-rustpython-ast = { path = "../ast", optional = true, version = "0.1" }
-rustpython-parser = { path = "../parser", optional = true, version = "0.1.2" }
-rustpython-compiler = { path = "../compiler/porcelain", optional = true, version = "0.1.2" }
-rustpython-compiler-core = { path = "../compiler", optional = true, version = "0.1.2" }
-rustpython-bytecode = { path = "../bytecode", version = "0.1.2" }
-rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" }
-rustpython-pylib = { path = "pylib-crate", version = "0.1.0" }
 rand = "0.8.5"
 getrandom = { version = "0.2.6", features = ["js"] }
 log = "0.4.16"

vm/pylib-crate/Cargo.toml

@@ -13,7 +13,7 @@ freeze-stdlib = []
 
 [dependencies]
 rustpython-derive = { version = "0.1.2", path = "../../derive" }
-rustpython-bytecode = { version = "0.1.2", path = "../../bytecode" }
+rustpython-bytecode = { version = "0.1.2", path = "../../compiler/bytecode" }
 
 [build-dependencies]
 glob = "0.3"

vm/src/stdlib/ast.rs

@@ -278,10 +278,9 @@ pub(crate) fn compile(
 ) -> PyResult {
     let opts = vm.compile_opts();
     let ast = Node::ast_from_object(vm, object)?;
-    let code =
-        rustpython_compiler_core::compile::compile_top(&ast, filename.to_owned(), mode, opts)
-            // TODO: use vm.new_syntax_error()
-            .map_err(|err| vm.new_value_error(err.to_string()))?;
+    let code = rustpython_codegen::compile::compile_top(&ast, filename.to_owned(), mode, opts)
+        // TODO: use vm.new_syntax_error()
+        .map_err(|err| vm.new_value_error(err.to_string()))?;
     Ok(vm.ctx.new_code(code).into())
 }
 

wasm/lib/Cargo.toml

@@ -16,11 +16,11 @@ freeze-stdlib = ["rustpython-vm/freeze-stdlib"]
 no-start-func = []
 
 [dependencies]
-rustpython-parser = { path = "../../parser" }
+rustpython-parser = { path = "../../compiler/parser" }
 rustpython-common = { path = "../../common" }
 # make sure no threading! otherwise wasm build will fail
-rustpython-vm = { path = "../../vm", default-features = false, features = ["compile-parse", "encodings"] }
-rustpython-stdlib = { path = "../../stdlib", default-features = false, features = ["compile-parse"], optional = true }
+rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings"] }
+rustpython-stdlib = { path = "../../stdlib", default-features = false, optional = true }
 wasm-bindgen = "0.2"
 wasm-bindgen-futures = "0.4"
 serde-wasm-bindgen = "0.3.1"