diff --git a/.gitattributes b/.gitattributes index b3d3fcc94a..d663b4830a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,7 +1,8 @@ Lib/** linguist-vendored Cargo.lock linguist-generated -merge +*.snap linguist-generated -merge ast/src/ast_gen.rs linguist-generated -merge vm/src/stdlib/ast/gen.rs linguist-generated -merge -parser/python.lalrpop text eol=LF +compiler/parser/python.lalrpop text eol=LF Lib/*.py text working-tree-encoding=UTF-8 eol=LF **/*.rs text working-tree-encoding=UTF-8 eol=LF diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 58309d57b3..2972927daf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -112,8 +112,8 @@ jobs: - name: Cache generated parser uses: actions/cache@v2 with: - path: parser/python.rs - key: lalrpop-${{ hashFiles('parser/python.lalrpop') }} + path: compiler/parser/python.rs + key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@stable - name: Set up the Windows environment shell: bash @@ -156,8 +156,8 @@ jobs: - name: Cache generated parser uses: actions/cache@v2 with: - path: parser/python.rs - key: lalrpop-${{ hashFiles('parser/python.lalrpop') }} + path: compiler/parser/python.rs + key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@stable with: @@ -226,8 +226,8 @@ jobs: - name: Cache generated parser uses: actions/cache@v2 with: - path: parser/python.rs - key: lalrpop-${{ hashFiles('parser/python.lalrpop') }} + path: compiler/parser/python.rs + key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@stable - uses: actions/setup-python@v2 @@ -288,17 +288,17 @@ jobs: - name: Cache generated parser uses: actions/cache@v2 with: - path: parser/python.rs - key: lalrpop-${{ hashFiles('parser/python.lalrpop') }} + path: compiler/parser/python.rs + key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - name: Check if cached generated parser exists id: generated_parser uses: andstor/file-existence-action@v1 with: - files: "parser/python.rs" + files: "compiler/parser/python.rs" - if: runner.os == 'Windows' name: Force python.lalrpop to be lf # actions@checkout ignore .gitattributes run: | - set file parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file + set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file - name: Install lalrpop if: steps.generated_parser.outputs.files_exists == 'false' uses: baptiste0928/cargo-install@v1 @@ -307,7 +307,7 @@ jobs: version: "0.19.8" - name: Run lalrpop if: steps.generated_parser.outputs.files_exists == 'false' - run: lalrpop parser/python.lalrpop + run: lalrpop compiler/parser/python.lalrpop lint: name: Check Rust code with rustfmt and clippy @@ -318,8 +318,8 @@ jobs: - name: Cache generated parser uses: actions/cache@v2 with: - path: parser/python.rs - key: lalrpop-${{ hashFiles('parser/python.lalrpop') }} + path: compiler/parser/python.rs + key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy @@ -351,8 +351,8 @@ jobs: - name: Cache generated parser uses: actions/cache@v2 with: - path: parser/python.rs - key: lalrpop-${{ hashFiles('parser/python.lalrpop') }} + path: compiler/parser/python.rs + key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@master with: toolchain: nightly @@ -372,8 +372,8 @@ jobs: - name: Cache generated parser uses: actions/cache@v2 with: - path: parser/python.rs - key: lalrpop-${{ hashFiles('parser/python.lalrpop') }} + path: compiler/parser/python.rs + key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v1 - name: install wasm-pack @@ -420,8 +420,8 @@ jobs: - name: Cache generated parser uses: actions/cache@v2 with: - path: parser/python.rs - key: lalrpop-${{ hashFiles('parser/python.lalrpop') }} + path: compiler/parser/python.rs + key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasi diff --git a/.gitignore b/.gitignore index 6a43a227ad..3b098f1c89 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,4 @@ flamescope.json extra_tests/snippets/resources extra_tests/not_impl.py -parser/python.rs +compiler/parser/python.rs diff --git a/Cargo.lock b/Cargo.lock index dacaafc765..1aab2bb5d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1754,6 +1754,22 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "rustpython-codegen" +version = "0.1.2" +dependencies = [ + "ahash", + "indexmap", + "insta", + "itertools", + "log", + "num-complex", + "num-traits", + "rustpython-ast", + "rustpython-bytecode", + "rustpython-parser", +] + [[package]] name = "rustpython-common" version = "0.0.0" @@ -1782,27 +1798,11 @@ name = "rustpython-compiler" version = "0.1.2" dependencies = [ "rustpython-bytecode", - "rustpython-compiler-core", + "rustpython-codegen", "rustpython-parser", "thiserror", ] -[[package]] -name = "rustpython-compiler-core" -version = "0.1.2" -dependencies = [ - "ahash", - "indexmap", - "insta", - "itertools", - "log", - "num-complex", - "num-traits", - "rustpython-ast", - "rustpython-bytecode", - "rustpython-parser", -] - [[package]] name = "rustpython-derive" version = "0.1.2" @@ -1922,7 +1922,6 @@ dependencies = [ "rand_core", "rustpython-common", "rustpython-derive", - "rustpython-parser", "rustpython-vm", "schannel", "sha-1", @@ -1992,9 +1991,9 @@ dependencies = [ "rustc_version", "rustpython-ast", "rustpython-bytecode", + "rustpython-codegen", "rustpython-common", "rustpython-compiler", - "rustpython-compiler-core", "rustpython-derive", "rustpython-jit", "rustpython-parser", diff --git a/Cargo.toml b/Cargo.toml index 5b2ee61de2..033ee84bb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,8 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"] [workspace] resolver = "2" members = [ - ".", "ast", "bytecode", "common", "compiler", "compiler/porcelain", - "derive", "jit", "parser", "vm", "vm/pylib-crate", "stdlib", "wasm/lib", + "compiler", "compiler/ast", "compiler/bytecode", "compiler/codegen", "compiler/parser", + ".", "common", "derive", "jit", "vm", "vm/pylib-crate", "stdlib", "wasm/lib", ] [features] @@ -36,10 +36,10 @@ ssl-vendor = ["rustpython-stdlib/ssl-vendor"] log = "0.4.16" env_logger = { version = "0.9.0", default-features = false, features = ["atty", "termcolor"] } clap = "2.34" -rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" } -rustpython-parser = { path = "parser", version = "0.1.1" } -rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] } -rustpython-stdlib = {path = "stdlib", optional = true, default-features = false, features = ["compile-parse"]} +rustpython-compiler = { path = "compiler", version = "0.1.1" } +rustpython-parser = { path = "compiler/parser", version = "0.1.1" } +rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compiler"] } +rustpython-stdlib = {path = "stdlib", optional = true, default-features = false} dirs = { package = "dirs-next", version = "2.0.0" } num-traits = "0.2.14" cfg-if = "1.0.0" diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 9f8f9fdcbc..26926b1960 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -115,10 +115,10 @@ exists a raw html viewer which is currently broken, and we welcome a PR to fix i Understanding a new codebase takes time. Here's a brief view of the repository's structure: -- `bytecode/src`: python bytecode representation in rust structures - `compiler/src`: python compilation to bytecode + - `bytecode/src`: python bytecode representation in rust structures + - `parser/src`: python lexing, parsing and ast - `derive/src`: Rust language extensions and macros specific to rustpython -- `parser/src`: python lexing, parsing and ast - `Lib`: Carefully selected / copied files from CPython sourcecode. This is the python side of the standard library. - `test`: CPython test suite @@ -137,7 +137,7 @@ implementation is found in the `src` directory (specifically, `src/lib.rs`). The top-level `rustpython` binary depends on several lower-level crates including: -- `rustpython-parser` (implementation in `parser/src`) +- `rustpython-parser` (implementation in `compiler/parser/src`) - `rustpython-compiler` (implementation in `compiler/src`) - `rustpython-vm` (implementation in `vm/src`) @@ -155,15 +155,15 @@ enable a line of code to go through a series of steps: This crate contains the lexer and parser to convert a line of code to an Abstract Syntax Tree (AST): -- Lexer: `parser/src/lexer.rs` converts Python source code into tokens -- Parser: `parser/src/parser.rs` takes the tokens generated by the lexer and parses +- Lexer: `compiler/parser/src/lexer.rs` converts Python source code into tokens +- Parser: `compiler/parser/src/parser.rs` takes the tokens generated by the lexer and parses the tokens into an AST (Abstract Syntax Tree) where the nodes of the syntax tree are Rust structs and enums. - The Parser relies on `LALRPOP`, a Rust parser generator framework. The - LALRPOP definition of Python's grammar is in `parser/src/python.lalrpop`. + LALRPOP definition of Python's grammar is in `compiler/parser/src/python.lalrpop`. - More information on parsers and a tutorial can be found in the [LALRPOP book](https://lalrpop.github.io/lalrpop/). -- AST: `ast/` implements in Rust the Python types and expressions +- AST: `compiler/ast/` implements in Rust the Python types and expressions represented by the AST nodes. ### rustpython-compiler diff --git a/compiler/Cargo.toml b/compiler/Cargo.toml index d2f9a5f029..df762199f8 100644 --- a/compiler/Cargo.toml +++ b/compiler/Cargo.toml @@ -1,22 +1,12 @@ [package] -name = "rustpython-compiler-core" +name = "rustpython-compiler" version = "0.1.2" -description = "Compiler for python code into bytecode for the rustpython VM." +description = "A usability wrapper around rustpython-parser and rustpython-compiler-core" authors = ["RustPython Team"] -repository = "https://github.com/RustPython/RustPython" -license = "MIT" edition = "2021" [dependencies] -indexmap = "1.8.1" -itertools = "0.10.3" -rustpython-bytecode = { path = "../bytecode", version = "0.1.1" } -rustpython-ast = { path = "../ast", features = ["unparse"] } -num-complex = { version = "0.4.0", features = ["serde"] } -num-traits = "0.2.14" -log = "0.4.16" -ahash = "0.7.6" - -[dev-dependencies] -rustpython-parser = { path = "../parser" } -insta = "1.14.0" +thiserror = "1.0" +rustpython-codegen = { path = "codegen" } +rustpython-parser = { path = "parser" } +rustpython-bytecode = { path = "bytecode" } diff --git a/ast/Cargo.toml b/compiler/ast/Cargo.toml similarity index 83% rename from ast/Cargo.toml rename to compiler/ast/Cargo.toml index 996a01d892..144ba6214f 100644 --- a/ast/Cargo.toml +++ b/compiler/ast/Cargo.toml @@ -12,5 +12,5 @@ unparse = ["rustpython-common"] [dependencies] num-bigint = "0.4.3" -rustpython-common = { path = "../common", optional = true } +rustpython-common = { path = "../../common", optional = true } rustpython-bytecode = { path = "../bytecode"} diff --git a/ast/Python.asdl b/compiler/ast/Python.asdl similarity index 100% rename from ast/Python.asdl rename to compiler/ast/Python.asdl diff --git a/ast/asdl.py b/compiler/ast/asdl.py similarity index 100% rename from ast/asdl.py rename to compiler/ast/asdl.py diff --git a/ast/asdl_rs.py b/compiler/ast/asdl_rs.py similarity index 100% rename from ast/asdl_rs.py rename to compiler/ast/asdl_rs.py diff --git a/ast/src/ast_gen.rs b/compiler/ast/src/ast_gen.rs similarity index 100% rename from ast/src/ast_gen.rs rename to compiler/ast/src/ast_gen.rs diff --git a/ast/src/constant.rs b/compiler/ast/src/constant.rs similarity index 100% rename from ast/src/constant.rs rename to compiler/ast/src/constant.rs diff --git a/ast/src/fold_helpers.rs b/compiler/ast/src/fold_helpers.rs similarity index 100% rename from ast/src/fold_helpers.rs rename to compiler/ast/src/fold_helpers.rs diff --git a/ast/src/impls.rs b/compiler/ast/src/impls.rs similarity index 100% rename from ast/src/impls.rs rename to compiler/ast/src/impls.rs diff --git a/ast/src/lib.rs b/compiler/ast/src/lib.rs similarity index 100% rename from ast/src/lib.rs rename to compiler/ast/src/lib.rs diff --git a/ast/src/location.rs b/compiler/ast/src/location.rs similarity index 100% rename from ast/src/location.rs rename to compiler/ast/src/location.rs diff --git a/ast/src/unparse.rs b/compiler/ast/src/unparse.rs similarity index 100% rename from ast/src/unparse.rs rename to compiler/ast/src/unparse.rs diff --git a/bytecode/Cargo.toml b/compiler/bytecode/Cargo.toml similarity index 100% rename from bytecode/Cargo.toml rename to compiler/bytecode/Cargo.toml diff --git a/bytecode/src/lib.rs b/compiler/bytecode/src/lib.rs similarity index 100% rename from bytecode/src/lib.rs rename to compiler/bytecode/src/lib.rs diff --git a/compiler/codegen/Cargo.toml b/compiler/codegen/Cargo.toml new file mode 100644 index 0000000000..3a770df3cf --- /dev/null +++ b/compiler/codegen/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "rustpython-codegen" +version = "0.1.2" +description = "Compiler for python code into bytecode for the rustpython VM." +authors = ["RustPython Team"] +repository = "https://github.com/RustPython/RustPython" +license = "MIT" +edition = "2021" + +[dependencies] +indexmap = "1.8.1" +itertools = "0.10.3" +rustpython-bytecode = { path = "../bytecode", version = "0.1.1" } +rustpython-ast = { path = "../ast", features = ["unparse"] } +num-complex = { version = "0.4.0", features = ["serde"] } +num-traits = "0.2.14" +log = "0.4.16" +ahash = "0.7.6" + +[dev-dependencies] +rustpython-parser = { path = "../parser" } +insta = "1.14.0" diff --git a/compiler/src/compile.rs b/compiler/codegen/src/compile.rs similarity index 99% rename from compiler/src/compile.rs rename to compiler/codegen/src/compile.rs index 6e5cbfc8ad..895e86904f 100644 --- a/compiler/src/compile.rs +++ b/compiler/codegen/src/compile.rs @@ -5,13 +5,11 @@ //! //! -use crate::ir::{self, CodeInfo}; -pub use crate::mode::Mode; -use crate::symboltable::{make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable}; -use crate::IndexSet; use crate::{ error::{CompileError, CompileErrorType}, - symboltable, + ir, + symboltable::{self, make_symbol_table, make_symbol_table_expr, SymbolScope, SymbolTable}, + IndexSet, }; use itertools::Itertools; use num_complex::Complex64; @@ -20,6 +18,8 @@ use rustpython_ast as ast; use rustpython_bytecode::{self as bytecode, CodeObject, ConstantData, Instruction}; use std::borrow::Cow; +pub use crate::mode::Mode; + type CompileResult = Result; #[derive(PartialEq, Eq, Clone, Copy)] @@ -61,7 +61,7 @@ fn is_forbidden_name(name: &str) -> bool { /// Main structure holding the state of compilation. struct Compiler { - code_stack: Vec, + code_stack: Vec, symbol_table_stack: Vec, source_path: String, current_source_location: ast::Location, @@ -199,7 +199,7 @@ pub fn compile_expression( impl Compiler { fn new(opts: CompileOpts, source_path: String, code_name: String) -> Self { - let module_code = CodeInfo { + let module_code = ir::CodeInfo { flags: bytecode::CodeFlags::NEW_LOCALS, posonlyarg_count: 0, arg_count: 0, @@ -278,7 +278,7 @@ impl Compiler { self.symbol_table_stack.push(table); - let info = CodeInfo { + let info = ir::CodeInfo { flags, posonlyarg_count, arg_count, @@ -318,7 +318,7 @@ impl Compiler { fn _name_inner( &mut self, name: &str, - cache: impl FnOnce(&mut CodeInfo) -> &mut IndexSet, + cache: impl FnOnce(&mut ir::CodeInfo) -> &mut IndexSet, ) -> bytecode::NameIdx { let name = self.mangle(name); let cache = cache(self.current_codeinfo()); @@ -2583,7 +2583,7 @@ impl Compiler { self.emit(Instruction::LoadConst { idx }) } - fn current_codeinfo(&mut self) -> &mut CodeInfo { + fn current_codeinfo(&mut self) -> &mut ir::CodeInfo { self.code_stack.last_mut().expect("no code on stack") } diff --git a/compiler/src/error.rs b/compiler/codegen/src/error.rs similarity index 100% rename from compiler/src/error.rs rename to compiler/codegen/src/error.rs diff --git a/compiler/src/ir.rs b/compiler/codegen/src/ir.rs similarity index 100% rename from compiler/src/ir.rs rename to compiler/codegen/src/ir.rs diff --git a/compiler/codegen/src/lib.rs b/compiler/codegen/src/lib.rs new file mode 100644 index 0000000000..3e3aca8f32 --- /dev/null +++ b/compiler/codegen/src/lib.rs @@ -0,0 +1,15 @@ +//! Compile a Python AST or source code into bytecode consumable by RustPython. +#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] +#![doc(html_root_url = "https://docs.rs/rustpython-compiler/")] + +#[macro_use] +extern crate log; + +type IndexMap = indexmap::IndexMap; +type IndexSet = indexmap::IndexSet; + +pub mod compile; +pub mod error; +pub mod ir; +pub mod mode; +pub mod symboltable; diff --git a/compiler/src/mode.rs b/compiler/codegen/src/mode.rs similarity index 100% rename from compiler/src/mode.rs rename to compiler/codegen/src/mode.rs diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap new file mode 100644 index 0000000000..93eb0ea6ee --- /dev/null +++ b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap @@ -0,0 +1,14 @@ +--- +source: compiler/codegen/src/compile.rs +expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")" +--- + 1 0 LoadConst (True) + 1 JumpIfFalse (6) + 2 LoadConst (False) + 3 JumpIfFalse (6) + 4 LoadConst (False) + 5 JumpIfFalse (6) + + 2 >> 6 LoadConst (None) + 7 ReturnValue + diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap new file mode 100644 index 0000000000..9594af69ee --- /dev/null +++ b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap @@ -0,0 +1,16 @@ +--- +source: compiler/codegen/src/compile.rs +expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")" +--- + 1 0 LoadConst (True) + 1 JumpIfFalse (4) + 2 LoadConst (False) + 3 JumpIfTrue (8) + >> 4 LoadConst (False) + 5 JumpIfFalse (8) + 6 LoadConst (True) + 7 JumpIfFalse (8) + + 2 >> 8 LoadConst (None) + 9 ReturnValue + diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap new file mode 100644 index 0000000000..dd582b7d7c --- /dev/null +++ b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap @@ -0,0 +1,14 @@ +--- +source: compiler/codegen/src/compile.rs +expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")" +--- + 1 0 LoadConst (True) + 1 JumpIfTrue (6) + 2 LoadConst (False) + 3 JumpIfTrue (6) + 4 LoadConst (False) + 5 JumpIfFalse (6) + + 2 >> 6 LoadConst (None) + 7 ReturnValue + diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap new file mode 100644 index 0000000000..33180c7c6c --- /dev/null +++ b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap @@ -0,0 +1,87 @@ +--- +source: compiler/codegen/src/compile.rs +expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with woohoo():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" +--- + 1 0 SetupLoop (69) + 1 LoadNameAny (0, StopIteration) + 2 LoadConst ("spam") + 3 CallFunctionPositional (1) + 4 LoadNameAny (1, StopAsyncIteration) + 5 LoadConst ("ham") + 6 CallFunctionPositional (1) + 7 BuildTuple (2, false) + 8 GetIter + >> 9 ForIter (68) + 10 StoreLocal (2, stop_exc) + + 2 11 LoadNameAny (3, self) + 12 LoadMethod (subTest) + 13 LoadNameAny (5, type) + 14 LoadNameAny (2, stop_exc) + 15 CallFunctionPositional (1) + 16 LoadConst (("type")) + 17 CallMethodKeyword (1) + 18 SetupWith (65) + 19 Pop + + 3 20 SetupExcept (40) + + 4 21 LoadNameAny (6, woohoo) + 22 CallFunctionPositional (0) + 23 BeforeAsyncWith + 24 GetAwaitable + 25 LoadConst (None) + 26 YieldFrom + 27 SetupAsyncWith (33) + 28 Pop + + 5 29 LoadNameAny (2, stop_exc) + 30 Raise (Raise) + + 4 31 PopBlock + 32 EnterFinally + >> 33 WithCleanupStart + 34 GetAwaitable + 35 LoadConst (None) + 36 YieldFrom + 37 WithCleanupFinish + 38 PopBlock + 39 Jump (54) + >> 40 Duplicate + + 6 41 LoadNameAny (7, Exception) + 42 TestOperation (ExceptionMatch) + 43 JumpIfFalse (53) + 44 StoreLocal (8, ex) + + 7 45 LoadNameAny (3, self) + 46 LoadMethod (assertIs) + 47 LoadNameAny (8, ex) + 48 LoadNameAny (2, stop_exc) + 49 CallMethodPositional (2) + 50 Pop + 51 PopException + 52 Jump (63) + >> 53 Raise (Reraise) + + 9 >> 54 LoadNameAny (3, self) + 55 LoadMethod (fail) + 56 LoadConst ("") + + 1 57 LoadNameAny (2, stop_exc) + 58 FormatValue (None) + + 9 59 LoadConst (" was suppressed") + 60 BuildString (2) + 61 CallMethodPositional (1) + 62 Pop + + 2 >> 63 PopBlock + 64 EnterFinally + >> 65 WithCleanupStart + 66 WithCleanupFinish + 67 Jump (9) + >> 68 PopBlock + >> 69 LoadConst (None) + 70 ReturnValue + diff --git a/compiler/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap b/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap similarity index 100% rename from compiler/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap rename to compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap diff --git a/compiler/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap b/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap similarity index 100% rename from compiler/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap rename to compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap diff --git a/compiler/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap b/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap similarity index 100% rename from compiler/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap rename to compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap diff --git a/compiler/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap b/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap similarity index 100% rename from compiler/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap rename to compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap diff --git a/compiler/src/symboltable.rs b/compiler/codegen/src/symboltable.rs similarity index 100% rename from compiler/src/symboltable.rs rename to compiler/codegen/src/symboltable.rs diff --git a/parser/Cargo.toml b/compiler/parser/Cargo.toml similarity index 100% rename from parser/Cargo.toml rename to compiler/parser/Cargo.toml diff --git a/parser/README.md b/compiler/parser/README.md similarity index 100% rename from parser/README.md rename to compiler/parser/README.md diff --git a/parser/build.rs b/compiler/parser/build.rs similarity index 97% rename from parser/build.rs rename to compiler/parser/build.rs index 57793dfe64..874b01667e 100644 --- a/parser/build.rs +++ b/compiler/parser/build.rs @@ -77,7 +77,7 @@ fn try_lalrpop(source: &str, target: &str) -> anyhow::Result<()> { } #[cfg(not(feature = "lalrpop"))] - panic!("try: cargo build --manifest-path=parser/Cargo.toml --features=lalrpop"); + panic!("try: cargo build --manifest-path=compiler/parser/Cargo.toml --features=lalrpop"); } fn sha_equal(expected_sha3_str: &str, actual_sha3: &[u8; 32]) -> bool { diff --git a/parser/python.lalrpop b/compiler/parser/python.lalrpop similarity index 100% rename from parser/python.lalrpop rename to compiler/parser/python.lalrpop diff --git a/parser/src/error.rs b/compiler/parser/src/error.rs similarity index 100% rename from parser/src/error.rs rename to compiler/parser/src/error.rs diff --git a/parser/src/fstring.rs b/compiler/parser/src/fstring.rs similarity index 100% rename from parser/src/fstring.rs rename to compiler/parser/src/fstring.rs diff --git a/parser/src/function.rs b/compiler/parser/src/function.rs similarity index 100% rename from parser/src/function.rs rename to compiler/parser/src/function.rs diff --git a/parser/src/lexer.rs b/compiler/parser/src/lexer.rs similarity index 100% rename from parser/src/lexer.rs rename to compiler/parser/src/lexer.rs diff --git a/parser/src/lib.rs b/compiler/parser/src/lib.rs similarity index 100% rename from parser/src/lib.rs rename to compiler/parser/src/lib.rs diff --git a/parser/src/mode.rs b/compiler/parser/src/mode.rs similarity index 100% rename from parser/src/mode.rs rename to compiler/parser/src/mode.rs diff --git a/parser/src/parser.rs b/compiler/parser/src/parser.rs similarity index 100% rename from parser/src/parser.rs rename to compiler/parser/src/parser.rs diff --git a/parser/src/python.rs b/compiler/parser/src/python.rs similarity index 100% rename from parser/src/python.rs rename to compiler/parser/src/python.rs diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap b/compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap rename to compiler/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_class.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_dict_comprehension.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_double_list_comprehension.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_empty.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_f_string.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_generator_comprehension.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_elif_else.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_if_else_generator_comprehension.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_kwargs.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_lambda.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_list_comprehension.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_named_expression_generator_comprehension.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_2.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_print_hello.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_string.snap diff --git a/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap b/compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap rename to compiler/parser/src/snapshots/rustpython_parser__parser__tests__parse_tuples.snap diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_1.snap diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_2.snap diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_f_string_concat_3.snap diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_string_concat.snap diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_1.snap diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_u_f_string_concat_2.snap diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_1.snap diff --git a/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap b/compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap similarity index 100% rename from parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap rename to compiler/parser/src/snapshots/rustpython_parser__string__tests__parse_u_string_concat_2.snap diff --git a/parser/src/string.rs b/compiler/parser/src/string.rs similarity index 100% rename from parser/src/string.rs rename to compiler/parser/src/string.rs diff --git a/parser/src/token.rs b/compiler/parser/src/token.rs similarity index 100% rename from parser/src/token.rs rename to compiler/parser/src/token.rs diff --git a/compiler/porcelain/Cargo.toml b/compiler/porcelain/Cargo.toml deleted file mode 100644 index 56bf737ed9..0000000000 --- a/compiler/porcelain/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "rustpython-compiler" -version = "0.1.2" -description = "A usability wrapper around rustpython-parser and rustpython-compiler-core" -authors = ["RustPython Team"] -edition = "2021" - -[dependencies] -thiserror = "1.0" -rustpython-compiler-core = { path = ".." } -rustpython-parser = { path = "../../parser" } -rustpython-bytecode = { path = "../../bytecode" } diff --git a/compiler/porcelain/src/lib.rs b/compiler/porcelain/src/lib.rs deleted file mode 100644 index c14c0ba394..0000000000 --- a/compiler/porcelain/src/lib.rs +++ /dev/null @@ -1,122 +0,0 @@ -use rustpython_bytecode::CodeObject; -use rustpython_compiler_core::{compile, symboltable}; -use rustpython_parser::ast::{fold::Fold, ConstantOptimizer, Location}; -use rustpython_parser::parser; -use std::fmt; - -pub use compile::{CompileOpts, Mode}; -pub use symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType}; - -#[derive(Debug, thiserror::Error)] -pub enum CompileErrorType { - #[error(transparent)] - Compile(#[from] rustpython_compiler_core::error::CompileErrorType), - #[error(transparent)] - Parse(#[from] rustpython_parser::error::ParseErrorType), -} - -#[derive(Debug, thiserror::Error)] -pub struct CompileError { - pub error: CompileErrorType, - pub statement: Option, - pub source_path: String, - pub location: Location, -} - -impl fmt::Display for CompileError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let loc = self.location; - if let Some(ref stmt) = self.statement { - // visualize the error when location and statement are provided - write!( - f, - "{}", - loc.visualize(stmt, &format_args!("{} at {}", self.error, loc)) - ) - } else { - write!(f, "{} at {}", self.error, loc) - } - } -} - -impl CompileError { - fn from_compile(error: rustpython_compiler_core::error::CompileError, source: &str) -> Self { - CompileError { - error: error.error.into(), - location: error.location, - source_path: error.source_path, - statement: get_statement(source, error.location), - } - } - fn from_parse( - error: rustpython_parser::error::ParseError, - source: &str, - source_path: String, - ) -> Self { - CompileError { - error: error.error.into(), - location: error.location, - source_path, - statement: get_statement(source, error.location), - } - } - fn from_symtable( - error: symboltable::SymbolTableError, - source: &str, - source_path: String, - ) -> Self { - Self::from_compile(error.into_compile_error(source_path), source) - } -} - -/// Compile a given sourcecode into a bytecode object. -pub fn compile( - source: &str, - mode: compile::Mode, - source_path: String, - opts: CompileOpts, -) -> Result { - let parser_mode = match mode { - compile::Mode::Exec => parser::Mode::Module, - compile::Mode::Eval => parser::Mode::Expression, - compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive, - }; - let mut ast = match parser::parse(source, parser_mode) { - Ok(x) => x, - Err(e) => return Err(CompileError::from_parse(e, source, source_path)), - }; - if opts.optimize > 0 { - ast = ConstantOptimizer::new() - .fold_mod(ast) - .unwrap_or_else(|e| match e {}); - } - compile::compile_top(&ast, source_path, mode, opts) - .map_err(|e| CompileError::from_compile(e, source)) -} - -pub fn compile_symtable( - source: &str, - mode: compile::Mode, - source_path: &str, -) -> Result { - let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned()); - let res = match mode { - compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => { - let ast = parser::parse_program(source).map_err(parse_err)?; - symboltable::make_symbol_table(&ast) - } - compile::Mode::Eval => { - let expr = parser::parse_expression(source).map_err(parse_err)?; - symboltable::make_symbol_table_expr(&expr) - } - }; - res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned())) -} - -fn get_statement(source: &str, loc: Location) -> Option { - if loc.column() == 0 || loc.row() == 0 { - return None; - } - let line = source.split('\n').nth(loc.row() - 1)?.to_owned(); - Some(line + "\n") -} diff --git a/compiler/src/lib.rs b/compiler/src/lib.rs index 3e3aca8f32..1a659a884d 100644 --- a/compiler/src/lib.rs +++ b/compiler/src/lib.rs @@ -1,15 +1,124 @@ -//! Compile a Python AST or source code into bytecode consumable by RustPython. -#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")] -#![doc(html_root_url = "https://docs.rs/rustpython-compiler/")] +use rustpython_bytecode::CodeObject; +use rustpython_codegen::{compile, symboltable}; +use rustpython_parser::{ + ast::{fold::Fold, ConstantOptimizer, Location}, + parser, +}; +use std::fmt; -#[macro_use] -extern crate log; +pub use compile::{CompileOpts, Mode}; +pub use symboltable::{Symbol, SymbolScope, SymbolTable, SymbolTableType}; -type IndexMap = indexmap::IndexMap; -type IndexSet = indexmap::IndexSet; +#[derive(Debug, thiserror::Error)] +pub enum CompileErrorType { + #[error(transparent)] + Compile(#[from] rustpython_codegen::error::CompileErrorType), + #[error(transparent)] + Parse(#[from] rustpython_parser::error::ParseErrorType), +} -pub mod compile; -pub mod error; -pub mod ir; -pub mod mode; -pub mod symboltable; +#[derive(Debug, thiserror::Error)] +pub struct CompileError { + pub error: CompileErrorType, + pub statement: Option, + pub source_path: String, + pub location: Location, +} + +impl fmt::Display for CompileError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + let loc = self.location; + if let Some(ref stmt) = self.statement { + // visualize the error when location and statement are provided + write!( + f, + "{}", + loc.visualize(stmt, &format_args!("{} at {}", self.error, loc)) + ) + } else { + write!(f, "{} at {}", self.error, loc) + } + } +} + +impl CompileError { + fn from_compile(error: rustpython_codegen::error::CompileError, source: &str) -> Self { + CompileError { + error: error.error.into(), + location: error.location, + source_path: error.source_path, + statement: get_statement(source, error.location), + } + } + fn from_parse( + error: rustpython_parser::error::ParseError, + source: &str, + source_path: String, + ) -> Self { + CompileError { + error: error.error.into(), + location: error.location, + source_path, + statement: get_statement(source, error.location), + } + } + fn from_symtable( + error: symboltable::SymbolTableError, + source: &str, + source_path: String, + ) -> Self { + Self::from_compile(error.into_compile_error(source_path), source) + } +} + +/// Compile a given sourcecode into a bytecode object. +pub fn compile( + source: &str, + mode: compile::Mode, + source_path: String, + opts: CompileOpts, +) -> Result { + let parser_mode = match mode { + compile::Mode::Exec => parser::Mode::Module, + compile::Mode::Eval => parser::Mode::Expression, + compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive, + }; + let mut ast = match parser::parse(source, parser_mode) { + Ok(x) => x, + Err(e) => return Err(CompileError::from_parse(e, source, source_path)), + }; + if opts.optimize > 0 { + ast = ConstantOptimizer::new() + .fold_mod(ast) + .unwrap_or_else(|e| match e {}); + } + compile::compile_top(&ast, source_path, mode, opts) + .map_err(|e| CompileError::from_compile(e, source)) +} + +pub fn compile_symtable( + source: &str, + mode: compile::Mode, + source_path: &str, +) -> Result { + let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned()); + let res = match mode { + compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => { + let ast = parser::parse_program(source).map_err(parse_err)?; + symboltable::make_symbol_table(&ast) + } + compile::Mode::Eval => { + let expr = parser::parse_expression(source).map_err(parse_err)?; + symboltable::make_symbol_table_expr(&expr) + } + }; + res.map_err(|e| CompileError::from_symtable(e, source, source_path.to_owned())) +} + +fn get_statement(source: &str, loc: Location) -> Option { + if loc.column() == 0 || loc.row() == 0 { + return None; + } + let line = source.split('\n').nth(loc.row() - 1)?.to_owned(); + Some(line + "\n") +} diff --git a/derive/Cargo.toml b/derive/Cargo.toml index afe1de788e..ef2b232eed 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -15,8 +15,8 @@ syn = { version = "1.0.91", features = ["full", "extra-traits"] } syn-ext = { version = "0.4.0", features = ["full"] } quote = "1.0.18" proc-macro2 = "1.0.37" -rustpython-compiler = { path = "../compiler/porcelain", version = "0.1.1" } -rustpython-bytecode = { path = "../bytecode", version = "0.1.1" } +rustpython-bytecode = { path = "../compiler/bytecode", version = "0.1.1" } +rustpython-compiler = { path = "../compiler", version = "0.1.1" } maplit = "1.0.2" once_cell = "1.10.0" textwrap = { version = "0.15.0", default-features = false } diff --git a/jit/Cargo.toml b/jit/Cargo.toml index b3e54164e6..c04a68545c 100644 --- a/jit/Cargo.toml +++ b/jit/Cargo.toml @@ -15,7 +15,7 @@ cranelift-module = "0.76.0" cranelift-jit = "0.76.0" num-traits = "0.2" libffi = "2.0.0" -rustpython-bytecode = { path = "../bytecode", version = "0.1.2" } +rustpython-bytecode = { path = "../compiler/bytecode", version = "0.1.2" } thiserror = "1.0" [dev-dependencies] diff --git a/scripts/update_asdl.sh b/scripts/update_asdl.sh index 4373b9b628..0f12735477 100755 --- a/scripts/update_asdl.sh +++ b/scripts/update_asdl.sh @@ -3,5 +3,5 @@ set -e cd "$(dirname "$(dirname "$0")")" -python ast/asdl_rs.py -D ast/src/ast_gen.rs -M vm/src/stdlib/ast/gen.rs ast/Python.asdl -rustfmt ast/src/ast_gen.rs vm/src/stdlib/ast/gen.rs +python compiler/ast/asdl_rs.py -D compiler/ast/src/ast_gen.rs -M vm/src/stdlib/ast/gen.rs compiler/ast/Python.asdl +rustfmt compiler/ast/src/ast_gen.rs vm/src/stdlib/ast/gen.rs diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index a80ae7998a..421b5e66de 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -6,7 +6,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -rustpython-parser = { path = "../parser", optional = true } rustpython-derive = { path = "../derive" } rustpython-vm = { path = "../vm" } rustpython-common = { path = "../common" } @@ -81,14 +80,7 @@ page_size = "0.4" termios = "0.3.3" [features] -default = ["compile-parse"] -compile-parse = ["rustpython-vm/compile-parse", "rustpython-parser"] threading = ["rustpython-common/threading", "rustpython-vm/threading"] -parser = ["rustpython-vm/parser"] -ast = ["rustpython-vm/ast"] -compiler = ["rustpython-vm/compiler"] -# compiler = ["rustpython-compiler", "rustpython-compiler-core", "ast"] -# parser = ["rustpython-parser", "ast"] zlib = ["libz-sys", "flate2/zlib"] bz2 = ["bzip2"] diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 521734a114..f85f1a3638 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -109,11 +109,6 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "_statistics" => statistics::make_module, // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, } - // parser related modules: - #[cfg(feature = "rustpython-ast")] - { - "_ast" => ast::make_module, - } #[cfg(any(unix, target_os = "wasi"))] { "fcntl" => fcntl::make_module, diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 4f10db8be9..a1954da576 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -9,7 +9,7 @@ edition = "2021" include = ["src/**/*.rs", "Cargo.toml", "build.rs", "Lib/**/*.py"] [features] -default = ["compile-parse"] +default = ["compiler"] importlib = [] encodings = ["importlib"] vm-tracing-logging = [] @@ -17,26 +17,27 @@ flame-it = ["flame", "flamer"] freeze-stdlib = ["rustpython-pylib/freeze-stdlib"] jit = ["rustpython-jit"] threading = ["rustpython-common/threading"] -compile-parse = ["parser", "compiler"] +compiler = ["parser", "codegen"] ast = ["rustpython-ast"] -compiler = ["rustpython-compiler", "rustpython-compiler-core", "ast"] +codegen = ["rustpython-compiler", "rustpython-codegen", "ast"] parser = ["rustpython-parser", "ast"] [dependencies] +rustpython-compiler = { path = "../compiler", optional = true, version = "0.1.2" } +rustpython-ast = { path = "../compiler/ast", optional = true, version = "0.1" } +rustpython-parser = { path = "../compiler/parser", optional = true, version = "0.1.2" } +rustpython-codegen = { path = "../compiler/codegen", optional = true, version = "0.1.2" } +rustpython-bytecode = { path = "../compiler/bytecode", version = "0.1.2" } +rustpython-common = { path = "../common" } +rustpython-derive = { path = "../derive", version = "0.1.2" } +rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" } +rustpython-pylib = { path = "pylib-crate", version = "0.1.0" } + num-complex = { version = "0.4.0", features = ["serde"] } num-bigint = { version = "0.4.3", features = ["serde"] } num-traits = "0.2.14" num-integer = "0.1.44" num-rational = "0.4.0" -rustpython-common = { path = "../common" } -rustpython-derive = { path = "../derive", version = "0.1.2" } -rustpython-ast = { path = "../ast", optional = true, version = "0.1" } -rustpython-parser = { path = "../parser", optional = true, version = "0.1.2" } -rustpython-compiler = { path = "../compiler/porcelain", optional = true, version = "0.1.2" } -rustpython-compiler-core = { path = "../compiler", optional = true, version = "0.1.2" } -rustpython-bytecode = { path = "../bytecode", version = "0.1.2" } -rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" } -rustpython-pylib = { path = "pylib-crate", version = "0.1.0" } rand = "0.8.5" getrandom = { version = "0.2.6", features = ["js"] } log = "0.4.16" diff --git a/vm/pylib-crate/Cargo.toml b/vm/pylib-crate/Cargo.toml index c81b3eee69..9e159156bb 100644 --- a/vm/pylib-crate/Cargo.toml +++ b/vm/pylib-crate/Cargo.toml @@ -13,7 +13,7 @@ freeze-stdlib = [] [dependencies] rustpython-derive = { version = "0.1.2", path = "../../derive" } -rustpython-bytecode = { version = "0.1.2", path = "../../bytecode" } +rustpython-bytecode = { version = "0.1.2", path = "../../compiler/bytecode" } [build-dependencies] glob = "0.3" diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 26c6d95f9f..9fd8f9d10b 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -278,10 +278,9 @@ pub(crate) fn compile( ) -> PyResult { let opts = vm.compile_opts(); let ast = Node::ast_from_object(vm, object)?; - let code = - rustpython_compiler_core::compile::compile_top(&ast, filename.to_owned(), mode, opts) - // TODO: use vm.new_syntax_error() - .map_err(|err| vm.new_value_error(err.to_string()))?; + let code = rustpython_codegen::compile::compile_top(&ast, filename.to_owned(), mode, opts) + // TODO: use vm.new_syntax_error() + .map_err(|err| vm.new_value_error(err.to_string()))?; Ok(vm.ctx.new_code(code).into()) } diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 92cd955b5f..2b8f2f5f0c 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -16,11 +16,11 @@ freeze-stdlib = ["rustpython-vm/freeze-stdlib"] no-start-func = [] [dependencies] -rustpython-parser = { path = "../../parser" } +rustpython-parser = { path = "../../compiler/parser" } rustpython-common = { path = "../../common" } # make sure no threading! otherwise wasm build will fail -rustpython-vm = { path = "../../vm", default-features = false, features = ["compile-parse", "encodings"] } -rustpython-stdlib = { path = "../../stdlib", default-features = false, features = ["compile-parse"], optional = true } +rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler", "encodings"] } +rustpython-stdlib = { path = "../../stdlib", default-features = false, optional = true } wasm-bindgen = "0.2" wasm-bindgen-futures = "0.4" serde-wasm-bindgen = "0.3.1"