mschoi/RustPython
1
0
Fork 0
forked from Rust-related/RustPython
Code Pull Requests Actions Activity

Adapt byte-offset location

Browse Source
This commit is contained in:
Jeong YunWon
2023-05-07 13:11:14 +09:00
parent a5ed4070bf
commit f1b261b4f8
21 changed files with 864 additions and 562 deletions
Download Patch File Download Diff File Expand all files Collapse all files

247
Cargo.lock generated
View File

@@ -147,6 +147,12 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24a6904aef64d73cf10ab17ebace7befb918b82164785cb89907993be7f83813"
[[package]]
name = "blake2"
version = "0.10.6"
@@ -260,7 +266,7 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
dependencies = [
"ansi_term",
"atty",
"bitflags",
"bitflags 1.3.2",
"strsim",
"textwrap 0.11.0",
"unicode-width",
@@ -879,9 +885,9 @@ dependencies = [
[[package]]
name = "getrandom"
version = "0.2.8"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
dependencies = [
"cfg-if",
"js-sys",
@@ -1019,9 +1025,9 @@ dependencies = [
[[package]]
name = "is-macro"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c068d4c6b922cd6284c609cfa6dec0e41615c9c5a1a4ba729a970d8daba05fb"
checksum = "8a7d079e129b77477a49c5c4f1cfe9ce6c2c909ef52520693e8e811a714c7b20"
dependencies = [
"Inflector",
"pmutil",
@@ -1075,6 +1081,28 @@ dependencies = [
"cpufeatures",
]
[[package]]
name = "lalrpop"
version = "0.19.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a1cbf952127589f2851ab2046af368fd20645491bb4b376f04b7f94d7a9837b"
dependencies = [
"ascii-canvas",
"bit-set",
"diff",
"ena",
"is-terminal",
"itertools",
"lalrpop-util 0.19.12",
"petgraph",
"regex",
"regex-syntax 0.6.28",
"string_cache",
"term",
"tiny-keccak",
"unicode-xid",
]
[[package]]
name = "lalrpop"
version = "0.20.0"
@@ -1087,7 +1115,7 @@ dependencies = [
"ena",
"is-terminal",
"itertools",
"lalrpop-util",
"lalrpop-util 0.20.0",
"petgraph",
"regex",
"regex-syntax 0.7.1",
@@ -1097,6 +1125,15 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "lalrpop-util"
version = "0.19.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3c48237b9604c5a4702de6b824e02006c3214327564636aef27c1028a8fa0ed"
dependencies = [
"regex",
]
[[package]]
name = "lalrpop-util"
version = "0.20.0"
@@ -1354,7 +1391,7 @@ version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cc",
"cfg-if",
"libc",
@@ -1367,7 +1404,7 @@ version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cfg-if",
"libc",
"memoffset 0.7.1",
@@ -1484,7 +1521,7 @@ version = "0.10.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cfg-if",
"foreign-types",
"libc",
@@ -1822,7 +1859,7 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
"bitflags 1.3.2",
]
[[package]]
@@ -1883,7 +1920,7 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"libc",
"mach",
"winapi",
@@ -1911,6 +1948,53 @@ dependencies = [
"syn-ext",
]
[[package]]
name = "ruff_python_ast"
version = "0.0.0"
source = "git+https://github.com/youknowone/ruff.git?rev=583df5c1fa43b2732896219f8ab425116c140c80#583df5c1fa43b2732896219f8ab425116c140c80"
dependencies = [
"anyhow",
"bitflags 2.2.1",
"is-macro",
"itertools",
"log",
"memchr",
"num-bigint",
"num-traits",
"once_cell",
"regex",
"ruff_rustpython",
"ruff_text_size 0.0.0 (git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a)",
"rustc-hash",
"rustpython-common 0.2.0 (git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a)",
"rustpython-parser 0.2.0 (git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a)",
"smallvec",
]
[[package]]
name = "ruff_rustpython"
version = "0.0.0"
source = "git+https://github.com/youknowone/ruff.git?rev=583df5c1fa43b2732896219f8ab425116c140c80#583df5c1fa43b2732896219f8ab425116c140c80"
dependencies = [
"anyhow",
"once_cell",
"rustpython-common 0.2.0 (git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a)",
"rustpython-parser 0.2.0 (git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a)",
]
[[package]]
name = "ruff_text_size"
version = "0.0.0"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d#09a6afdd04587ccc8841a88e34890c9409d8649d"
[[package]]
name = "ruff_text_size"
version = "0.0.0"
source = "git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a#c3147d2c1524ebd0e90cf1c2938d770314fd5a5a"
dependencies = [
"serde",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
@@ -1932,7 +2016,7 @@ version = "0.37.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"errno",
"io-lifetimes",
"libc",
@@ -1957,7 +2041,7 @@ dependencies = [
"log",
"python3-sys",
"rustpython-compiler",
"rustpython-parser",
"rustpython-parser 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-pylib",
"rustpython-stdlib",
"rustpython-vm",
@@ -1967,28 +2051,37 @@ dependencies = [
[[package]]
name = "rustpython-ast"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=7b8844bd3e470d6b835147ecb445202130ec18d8#7b8844bd3e470d6b835147ecb445202130ec18d8"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d#09a6afdd04587ccc8841a88e34890c9409d8649d"
dependencies = [
"num-bigint",
"rustpython-compiler-core",
"rustpython-compiler-core 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-literal",
]
[[package]]
name = "rustpython-ast"
version = "0.2.0"
source = "git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a#c3147d2c1524ebd0e90cf1c2938d770314fd5a5a"
dependencies = [
"num-bigint",
"ruff_text_size 0.0.0 (git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a)",
]
[[package]]
name = "rustpython-codegen"
version = "0.2.0"
dependencies = [
"ahash",
"bitflags",
"bitflags 1.3.2",
"indexmap",
"insta",
"itertools",
"log",
"num-complex",
"num-traits",
"rustpython-ast",
"rustpython-compiler-core",
"rustpython-parser",
"rustpython-ast 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-compiler-core 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-parser 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
]
[[package]]
@@ -1996,7 +2089,7 @@ name = "rustpython-common"
version = "0.2.0"
dependencies = [
"ascii",
"bitflags",
"bitflags 1.3.2",
"bstr",
"cfg-if",
"itertools",
@@ -2015,26 +2108,66 @@ dependencies = [
"widestring",
]
[[package]]
name = "rustpython-common"
version = "0.2.0"
source = "git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a#c3147d2c1524ebd0e90cf1c2938d770314fd5a5a"
dependencies = [
"ascii",
"bitflags 1.3.2",
"bstr",
"cfg-if",
"getrandom",
"hexf-parse",
"itertools",
"lexical-parse-float",
"libc",
"lock_api",
"num-bigint",
"num-traits",
"once_cell",
"radium",
"rand",
"siphasher",
"unic-ucd-category",
"volatile",
"widestring",
]
[[package]]
name = "rustpython-compiler"
version = "0.2.0"
dependencies = [
"rustpython-codegen",
"rustpython-compiler-core",
"rustpython-parser",
"rustpython-compiler-core 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-parser 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
]
[[package]]
name = "rustpython-compiler-core"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=7b8844bd3e470d6b835147ecb445202130ec18d8#7b8844bd3e470d6b835147ecb445202130ec18d8"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d#09a6afdd04587ccc8841a88e34890c9409d8649d"
dependencies = [
"bitflags",
"bstr",
"bitflags 1.3.2",
"itertools",
"lz4_flex",
"num-bigint",
"num-complex",
"ruff_python_ast",
"ruff_text_size 0.0.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
]
[[package]]
name = "rustpython-compiler-core"
version = "0.2.0"
source = "git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a#c3147d2c1524ebd0e90cf1c2938d770314fd5a5a"
dependencies = [
"bitflags 1.3.2",
"itertools",
"lz4_flex",
"num-bigint",
"num-complex",
"ruff_text_size 0.0.0 (git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a)",
]
[[package]]
@@ -2056,7 +2189,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
"rustpython-compiler-core",
"rustpython-compiler-core 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-doc",
"syn",
"syn-ext",
@@ -2081,7 +2214,7 @@ dependencies = [
"cranelift-module",
"libffi",
"num-traits",
"rustpython-compiler-core",
"rustpython-compiler-core 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-derive",
"thiserror",
]
@@ -2089,7 +2222,7 @@ dependencies = [
[[package]]
name = "rustpython-literal"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=7b8844bd3e470d6b835147ecb445202130ec18d8#7b8844bd3e470d6b835147ecb445202130ec18d8"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d#09a6afdd04587ccc8841a88e34890c9409d8649d"
dependencies = [
"hexf-parse",
"lexical-parse-float",
@@ -2100,21 +2233,45 @@ dependencies = [
[[package]]
name = "rustpython-parser"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=7b8844bd3e470d6b835147ecb445202130ec18d8#7b8844bd3e470d6b835147ecb445202130ec18d8"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d#09a6afdd04587ccc8841a88e34890c9409d8649d"
dependencies = [
"ahash",
"anyhow",
"itertools",
"lalrpop",
"lalrpop-util",
"lalrpop 0.20.0",
"lalrpop-util 0.20.0",
"log",
"num-bigint",
"num-traits",
"phf",
"phf_codegen",
"rustc-hash",
"rustpython-ast",
"rustpython-compiler-core",
"rustpython-ast 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-compiler-core 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"tiny-keccak",
"unic-emoji-char",
"unic-ucd-ident",
"unicode_names2",
]
[[package]]
name = "rustpython-parser"
version = "0.2.0"
source = "git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a#c3147d2c1524ebd0e90cf1c2938d770314fd5a5a"
dependencies = [
"anyhow",
"itertools",
"lalrpop 0.19.12",
"lalrpop-util 0.19.12",
"log",
"num-bigint",
"num-traits",
"phf",
"phf_codegen",
"ruff_text_size 0.0.0 (git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a)",
"rustc-hash",
"rustpython-ast 0.2.0 (git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a)",
"rustpython-compiler-core 0.2.0 (git+https://github.com/charliermarsh/RustPython.git?rev=c3147d2c1524ebd0e90cf1c2938d770314fd5a5a)",
"tiny-keccak",
"unic-emoji-char",
"unic-ucd-ident",
@@ -2126,7 +2283,7 @@ name = "rustpython-pylib"
version = "0.2.0"
dependencies = [
"glob",
"rustpython-compiler-core",
"rustpython-compiler-core 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-derive",
]
@@ -2178,7 +2335,7 @@ dependencies = [
"puruspe",
"rand",
"rand_core",
"rustpython-common",
"rustpython-common 0.2.0",
"rustpython-derive",
"rustpython-vm",
"schannel",
@@ -2210,7 +2367,7 @@ dependencies = [
"ahash",
"ascii",
"atty",
"bitflags",
"bitflags 1.3.2",
"bstr",
"caseless",
"cfg-if",
@@ -2245,15 +2402,15 @@ dependencies = [
"rand",
"result-like",
"rustc_version",
"rustpython-ast",
"rustpython-ast 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-codegen",
"rustpython-common",
"rustpython-common 0.2.0",
"rustpython-compiler",
"rustpython-compiler-core",
"rustpython-compiler-core 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-derive",
"rustpython-jit",
"rustpython-literal",
"rustpython-parser",
"rustpython-parser 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustyline",
"schannel",
"serde",
@@ -2284,8 +2441,8 @@ version = "0.2.0"
dependencies = [
"console_error_panic_hook",
"js-sys",
"rustpython-common",
"rustpython-parser",
"rustpython-common 0.2.0",
"rustpython-parser 0.2.0 (git+https://github.com/youknowone/RustPython-parser.git?rev=09a6afdd04587ccc8841a88e34890c9409d8649d)",
"rustpython-pylib",
"rustpython-stdlib",
"rustpython-vm",
@@ -2308,7 +2465,7 @@ version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dfc8644681285d1fb67a467fb3021bfea306b99b4146b166a1fe3ada965eece"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cfg-if",
"clipboard-win",
"dirs-next",
@@ -2492,7 +2649,7 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a490c5c46c35dba9a6f5e7ee8e4d67e775eb2d2da0f115750b8d10e1c1ac2d28"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"num_enum",
"optional",
]
@@ -2579,7 +2736,7 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d75182f12f490e953596550b65ee31bda7c8e043d9386174b353bda50838c3fd"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"core-foundation",
"system-configuration-sys",
]

8
Cargo.toml
View File

@@ -17,10 +17,10 @@ members = [
]
[workspace.dependencies]
rustpython-literal = { git = "https://github.com/RustPython/Parser.git", rev = "7b8844bd3e470d6b835147ecb445202130ec18d8" }
rustpython-compiler-core = { git = "https://github.com/RustPython/Parser.git", rev = "7b8844bd3e470d6b835147ecb445202130ec18d8" }
rustpython-parser = { git = "https://github.com/RustPython/Parser.git", rev = "7b8844bd3e470d6b835147ecb445202130ec18d8" }
rustpython-ast = { git = "https://github.com/RustPython/Parser.git", rev = "7b8844bd3e470d6b835147ecb445202130ec18d8" }
rustpython-literal = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "09a6afdd04587ccc8841a88e34890c9409d8649d" }
rustpython-compiler-core = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "09a6afdd04587ccc8841a88e34890c9409d8649d" }
rustpython-parser = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "09a6afdd04587ccc8841a88e34890c9409d8649d" }
rustpython-ast = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "09a6afdd04587ccc8841a88e34890c9409d8649d" }
# rustpython-literal = { path = "../RustPython-parser/literal" }
# rustpython-compiler-core = { path = "../RustPython-parser/core" }
# rustpython-parser = { path = "../RustPython-parser/parser" }

213
compiler/codegen/src/compile.rs
View File

@@ -18,8 +18,8 @@ use num_complex::Complex64;
use num_traits::ToPrimitive;
use rustpython_ast as ast;
use rustpython_compiler_core::{
self as bytecode, Arg as OpArgMarker, CodeObject, ConstantData, Instruction, Location, OpArg,
OpArgType,
self as bytecode, source_code::SourceLocation, Arg as OpArgMarker, CodeObject, ConstantData,
Instruction, LineNumber, OpArg, OpArgType,
};
use std::borrow::Cow;
@@ -52,7 +52,7 @@ struct Compiler {
code_stack: Vec<ir::CodeInfo>,
symbol_table_stack: Vec<SymbolTable>,
source_path: String,
current_source_location: Location,
current_source_location: SourceLocation,
qualified_path: Vec<String>,
done_with_future_stmts: bool,
future_annotations: bool,
@@ -90,7 +90,7 @@ impl CompileContext {
/// Compile an ast::Mod produced from rustpython_parser::parse()
pub fn compile_top(
ast: &ast::Mod,
ast: &ast::located::Mod,
source_path: String,
mode: Mode,
opts: CompileOpts,
@@ -131,7 +131,7 @@ fn compile_impl<Ast: ?Sized>(
/// Compile a standard Python program to bytecode
pub fn compile_program(
ast: &[ast::Stmt],
ast: &[ast::located::Stmt],
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
@@ -146,7 +146,7 @@ pub fn compile_program(
/// Compile a Python program to bytecode for the context of a REPL
pub fn compile_program_single(
ast: &[ast::Stmt],
ast: &[ast::located::Stmt],
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
@@ -160,7 +160,7 @@ pub fn compile_program_single(
}
pub fn compile_block_expression(
ast: &[ast::Stmt],
ast: &[ast::located::Stmt],
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
@@ -174,7 +174,7 @@ pub fn compile_block_expression(
}
pub fn compile_expression(
ast: &ast::Expr,
ast: &ast::located::Expr,
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
@@ -210,7 +210,7 @@ impl Compiler {
arg_count: 0,
kwonlyarg_count: 0,
source_path: source_path.clone(),
first_line_number: 0,
first_line_number: LineNumber::MIN,
obj_name: code_name,
blocks: vec![ir::Block::default()],
@@ -225,7 +225,7 @@ impl Compiler {
code_stack: vec![module_code],
symbol_table_stack: Vec::new(),
source_path,
current_source_location: Location::default(),
current_source_location: SourceLocation::MIN,
qualified_path: Vec::new(),
done_with_future_stmts: false,
future_annotations: false,
@@ -239,13 +239,13 @@ impl Compiler {
}
}
fn error(&self, error: CodegenErrorType) -> CodegenError {
fn error(&mut self, error: CodegenErrorType) -> CodegenError {
self.error_loc(error, self.current_source_location)
}
fn error_loc(&self, error: CodegenErrorType, location: Location) -> CodegenError {
fn error_loc(&mut self, error: CodegenErrorType, location: SourceLocation) -> CodegenError {
CodegenError {
error,
location,
location: Some(location),
source_path: self.source_path.clone(),
}
}
@@ -342,7 +342,7 @@ impl Compiler {
fn compile_program(
&mut self,
body: &[ast::Stmt],
body: &[ast::located::Stmt],
symbol_table: SymbolTable,
) -> CompileResult<()> {
let size_before = self.code_stack.len();
@@ -371,7 +371,7 @@ impl Compiler {
fn compile_program_single(
&mut self,
body: &[ast::Stmt],
body: &[ast::located::Stmt],
symbol_table: SymbolTable,
) -> CompileResult<()> {
self.symbol_table_stack.push(symbol_table);
@@ -404,7 +404,7 @@ impl Compiler {
fn compile_block_expr(
&mut self,
body: &[ast::Stmt],
body: &[ast::located::Stmt],
symbol_table: SymbolTable,
) -> CompileResult<()> {
self.symbol_table_stack.push(symbol_table);
@@ -434,7 +434,7 @@ impl Compiler {
// Compile statement in eval mode:
fn compile_eval(
&mut self,
expression: &ast::Expr,
expression: &ast::located::Expr,
symbol_table: SymbolTable,
) -> CompileResult<()> {
self.symbol_table_stack.push(symbol_table);
@@ -443,7 +443,7 @@ impl Compiler {
Ok(())
}
fn compile_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> {
fn compile_statements(&mut self, statements: &[ast::located::Stmt]) -> CompileResult<()> {
for statement in statements {
self.compile_statement(statement)?
}
@@ -462,7 +462,7 @@ impl Compiler {
symboltable::mangle_name(self.class_name.as_deref(), name)
}
fn check_forbidden_name(&self, name: &str, usage: NameUsage) -> CompileResult<()> {
fn check_forbidden_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> {
let msg = match usage {
NameUsage::Store if is_forbidden_name(name) => "cannot assign to",
NameUsage::Delete if is_forbidden_name(name) => "cannot delete",
@@ -554,9 +554,9 @@ impl Compiler {
Ok(())
}
fn compile_statement(&mut self, statement: &ast::Stmt) -> CompileResult<()> {
fn compile_statement(&mut self, statement: &ast::located::Stmt) -> CompileResult<()> {
trace!("Compiling {:?}", statement);
self.set_source_location(statement.start());
self.set_source_location(statement.location());
use ast::{StmtKind::*, *};
match &statement.node {
@@ -602,9 +602,10 @@ impl Compiler {
let from_list = if import_star {
if self.ctx.in_func() {
return Err(
self.error_loc(CodegenErrorType::FunctionImportStar, statement.start())
);
return Err(self.error_loc(
CodegenErrorType::FunctionImportStar,
statement.location(),
));
}
vec![ConstantData::Str {
value: "*".to_owned(),
@@ -805,7 +806,9 @@ impl Compiler {
emit!(self, Instruction::Break { target: end });
}
None => {
return Err(self.error_loc(CodegenErrorType::InvalidBreak, statement.start()));
return Err(
self.error_loc(CodegenErrorType::InvalidBreak, statement.location())
);
}
},
Continue => match self.ctx.loop_data {
@@ -814,13 +817,15 @@ impl Compiler {
}
None => {
return Err(
self.error_loc(CodegenErrorType::InvalidContinue, statement.start())
self.error_loc(CodegenErrorType::InvalidContinue, statement.location())
);
}
},
Return(StmtReturn { value }) => {
if !self.ctx.in_func() {
return Err(self.error_loc(CodegenErrorType::InvalidReturn, statement.start()));
return Err(
self.error_loc(CodegenErrorType::InvalidReturn, statement.location())
);
}
match value {
Some(v) => {
@@ -830,8 +835,10 @@ impl Compiler {
.flags
.contains(bytecode::CodeFlags::IS_GENERATOR)
{
return Err(self
.error_loc(CodegenErrorType::AsyncReturnValue, statement.start()));
return Err(self.error_loc(
CodegenErrorType::AsyncReturnValue,
statement.location(),
));
}
self.compile_expression(v)?;
}
@@ -873,7 +880,7 @@ impl Compiler {
Ok(())
}
fn compile_delete(&mut self, expression: &ast::Expr) -> CompileResult<()> {
fn compile_delete(&mut self, expression: &ast::located::Expr) -> CompileResult<()> {
match &expression.node {
ast::ExprKind::Name(ast::ExprName { id, .. }) => {
self.compile_name(id, NameUsage::Delete)?
@@ -907,7 +914,7 @@ impl Compiler {
fn enter_function(
&mut self,
name: &str,
args: &ast::Arguments,
args: &ast::located::Arguments,
) -> CompileResult<bytecode::MakeFunctionFlags> {
let have_defaults = !args.defaults.is_empty();
if have_defaults {
@@ -974,14 +981,14 @@ impl Compiler {
Ok(func_flags)
}
fn prepare_decorators(&mut self, decorator_list: &[ast::Expr]) -> CompileResult<()> {
fn prepare_decorators(&mut self, decorator_list: &[ast::located::Expr]) -> CompileResult<()> {
for decorator in decorator_list {
self.compile_expression(decorator)?;
}
Ok(())
}
fn apply_decorators(&mut self, decorator_list: &[ast::Expr]) {
fn apply_decorators(&mut self, decorator_list: &[ast::located::Expr]) {
// Apply decorators:
for _ in decorator_list {
emit!(self, Instruction::CallFunctionPositional { nargs: 1 });
@@ -990,10 +997,10 @@ impl Compiler {
fn compile_try_statement(
&mut self,
body: &[ast::Stmt],
handlers: &[ast::Excepthandler],
orelse: &[ast::Stmt],
finalbody: &[ast::Stmt],
body: &[ast::located::Stmt],
handlers: &[ast::located::Excepthandler],
orelse: &[ast::located::Stmt],
finalbody: &[ast::located::Stmt],
) -> CompileResult<()> {
let handler_block = self.new_block();
let finally_block = self.new_block();
@@ -1122,10 +1129,10 @@ impl Compiler {
fn compile_try_star_statement(
&mut self,
_body: &[ast::Stmt],
_handlers: &[ast::Excepthandler],
_orelse: &[ast::Stmt],
_finalbody: &[ast::Stmt],
_body: &[ast::located::Stmt],
_handlers: &[ast::located::Excepthandler],
_orelse: &[ast::located::Stmt],
_finalbody: &[ast::located::Stmt],
) -> CompileResult<()> {
Err(self.error(CodegenErrorType::NotImplementedYet))
}
@@ -1137,10 +1144,10 @@ impl Compiler {
fn compile_function_def(
&mut self,
name: &str,
args: &ast::Arguments,
body: &[ast::Stmt],
decorator_list: &[ast::Expr],
returns: Option<&ast::Expr>, // TODO: use type hint somehow..
args: &ast::located::Arguments,
body: &[ast::located::Stmt],
decorator_list: &[ast::located::Expr],
returns: Option<&ast::located::Expr>, // TODO: use type hint somehow..
is_async: bool,
) -> CompileResult<()> {
// Create bytecode for this function:
@@ -1295,7 +1302,7 @@ impl Compiler {
}
// Python/compile.c find_ann
fn find_ann(body: &[ast::Stmt]) -> bool {
fn find_ann(body: &[ast::located::Stmt]) -> bool {
use ast::StmtKind::*;
for statement in body {
@@ -1329,10 +1336,10 @@ impl Compiler {
fn compile_class_def(
&mut self,
name: &str,
body: &[ast::Stmt],
bases: &[ast::Expr],
keywords: &[ast::Keyword],
decorator_list: &[ast::Expr],
body: &[ast::located::Stmt],
bases: &[ast::located::Expr],
keywords: &[ast::located::Keyword],
decorator_list: &[ast::located::Expr],
) -> CompileResult<()> {
self.prepare_decorators(decorator_list)?;
@@ -1448,9 +1455,9 @@ impl Compiler {
fn compile_while(
&mut self,
test: &ast::Expr,
body: &[ast::Stmt],
orelse: &[ast::Stmt],
test: &ast::located::Expr,
body: &[ast::located::Stmt],
orelse: &[ast::located::Stmt],
) -> CompileResult<()> {
let while_block = self.new_block();
let else_block = self.new_block();
@@ -1479,8 +1486,8 @@ impl Compiler {
fn compile_with(
&mut self,
items: &[ast::Withitem],
body: &[ast::Stmt],
items: &[ast::located::Withitem],
body: &[ast::located::Stmt],
is_async: bool,
) -> CompileResult<()> {
let with_location = self.current_source_location;
@@ -1506,7 +1513,7 @@ impl Compiler {
match &item.optional_vars {
Some(var) => {
self.set_source_location(var.start());
self.set_source_location(var.location());
self.compile_store(var)?;
}
None => {
@@ -1549,10 +1556,10 @@ impl Compiler {
fn compile_for(
&mut self,
target: &ast::Expr,
iter: &ast::Expr,
body: &[ast::Stmt],
orelse: &[ast::Stmt],
target: &ast::located::Expr,
iter: &ast::located::Expr,
body: &[ast::located::Stmt],
orelse: &[ast::located::Stmt],
is_async: bool,
) -> CompileResult<()> {
// Start loop
@@ -1610,8 +1617,8 @@ impl Compiler {
fn compile_match(
&mut self,
subject: &ast::Expr,
cases: &[ast::MatchCase],
subject: &ast::located::Expr,
cases: &[ast::located::MatchCase],
) -> CompileResult<()> {
eprintln!("match subject: {subject:?}");
eprintln!("match cases: {cases:?}");
@@ -1620,9 +1627,9 @@ impl Compiler {
fn compile_chained_comparison(
&mut self,
left: &ast::Expr,
left: &ast::located::Expr,
ops: &[ast::Cmpop],
exprs: &[ast::Expr],
exprs: &[ast::located::Expr],
) -> CompileResult<()> {
assert!(!ops.is_empty());
assert_eq!(exprs.len(), ops.len());
@@ -1706,7 +1713,7 @@ impl Compiler {
Ok(())
}
fn compile_annotation(&mut self, annotation: &ast::Expr) -> CompileResult<()> {
fn compile_annotation(&mut self, annotation: &ast::located::Expr) -> CompileResult<()> {
if self.future_annotations {
self.emit_constant(ConstantData::Str {
value: annotation.to_string(),
@@ -1719,9 +1726,9 @@ impl Compiler {
fn compile_annotated_assign(
&mut self,
target: &ast::Expr,
annotation: &ast::Expr,
value: Option<&ast::Expr>,
target: &ast::located::Expr,
annotation: &ast::located::Expr,
value: Option<&ast::located::Expr>,
) -> CompileResult<()> {
if let Some(value) = value {
self.compile_expression(value)?;
@@ -1752,7 +1759,7 @@ impl Compiler {
Ok(())
}
fn compile_store(&mut self, target: &ast::Expr) -> CompileResult<()> {
fn compile_store(&mut self, target: &ast::located::Expr) -> CompileResult<()> {
match &target.node {
ast::ExprKind::Name(ast::ExprName { id, .. }) => self.store_name(id)?,
ast::ExprKind::Subscript(ast::ExprSubscript { value, slice, .. }) => {
@@ -1783,7 +1790,7 @@ impl Compiler {
.ok_or_else(|| {
self.error_loc(
CodegenErrorType::TooManyStarUnpack,
target.start(),
target.location(),
)
})?;
let args = bytecode::UnpackExArgs { before, after };
@@ -1824,9 +1831,9 @@ impl Compiler {
fn compile_augassign(
&mut self,
target: &ast::Expr,
target: &ast::located::Expr,
op: &ast::Operator,
value: &ast::Expr,
value: &ast::located::Expr,
) -> CompileResult<()> {
enum AugAssignKind<'a> {
Name { id: &'a str },
@@ -1915,7 +1922,7 @@ impl Compiler {
/// (indicated by the condition parameter).
fn compile_jump_if(
&mut self,
expression: &ast::Expr,
expression: &ast::located::Expr,
condition: bool,
target_block: ir::BlockIdx,
) -> CompileResult<()> {
@@ -1998,7 +2005,11 @@ impl Compiler {
/// Compile a boolean operation as an expression.
/// This means, that the last value remains on the stack.
fn compile_bool_op(&mut self, op: &ast::Boolop, values: &[ast::Expr]) -> CompileResult<()> {
fn compile_bool_op(
&mut self,
op: &ast::Boolop,
values: &[ast::located::Expr],
) -> CompileResult<()> {
let after_block = self.new_block();
let (last_value, values) = values.split_last().unwrap();
@@ -2033,8 +2044,8 @@ impl Compiler {
fn compile_dict(
&mut self,
keys: &[Option<ast::Expr>],
values: &[ast::Expr],
keys: &[Option<ast::located::Expr>],
values: &[ast::located::Expr],
) -> CompileResult<()> {
let mut size = 0;
let (packed, unpacked): (Vec<_>, Vec<_>) = keys
@@ -2056,9 +2067,10 @@ impl Compiler {
Ok(())
}
fn compile_expression(&mut self, expression: &ast::Expr) -> CompileResult<()> {
fn compile_expression(&mut self, expression: &ast::located::Expr) -> CompileResult<()> {
trace!("Compiling {:?}", expression);
self.set_source_location(expression.start());
let location = expression.location();
self.set_source_location(location);
use ast::ExprKind::*;
match &expression.node {
@@ -2135,7 +2147,7 @@ impl Compiler {
self.compile_dict(keys, values)?;
}
Slice(ast::ExprSlice { lower, upper, step }) => {
let mut compile_bound = |bound: Option<&ast::Expr>| match bound {
let mut compile_bound = |bound: Option<&ast::located::Expr>| match bound {
Some(exp) => self.compile_expression(exp),
None => {
self.emit_constant(ConstantData::None);
@@ -2362,7 +2374,7 @@ impl Compiler {
Ok(())
}
fn compile_keywords(&mut self, keywords: &[ast::Keyword]) -> CompileResult<()> {
fn compile_keywords(&mut self, keywords: &[ast::located::Keyword]) -> CompileResult<()> {
let mut size = 0;
let groupby = keywords.iter().group_by(|e| e.node.arg.is_none());
for (is_unpacking, sub_keywords) in &groupby {
@@ -2394,9 +2406,9 @@ impl Compiler {
fn compile_call(
&mut self,
func: &ast::Expr,
args: &[ast::Expr],
keywords: &[ast::Keyword],
func: &ast::located::Expr,
args: &[ast::located::Expr],
keywords: &[ast::located::Keyword],
) -> CompileResult<()> {
let method =
if let ast::ExprKind::Attribute(ast::ExprAttribute { value, attr, .. }) = &func.node {
@@ -2439,8 +2451,8 @@ impl Compiler {
fn compile_call_inner(
&mut self,
additional_positional: u32,
args: &[ast::Expr],
keywords: &[ast::Keyword],
args: &[ast::located::Expr],
keywords: &[ast::located::Keyword],
) -> CompileResult<CallType> {
let count = (args.len() + keywords.len()).to_u32() + additional_positional;
@@ -2498,7 +2510,7 @@ impl Compiler {
fn gather_elements(
&mut self,
before: u32,
elements: &[ast::Expr],
elements: &[ast::located::Expr],
) -> CompileResult<(u32, bool)> {
// First determine if we have starred elements:
let has_stars = elements
@@ -2549,7 +2561,7 @@ impl Compiler {
Ok((size, has_stars))
}
fn compile_comprehension_element(&mut self, element: &ast::Expr) -> CompileResult<()> {
fn compile_comprehension_element(&mut self, element: &ast::located::Expr) -> CompileResult<()> {
self.compile_expression(element).map_err(|e| {
if let CodegenErrorType::InvalidStarExpr = e.error {
self.error(CodegenErrorType::SyntaxError(
@@ -2565,7 +2577,7 @@ impl Compiler {
&mut self,
name: &str,
init_collection: Option<Instruction>,
generators: &[ast::Comprehension],
generators: &[ast::located::Comprehension],
compile_element: &dyn Fn(&mut Self) -> CompileResult<()>,
) -> CompileResult<()> {
let prev_ctx = self.ctx;
@@ -2684,7 +2696,10 @@ impl Compiler {
Ok(())
}
fn compile_future_features(&mut self, features: &[ast::Alias]) -> Result<(), CodegenError> {
fn compile_future_features(
&mut self,
features: &[ast::located::Alias],
) -> Result<(), CodegenError> {
if self.done_with_future_stmts {
return Err(self.error(CodegenErrorType::InvalidFuturePlacement));
}
@@ -2705,7 +2720,7 @@ impl Compiler {
// Low level helper functions:
fn _emit(&mut self, instr: Instruction, arg: OpArg, target: ir::BlockIdx) {
let location = compile_location(&self.current_source_location);
let location = self.current_source_location;
// TODO: insert source filename
self.current_block().instructions.push(ir::InstructionInfo {
instr,
@@ -2770,12 +2785,13 @@ impl Compiler {
code.current_block = block;
}
fn set_source_location(&mut self, location: Location) {
fn set_source_location(&mut self, location: SourceLocation) {
self.current_source_location = location;
}
fn get_source_line_number(&self) -> u32 {
self.current_source_location.row().to_u32()
fn get_source_line_number(&mut self) -> LineNumber {
let location = self.current_source_location;
location.row
}
fn push_qualified_path(&mut self, name: &str) {
@@ -2811,7 +2827,7 @@ impl EmitArg<bytecode::Label> for ir::BlockIdx {
}
}
fn split_doc(body: &[ast::Stmt]) -> (Option<String>, &[ast::Stmt]) {
fn split_doc(body: &[ast::located::Stmt]) -> (Option<String>, &[ast::located::Stmt]) {
if let Some((val, body_rest)) = body.split_first() {
if let ast::StmtKind::Expr(ast::StmtExpr { value }) = &val.node {
if let Some(doc) = try_get_constant_string(std::slice::from_ref(value)) {
@@ -2822,8 +2838,8 @@ fn split_doc(body: &[ast::Stmt]) -> (Option<String>, &[ast::Stmt]) {
(None, body)
}
fn try_get_constant_string(values: &[ast::Expr]) -> Option<String> {
fn get_constant_string_inner(out_string: &mut String, value: &ast::Expr) -> bool {
fn try_get_constant_string(values: &[ast::located::Expr]) -> Option<String> {
fn get_constant_string_inner(out_string: &mut String, value: &ast::located::Expr) -> bool {
match &value.node {
ast::ExprKind::Constant(ast::ExprConstant {
value: ast::Constant::Str(s),
@@ -2849,10 +2865,6 @@ fn try_get_constant_string(values: &[ast::Expr]) -> Option<String> {
}
}
fn compile_location(location: &Location) -> bytecode::Location {
bytecode::Location::new(location.row(), location.column())
}
fn compile_constant(value: &ast::Constant) -> ConstantData {
match value {
ast::Constant::None => ConstantData::None,
@@ -2892,6 +2904,7 @@ mod tests {
CompileOpts::default(),
"source_path".to_owned(),
"<module>".to_owned(),
source,
);
let ast = parser::parse_program(source, "<test>").unwrap();
let symbol_scope = SymbolTable::scan_program(&ast).unwrap();

2
compiler/codegen/src/error.rs
View File

@@ -1,6 +1,6 @@
use std::fmt;
pub type CodegenError = rustpython_compiler_core::BaseError<CodegenErrorType>;
pub type CodegenError = rustpython_compiler_core::LocatedError<CodegenErrorType>;
#[derive(Debug)]
#[non_exhaustive]

8
compiler/codegen/src/ir.rs
View File

@@ -2,8 +2,8 @@ use std::ops;
use crate::IndexSet;
use rustpython_compiler_core::{
CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label,
Location, OpArg,
source_code::SourceLocation, CodeFlags, CodeObject, CodeUnit, ConstantData,
InstrDisplayContext, Instruction, Label, LineNumber, OpArg,
};
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
@@ -42,7 +42,7 @@ pub struct InstructionInfo {
pub instr: Instruction,
pub arg: OpArg,
pub target: BlockIdx,
pub location: Location,
pub location: SourceLocation,
}
// spell-checker:ignore petgraph
@@ -68,7 +68,7 @@ pub struct CodeInfo {
pub arg_count: u32,
pub kwonlyarg_count: u32,
pub source_path: String,
pub first_line_number: u32,
pub first_line_number: LineNumber,
pub obj_name: String, // Name of the object that created this code object
pub blocks: Vec<Block>,

99
compiler/codegen/src/symboltable.rs
View File

@@ -13,7 +13,10 @@ use crate::{
};
use bitflags::bitflags;
use rustpython_ast as ast;
use rustpython_compiler_core::Location;
use rustpython_compiler_core::{
source_code::{OneIndexed, SourceLocation},
LineNumber,
};
use std::{borrow::Cow, fmt};
/// Captures all symbols in the current scope, and has a list of sub-scopes in this scope.
@@ -26,7 +29,7 @@ pub struct SymbolTable {
pub typ: SymbolTableType,
/// The line number in the source code where this symboltable begins.
pub line_number: usize,
pub line_number: LineNumber,
// Return True if the block is a nested class or function
pub is_nested: bool,
@@ -40,7 +43,7 @@ pub struct SymbolTable {
}
impl SymbolTable {
fn new(name: String, typ: SymbolTableType, line_number: usize, is_nested: bool) -> Self {
fn new(name: String, typ: SymbolTableType, line_number: LineNumber, is_nested: bool) -> Self {
SymbolTable {
name,
typ,
@@ -51,13 +54,13 @@ impl SymbolTable {
}
}
pub fn scan_program(program: &[ast::Stmt]) -> SymbolTableResult<Self> {
pub fn scan_program(program: &[ast::located::Stmt]) -> SymbolTableResult<Self> {
let mut builder = SymbolTableBuilder::new();
builder.scan_statements(program)?;
builder.finish()
}
pub fn scan_expr(expr: &ast::Expr) -> SymbolTableResult<Self> {
pub fn scan_expr(expr: &ast::located::Expr) -> SymbolTableResult<Self> {
let mut builder = SymbolTableBuilder::new();
builder.scan_expression(expr, ExpressionContext::Load)?;
builder.finish()
@@ -160,14 +163,17 @@ impl Symbol {
#[derive(Debug)]
pub struct SymbolTableError {
error: String,
location: Location,
location: Option<SourceLocation>,
}
impl SymbolTableError {
pub fn into_codegen_error(self, source_path: String) -> CodegenError {
CodegenError {
error: CodegenErrorType::SyntaxError(self.error),
location: self.location.with_col_offset(1),
location: self.location.map(|l| SourceLocation {
row: l.row,
column: OneIndexed::MIN,
}),
source_path,
}
}
@@ -318,7 +324,7 @@ impl SymbolTableAnalyzer {
return Err(SymbolTableError {
error: format!("no binding for nonlocal '{}' found", symbol.name),
// TODO: accurate location info, somehow
location: Location::default(),
location: None,
});
}
} else {
@@ -328,7 +334,7 @@ impl SymbolTableAnalyzer {
symbol.name
),
// TODO: accurate location info, somehow
location: Location::default(),
location: None,
});
}
}
@@ -451,7 +457,7 @@ impl SymbolTableAnalyzer {
symbol.name
),
// TODO: accurate location info, somehow
location: Location::default(),
location: None,
});
}
@@ -464,7 +470,7 @@ impl SymbolTableAnalyzer {
return Err(SymbolTableError {
error: "assignment expression within a comprehension cannot be used in a class body".to_string(),
// TODO: accurate location info, somehow
location: Location::default(),
location: None,
});
}
SymbolTableType::Function => {
@@ -493,8 +499,7 @@ impl SymbolTableAnalyzer {
if parent_symbol.flags.contains(SymbolFlags::ITER) {
return Err(SymbolTableError {
error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name),
// TODO: accurate location info, somehow
location: Location::default(),
location: None,
});
}
@@ -560,10 +565,12 @@ impl SymbolTableBuilder {
tables: vec![],
future_annotations: false,
};
this.enter_scope("top", SymbolTableType::Module, 0);
this.enter_scope("top", SymbolTableType::Module, LineNumber::MIN);
this
}
}
impl SymbolTableBuilder {
fn finish(mut self) -> Result<SymbolTable, SymbolTableError> {
assert_eq!(self.tables.len(), 1);
let mut symbol_table = self.tables.pop().unwrap();
@@ -571,7 +578,7 @@ impl SymbolTableBuilder {
Ok(symbol_table)
}
fn enter_scope(&mut self, name: &str, typ: SymbolTableType, line_number: usize) {
fn enter_scope(&mut self, name: &str, typ: SymbolTableType, line_number: LineNumber) {
let is_nested = self
.tables
.last()
@@ -587,44 +594,47 @@ impl SymbolTableBuilder {
self.tables.last_mut().unwrap().sub_tables.push(table);
}
fn scan_statements(&mut self, statements: &[ast::Stmt]) -> SymbolTableResult {
fn scan_statements(&mut self, statements: &[ast::located::Stmt]) -> SymbolTableResult {
for statement in statements {
self.scan_statement(statement)?;
}
Ok(())
}
fn scan_parameters(&mut self, parameters: &[ast::Arg]) -> SymbolTableResult {
fn scan_parameters(&mut self, parameters: &[ast::located::Arg]) -> SymbolTableResult {
for parameter in parameters {
self.scan_parameter(parameter)?;
}
Ok(())
}
fn scan_parameter(&mut self, parameter: &ast::Arg) -> SymbolTableResult {
fn scan_parameter(&mut self, parameter: &ast::located::Arg) -> SymbolTableResult {
let usage = if parameter.node.annotation.is_some() {
SymbolUsage::AnnotationParameter
} else {
SymbolUsage::Parameter
};
self.register_name(&parameter.node.arg, usage, parameter.start())
self.register_name(&parameter.node.arg, usage, parameter.location())
}
fn scan_parameters_annotations(&mut self, parameters: &[ast::Arg]) -> SymbolTableResult {
fn scan_parameters_annotations(
&mut self,
parameters: &[ast::located::Arg],
) -> SymbolTableResult {
for parameter in parameters {
self.scan_parameter_annotation(parameter)?;
}
Ok(())
}
fn scan_parameter_annotation(&mut self, parameter: &ast::Arg) -> SymbolTableResult {
fn scan_parameter_annotation(&mut self, parameter: &ast::located::Arg) -> SymbolTableResult {
if let Some(annotation) = &parameter.node.annotation {
self.scan_annotation(annotation)?;
}
Ok(())
}
fn scan_annotation(&mut self, annotation: &ast::Expr) -> SymbolTableResult {
fn scan_annotation(&mut self, annotation: &ast::located::Expr) -> SymbolTableResult {
if self.future_annotations {
Ok(())
} else {
@@ -632,9 +642,9 @@ impl SymbolTableBuilder {
}
}
fn scan_statement(&mut self, statement: &ast::Stmt) -> SymbolTableResult {
fn scan_statement(&mut self, statement: &ast::located::Stmt) -> SymbolTableResult {
use ast::{StmtKind::*, *};
let location = statement.start();
let location = statement.location();
if let ImportFrom(StmtImportFrom { module, names, .. }) = &statement.node {
if module.as_deref() == Some("__future__") {
for feature in names {
@@ -676,7 +686,7 @@ impl SymbolTableBuilder {
if let Some(expression) = returns {
self.scan_annotation(expression)?;
}
self.enter_function(name, args, location.row())?;
self.enter_function(name, args, location.row)?;
self.scan_statements(body)?;
self.leave_scope();
}
@@ -687,7 +697,7 @@ impl SymbolTableBuilder {
keywords,
decorator_list,
}) => {
self.enter_scope(name, SymbolTableType::Class, location.row());
self.enter_scope(name, SymbolTableType::Class, location.row);
let prev_class = std::mem::replace(&mut self.class_name, Some(name.to_owned()));
self.register_name("__module__", SymbolUsage::Assigned, location)?;
self.register_name("__qualname__", SymbolUsage::Assigned, location)?;
@@ -832,13 +842,10 @@ impl SymbolTableBuilder {
self.scan_statements(orelse)?;
self.scan_statements(finalbody)?;
}
Match(StmtMatch {
subject: _,
cases: _,
}) => {
Match(StmtMatch { subject, cases: _ }) => {
return Err(SymbolTableError {
error: "match expression is not implemented yet".to_owned(),
location: Location::default(),
location: Some(subject.location()),
});
}
Raise(StmtRaise { exc, cause }) => {
@@ -855,7 +862,7 @@ impl SymbolTableBuilder {
fn scan_expressions(
&mut self,
expressions: &[ast::Expr],
expressions: &[ast::located::Expr],
context: ExpressionContext,
) -> SymbolTableResult {
for expression in expressions {
@@ -866,11 +873,11 @@ impl SymbolTableBuilder {
fn scan_expression(
&mut self,
expression: &ast::Expr,
expression: &ast::located::Expr,
context: ExpressionContext,
) -> SymbolTableResult {
use ast::{ExprKind::*, *};
let location = expression.start();
let location = expression.location();
match &expression.node {
BinOp(ExprBinOp { left, right, .. }) => {
self.scan_expression(left, context)?;
@@ -1009,7 +1016,7 @@ impl SymbolTableBuilder {
}
}
Lambda(ExprLambda { args, body }) => {
self.enter_function("lambda", args, expression.start().row())?;
self.enter_function("lambda", args, expression.location().row)?;
match context {
ExpressionContext::IterDefinitionExp => {
self.scan_expression(body, ExpressionContext::IterDefinitionExp)?;
@@ -1032,8 +1039,7 @@ impl SymbolTableBuilder {
if let ExpressionContext::IterDefinitionExp = context {
return Err(SymbolTableError {
error: "assignment expression cannot be used in a comprehension iterable expression".to_string(),
// TODO: accurate location info, somehow
location: Location::default(),
location: Some(target.location()),
});
}
@@ -1068,14 +1074,13 @@ impl SymbolTableBuilder {
fn scan_comprehension(
&mut self,
scope_name: &str,
elt1: &ast::Expr,
elt2: Option<&ast::Expr>,
generators: &[ast::Comprehension],
location: Location,
elt1: &ast::located::Expr,
elt2: Option<&ast::located::Expr>,
generators: &[ast::located::Comprehension],
location: SourceLocation,
) -> SymbolTableResult {
// Comprehensions are compiled as functions, so create a scope for them:
self.enter_scope(scope_name, SymbolTableType::Comprehension, location.row());
self.enter_scope(scope_name, SymbolTableType::Comprehension, location.row);
// Register the passed argument to the generator function as the name ".0"
self.register_name(".0", SymbolUsage::Parameter, location)?;
@@ -1111,8 +1116,8 @@ impl SymbolTableBuilder {
fn enter_function(
&mut self,
name: &str,
args: &ast::Arguments,
line_number: usize,
args: &ast::located::Arguments,
line_number: LineNumber,
) -> SymbolTableResult {
// Evaluate eventual default parameters:
self.scan_expressions(&args.defaults, ExpressionContext::Load)?;
@@ -1150,13 +1155,13 @@ impl SymbolTableBuilder {
&mut self,
name: &str,
role: SymbolUsage,
location: Location,
location: SourceLocation,
) -> SymbolTableResult {
let location = Some(location);
let scope_depth = self.tables.len();
let table = self.tables.last_mut().unwrap();
let name = mangle_name(self.class_name.as_deref(), name);
// Some checks for the symbol that present on this scope level:
let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) {
let flags = &symbol.flags;

17
compiler/src/lib.rs
View File

@@ -2,7 +2,7 @@ use rustpython_codegen::{compile, symboltable};
use rustpython_parser::ast::{fold::Fold, ConstantOptimizer};
pub use rustpython_codegen::compile::CompileOpts;
pub use rustpython_compiler_core::{CodeObject, Mode};
pub use rustpython_compiler_core::{CodeObject, Mode, SourceLocator};
// these modules are out of repository. re-exporting them here for convenience.
pub use rustpython_codegen as codegen;
@@ -45,7 +45,7 @@ impl From<parser::ParseErrorType> for CompileErrorType {
}
}
pub type CompileError = rustpython_compiler_core::BaseError<CompileErrorType>;
pub type CompileError = rustpython_compiler_core::LocatedError<CompileErrorType>;
/// Compile a given source code into a bytecode object.
pub fn compile(
@@ -54,15 +54,17 @@ pub fn compile(
source_path: String,
opts: CompileOpts,
) -> Result<CodeObject, CompileError> {
let mut locator = SourceLocator::new(source);
let mut ast = match parser::parse(source, mode.into(), &source_path) {
Ok(x) => x,
Err(e) => return Err(e.into()),
Err(e) => return Err(e.into_located(&mut locator)),
};
if opts.optimize > 0 {
ast = ConstantOptimizer::new()
.fold_mod(ast)
.unwrap_or_else(|e| match e {});
}
let ast = locator.fold_mod(ast).unwrap_or_else(|e| match e {});
compile::compile_top(&ast, source_path, mode, opts).map_err(|e| e.into())
}
@@ -71,13 +73,18 @@ pub fn compile_symtable(
mode: compile::Mode,
source_path: &str,
) -> Result<symboltable::SymbolTable, CompileError> {
let mut locator = SourceLocator::new(source);
let res = match mode {
compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
let ast = parser::parse_program(source, source_path).map_err(|e| e.into())?;
let ast = parser::parse_program(source, source_path)
.map_err(|e| e.into_located(&mut locator))?;
let ast = rustpython_parser::ast::locate(&mut locator, ast);
symboltable::SymbolTable::scan_program(&ast)
}
compile::Mode::Eval => {
let expr = parser::parse_expression(source, source_path).map_err(|e| e.into())?;
let expr = parser::parse_expression(source, source_path)
.map_err(|e| e.into_located(&mut locator))?;
let expr = rustpython_parser::ast::locate(&mut locator, expr);
symboltable::SymbolTable::scan_expr(&expr)
}
};

2
src/shell.rs
View File

@@ -55,7 +55,7 @@ fn shell_exec(
p,
ParseErrorType::Lexical(LexicalErrorType::IndentationError)
) {
continuing && err.location.column() != 0
continuing && err.location.is_some()
} else {
!matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _))
}

2
stdlib/src/faulthandler.rs
View File

@@ -10,7 +10,7 @@ mod decl {
stderr,
" File \"{}\", line {} in {}",
frame.code.source_path,
frame.current_location().row(),
frame.current_location().row.to_one_indexed(),
frame.code.obj_name
)
}

2
vm/Cargo.toml
View File

@@ -69,7 +69,7 @@ caseless = "0.2.1"
getrandom = { version = "0.2.6", features = ["js"] }
flamer = { version = "0.4", optional = true }
half = "1.8.2"
is-macro = "0.2.0"
is-macro = "0.2.2"
memchr = "2.4.1"
memoffset = "0.6.5"
optional = "0.5.0"

7
vm/src/builtins/code.rs
View File

@@ -9,6 +9,7 @@ use crate::{
class::{PyClassImpl, StaticType},
convert::ToPyObject,
function::{FuncArgs, OptionalArg},
source::try_location_field,
types::Representable,
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine,
};
@@ -26,7 +27,7 @@ pub struct ReplaceArgs {
#[pyarg(named, optional)]
co_filename: OptionalArg<PyStrRef>,
#[pyarg(named, optional)]
co_firstlineno: OptionalArg<u32>,
co_firstlineno: OptionalArg<usize>,
#[pyarg(named, optional)]
co_consts: OptionalArg<Vec<PyObjectRef>>,
#[pyarg(named, optional)]
@@ -276,7 +277,7 @@ impl PyCode {
#[pygetset]
fn co_firstlineno(&self) -> usize {
self.code.first_line_number as usize
self.code.first_line_number.to_one_indexed()
}
#[pygetset]
@@ -348,7 +349,7 @@ impl PyCode {
};
let first_line_number = match args.co_firstlineno {
OptionalArg::Present(first_line_number) => first_line_number,
OptionalArg::Present(first_line_number) => try_location_field(first_line_number, vm)?,
OptionalArg::Missing => self.code.first_line_number,
};

2
vm/src/builtins/frame.rs
View File

@@ -60,7 +60,7 @@ impl Frame {
#[pygetset]
pub fn f_lineno(&self) -> usize {
self.current_location().row()
self.current_location().row.to_one_indexed()
}
#[pygetset]

7
vm/src/builtins/traceback.rs
View File

@@ -2,6 +2,7 @@ use rustpython_common::lock::PyMutex;
use super::PyType;
use crate::{class::PyClassImpl, frame::FrameRef, Context, Py, PyPayload, PyRef};
use rustpython_compiler_core::LineNumber;
#[pyclass(module = false, name = "traceback", traverse)]
#[derive(Debug)]
@@ -11,7 +12,7 @@ pub struct PyTraceback {
#[pytraverse(skip)]
pub lasti: u32,
#[pytraverse(skip)]
pub lineno: usize,
pub lineno: LineNumber,
}
pub type PyTracebackRef = PyRef<PyTraceback>;
@@ -24,7 +25,7 @@ impl PyPayload for PyTraceback {
#[pyclass]
impl PyTraceback {
pub fn new(next: Option<PyRef<Self>>, frame: FrameRef, lasti: u32, lineno: usize) -> Self {
pub fn new(next: Option<PyRef<Self>>, frame: FrameRef, lasti: u32, lineno: LineNumber) -> Self {
PyTraceback {
next: PyMutex::new(next),
frame,
@@ -45,7 +46,7 @@ impl PyTraceback {
#[pygetset]
fn tb_lineno(&self) -> usize {
self.lineno
self.lineno.to_one_indexed()
}
#[pygetset]

2
vm/src/exceptions.rs
View File

@@ -259,7 +259,7 @@ fn write_traceback_entry<W: Write>(
r##" File "{}", line {}, in {}"##,
filename, tb_entry.lineno, tb_entry.frame.code.obj_name
)?;
print_source_line(output, filename, tb_entry.lineno)?;
print_source_line(output, filename, tb_entry.lineno.to_one_indexed())?;
Ok(())
}

11
vm/src/frame.rs
View File

@@ -20,6 +20,7 @@ use crate::{
};
use indexmap::IndexMap;
use itertools::Itertools;
use rustpython_compiler_core::source_code::SourceLocation;
use std::fmt;
use std::iter::zip;
#[cfg(feature = "threading")]
@@ -165,7 +166,7 @@ impl Frame {
}
}
pub fn current_location(&self) -> bytecode::Location {
pub fn current_location(&self) -> SourceLocation {
self.code.locations[self.lasti() as usize - 1]
}
@@ -376,12 +377,8 @@ impl ExecutingFrame<'_> {
let loc = frame.code.locations[idx];
let next = exception.traceback();
let new_traceback = PyTraceback::new(
next,
frame.object.to_owned(),
frame.lasti(),
loc.row(),
);
let new_traceback =
PyTraceback::new(next, frame.object.to_owned(), frame.lasti(), loc.row);
vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.row());
exception.set_traceback(Some(new_traceback.into_ref(&vm.ctx)));

1
vm/src/lib.rs
View File

@@ -72,6 +72,7 @@ pub mod scope;
pub mod sequence;
pub mod signal;
pub mod sliceable;
mod source;
pub mod stdlib;
pub mod suggestion;
pub mod types;

85
vm/src/stdlib/ast.rs
View File

@@ -10,14 +10,21 @@ use crate::{
class::{PyClassImpl, StaticType},
compiler::CompileError,
convert::ToPyException,
source::try_location_field,
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject,
VirtualMachine,
};
use num_complex::Complex64;
use num_traits::{ToPrimitive, Zero};
use rustpython_ast as ast;
use rustpython_ast::{
self as ast,
fold::Fold,
location::{SourceLocation, SourceRange},
SourceLocator,
};
#[cfg(feature = "rustpython-codegen")]
use rustpython_codegen as codegen;
use rustpython_compiler_core::text_size::{TextRange, TextSize};
#[cfg(feature = "rustpython-parser")]
use rustpython_parser as parser;
@@ -144,35 +151,45 @@ impl<T: Node> Node for Option<T> {
}
}
impl<T: NamedNode> Node for ast::Located<T> {
impl<T: NamedNode> Node for ast::located::Located<T> {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
let location = self.location();
let end_location = self.end_location();
let obj = self.node.ast_to_object(vm);
node_add_location(&obj, self.location, self.end_location, vm);
node_add_location(&obj, location, end_location, vm);
obj
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
let location = ast::Location::new(
Node::ast_from_object(vm, get_node_field(vm, &object, "lineno", T::NAME)?)?,
Node::ast_from_object(vm, get_node_field(vm, &object, "col_offset", T::NAME)?)?,
);
let end_location = if let (Some(end_lineno), Some(end_col_offset)) = (
get_node_field_opt(vm, &object, "end_lineno")?
.map(|obj| Node::ast_from_object(vm, obj))
.transpose()?,
get_node_field_opt(vm, &object, "end_col_offset")?
.map(|obj| Node::ast_from_object(vm, obj))
.transpose()?,
) {
Some(ast::Location::new(end_lineno, end_col_offset))
let row = Node::ast_from_object(vm, get_node_field(vm, &object, "lineno", T::NAME)?)?;
let column =
Node::ast_from_object(vm, get_node_field(vm, &object, "col_offset", T::NAME)?)?;
let location = SourceLocation {
row: try_location_field(row, vm)?,
column: try_location_field(column, vm)?,
};
let end_row = get_node_field_opt(vm, &object, "end_lineno")?
.map(|obj| Node::ast_from_object(vm, obj))
.transpose()?;
let end_column = get_node_field_opt(vm, &object, "end_col_offset")?
.map(|obj| Node::ast_from_object(vm, obj))
.transpose()?;
let end_location = if let (Some(row), Some(column)) = (end_row, end_column) {
let location = SourceLocation {
row: try_location_field(row, vm)?,
column: try_location_field(column, vm)?,
};
Some(location)
} else {
None
};
let node = T::ast_from_object(vm, object)?;
Ok(ast::Located {
location,
end_location,
custom: (),
Ok(ast::located::Located {
range: TextRange::empty(TextSize::new(0)),
custom: SourceRange {
start: location,
end: end_location,
},
node,
})
}
@@ -180,21 +197,29 @@ impl<T: NamedNode> Node for ast::Located<T> {
fn node_add_location(
node: &PyObject,
location: ast::Location,
end_location: Option<ast::Location>,
location: SourceLocation,
end_location: Option<SourceLocation>,
vm: &VirtualMachine,
) {
let dict = node.dict().unwrap();
dict.set_item("lineno", vm.ctx.new_int(location.row()).into(), vm)
.unwrap();
dict.set_item("col_offset", vm.ctx.new_int(location.column()).into(), vm)
dict.set_item("lineno", vm.ctx.new_int(location.row.get()).into(), vm)
.unwrap();
dict.set_item(
"col_offset",
vm.ctx.new_int(location.column.get()).into(),
vm,
)
.unwrap();
if let Some(end_location) = end_location {
dict.set_item("end_lineno", vm.ctx.new_int(end_location.row()).into(), vm)
.unwrap();
dict.set_item(
"end_lineno",
vm.ctx.new_int(end_location.row.get()).into(),
vm,
)
.unwrap();
dict.set_item(
"end_col_offset",
vm.ctx.new_int(end_location.column()).into(),
vm.ctx.new_int(end_location.column.get()).into(),
vm,
)
.unwrap();
@@ -307,7 +332,9 @@ pub(crate) fn parse(
source: &str,
mode: parser::Mode,
) -> Result<PyObjectRef, CompileError> {
let top = parser::parse(source, mode, "<unknown>").map_err(CompileError::from)?;
let mut locator = SourceLocator::new(source);
let top = parser::parse(source, mode, "<unknown>").map_err(|e| e.into_located(&mut locator))?;
let top = locator.fold_mod(top).unwrap();
Ok(top.ast_to_object(vm))
}

671
vm/src/stdlib/ast/gen.rs generated
View File

File diff suppressed because it is too large Load Diff

3
vm/src/stdlib/marshal.rs
View File

@@ -215,6 +215,9 @@ mod decl {
marshal::MarshalError::InvalidUtf8 => {
vm.new_value_error("invalid utf8 in marshalled string".to_owned())
}
marshal::MarshalError::InvalidLocation => {
vm.new_value_error("invalid location in marshalled object".to_owned())
}
marshal::MarshalError::BadType => {
vm.new_value_error("bad marshal data (unknown type code)".to_owned())
}

2
vm/src/stdlib/symtable.rs
View File

@@ -60,7 +60,7 @@ mod symtable {
#[pymethod]
fn get_lineno(&self) -> usize {
self.symtable.line_number
self.symtable.line_number.to_one_indexed()
}
#[pymethod]

35
vm/src/vm/vm_new.rs
View File

@@ -8,9 +8,11 @@ use crate::{
convert::ToPyObject,
function::{IntoPyNativeFn, PyMethodFlags},
scope::Scope,
source::AtLocation,
vm::VirtualMachine,
AsObject, Py, PyObject, PyObjectRef, PyRef,
};
use rustpython_ast::location::SourceLocation;
/// Collection of object creation helpers
impl VirtualMachine {
@@ -266,11 +268,12 @@ impl VirtualMachine {
}
.to_owned();
fn get_statement(source: &str, loc: rustpython_compiler_core::Location) -> Option<String> {
if loc.column() == 0 || loc.row() == 0 {
return None;
}
let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
// TODO: replace to SourceCode
fn get_statement(source: &str, loc: Option<SourceLocation>) -> Option<String> {
let line = source
.split('\n')
.nth(loc?.row.to_zero_indexed())?
.to_owned();
Some(line + "\n")
}
@@ -288,10 +291,21 @@ impl VirtualMachine {
let loc = error.location;
if let Some(ref stmt) = statement {
// visualize the error when location and statement are provided
loc.fmt_with(f, &error.error)?;
write!(f, "\n{stmt}{arrow:>pad$}", pad = loc.column(), arrow = "^")
write!(
f,
"{error}{at_location}\n{stmt}{arrow:>pad$}",
error = error.error,
at_location = AtLocation(loc.as_ref()),
pad = loc.map_or(0, |loc| loc.column.to_one_indexed()),
arrow = "^"
)
} else {
loc.fmt_with(f, &error.error)
write!(
f,
"{error}{at_location}",
error = error.error,
at_location = AtLocation(loc.as_ref()),
)
}
}
@@ -299,8 +313,9 @@ impl VirtualMachine {
fmt(error, statement.as_deref(), &mut msg).unwrap();
let syntax_error = self.new_exception_msg(syntax_error_type, msg);
let lineno = self.ctx.new_int(error.location.row());
let offset = self.ctx.new_int(error.location.column());
let (lineno, offset) = error.python_location();
let lineno = self.ctx.new_int(lineno);
let offset = self.ctx.new_int(offset);
syntax_error
.as_object()
.set_attr("lineno", lineno, self)