mschoi/RustPython
1
0
Fork 0
forked from Rust-related/RustPython
Code Pull Requests Actions Activity

Merge pull request #4961 from youknowone/byte-offset

Browse Source 
location to byte offset (TextSize)
This commit is contained in:
Jeong, YunWon
2023-05-10 21:26:28 +09:00
committed by GitHub
parent d03c7267d6 4064c84cb4
commit b85e134477
51 changed files with 3166 additions and 926 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitattributes vendored
View File

@@ -1,8 +1,6 @@
Lib/** linguist-vendored
Cargo.lock linguist-generated -merge
*.snap linguist-generated -merge
ast/src/ast_gen.rs linguist-generated -merge
vm/src/stdlib/ast/gen.rs linguist-generated -merge
compiler/parser/python.lalrpop text eol=LF
Lib/*.py text working-tree-encoding=UTF-8 eol=LF
**/*.rs text working-tree-encoding=UTF-8 eol=LF

15
.github/workflows/ci.yaml vendored
View File

@@ -16,15 +16,6 @@ concurrency:
env:
CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit
NON_WASM_PACKAGES: >-
-p rustpython-common
-p rustpython-compiler
-p rustpython-codegen
-p rustpython-vm
-p rustpython-stdlib
-p rustpython-jit
-p rustpython-derive
-p rustpython
PLATFORM_INDEPENDENT_TESTS: >-
test_argparse
test_array
@@ -126,14 +117,14 @@ jobs:
- uses: Swatinem/rust-cache@v2
- name: run clippy
run: cargo clippy ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }} -- -Dwarnings
run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --exclude rustpython_wasm -- -Dwarnings
- name: run rust tests
run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }} ${{ env.NON_WASM_PACKAGES }}
run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }}
if: runner.os != 'macOS'
# temp skip ssl linking for Mac to avoid CI failure
- name: run rust tests (MacOS no ssl)
run: cargo test --workspace --exclude rustpython_wasm --verbose --no-default-features --features threading,stdlib,zlib,importlib,encodings,jit ${{ env.NON_WASM_PACKAGES }}
run: cargo test --workspace --exclude rustpython_wasm --verbose --no-default-features --features threading,stdlib,zlib,importlib,encodings,jit
if: runner.os == 'macOS'
- name: check compilation without threading

56
.github/workflows/cron-ci.yaml vendored
View File

@@ -11,15 +11,9 @@ env:
jobs:
codecov:
name: Collect code coverage data
needs: lalrpop
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Cache generated parser
uses: actions/cache@v3
with:
path: compiler/parser/python.rs
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
- uses: dtolnay/rust-toolchain@stable
with:
components: llvm-tools-preview
@@ -62,15 +56,9 @@ jobs:
testdata:
name: Collect regression test data
needs: lalrpop
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Cache generated parser
uses: actions/cache@v3
with:
path: compiler/parser/python.rs
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
- uses: dtolnay/rust-toolchain@stable
- name: build rustpython
run: cargo build --release --verbose
@@ -97,15 +85,9 @@ jobs:
whatsleft:
name: Collect what is left data
needs: lalrpop
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Cache generated parser
uses: actions/cache@v3
with:
path: compiler/parser/python.rs
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
- uses: dtolnay/rust-toolchain@stable
- name: build rustpython
run: cargo build --release --verbose
@@ -135,15 +117,9 @@ jobs:
benchmark:
name: Collect benchmark data
needs: lalrpop
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Cache generated parser
uses: actions/cache@v3
with:
path: compiler/parser/python.rs
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
- uses: dtolnay/rust-toolchain@stable
- uses: actions/setup-python@v4
with:
@@ -183,35 +159,3 @@ jobs:
if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then
git push
fi
lalrpop:
name: Generate parser with lalrpop
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Cache generated parser
uses: actions/cache@v3
with:
path: compiler/parser/python.rs
key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }}
- name: Check if cached generated parser exists
id: generated_parser
uses: andstor/file-existence-action@v2
with:
files: "compiler/parser/python.rs"
- if: runner.os == 'Windows'
name: Force python.lalrpop to be lf # actions@checkout ignore .gitattributes
run: |
set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file
- name: Install lalrpop
if: steps.generated_parser.outputs.files_exists == 'false'
uses: baptiste0928/cargo-install@v2
with:
crate: lalrpop
version: "0.19.9"
- name: Run lalrpop
if: steps.generated_parser.outputs.files_exists == 'false'
run: lalrpop compiler/parser/python.lalrpop

242
Cargo.lock generated
View File

@@ -85,15 +85,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16"
[[package]]
name = "ascii-canvas"
version = "3.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6"
dependencies = [
"term",
]
[[package]]
name = "atomic"
version = "0.5.1"
@@ -126,27 +117,18 @@ version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "bit-set"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
dependencies = [
"bit-vec",
]
[[package]]
name = "bit-vec"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24a6904aef64d73cf10ab17ebace7befb918b82164785cb89907993be7f83813"
[[package]]
name = "blake2"
version = "0.10.6"
@@ -260,7 +242,7 @@ checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
dependencies = [
"ansi_term",
"atty",
"bitflags",
"bitflags 1.3.2",
"strsim",
"textwrap 0.11.0",
"unicode-width",
@@ -632,12 +614,6 @@ dependencies = [
"syn",
]
[[package]]
name = "diff"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
[[package]]
name = "digest"
version = "0.10.6"
@@ -694,15 +670,6 @@ version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]]
name = "ena"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3"
dependencies = [
"log",
]
[[package]]
name = "encode_unicode"
version = "0.3.6"
@@ -774,12 +741,6 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "fixedbitset"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "flame"
version = "0.2.2"
@@ -1019,9 +980,9 @@ dependencies = [
[[package]]
name = "is-macro"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c068d4c6b922cd6284c609cfa6dec0e41615c9c5a1a4ba729a970d8daba05fb"
checksum = "8a7d079e129b77477a49c5c4f1cfe9ce6c2c909ef52520693e8e811a714c7b20"
dependencies = [
"Inflector",
"pmutil",
@@ -1030,18 +991,6 @@ dependencies = [
"syn",
]
[[package]]
name = "is-terminal"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f"
dependencies = [
"hermit-abi 0.3.1",
"io-lifetimes",
"rustix",
"windows-sys 0.48.0",
]
[[package]]
name = "itertools"
version = "0.10.5"
@@ -1075,28 +1024,6 @@ dependencies = [
"cpufeatures",
]
[[package]]
name = "lalrpop"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da4081d44f4611b66c6dd725e6de3169f9f63905421e8626fcb86b6a898998b8"
dependencies = [
"ascii-canvas",
"bit-set",
"diff",
"ena",
"is-terminal",
"itertools",
"lalrpop-util",
"petgraph",
"regex",
"regex-syntax 0.7.1",
"string_cache",
"term",
"tiny-keccak",
"unicode-xid",
]
[[package]]
name = "lalrpop-util"
version = "0.20.0"
@@ -1333,12 +1260,6 @@ dependencies = [
"rand_core",
]
[[package]]
name = "new_debug_unreachable"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
[[package]]
name = "nibble_vec"
version = "0.1.0"
@@ -1354,7 +1275,7 @@ version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cc",
"cfg-if",
"libc",
@@ -1367,7 +1288,7 @@ version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cfg-if",
"libc",
"memoffset 0.7.1",
@@ -1484,7 +1405,7 @@ version = "0.10.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "518915b97df115dd36109bfa429a48b8f737bd05508cf9588977b599648926d2"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cfg-if",
"foreign-types",
"libc",
@@ -1578,23 +1499,13 @@ version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
[[package]]
name = "petgraph"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4"
dependencies = [
"fixedbitset",
"indexmap",
]
[[package]]
name = "phf"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c"
dependencies = [
"phf_shared 0.11.1",
"phf_shared",
]
[[package]]
@@ -1604,7 +1515,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770"
dependencies = [
"phf_generator",
"phf_shared 0.11.1",
"phf_shared",
]
[[package]]
@@ -1613,19 +1524,10 @@ version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf"
dependencies = [
"phf_shared 0.11.1",
"phf_shared",
"rand",
]
[[package]]
name = "phf_shared"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
dependencies = [
"siphasher",
]
[[package]]
name = "phf_shared"
version = "0.11.1"
@@ -1692,12 +1594,6 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
[[package]]
name = "precomputed-hash"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "proc-macro-crate"
version = "1.3.0"
@@ -1822,7 +1718,7 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags",
"bitflags 1.3.2",
]
[[package]]
@@ -1856,7 +1752,7 @@ checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.6.28",
"regex-syntax",
]
[[package]]
@@ -1871,19 +1767,13 @@ version = "0.6.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
[[package]]
name = "regex-syntax"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c"
[[package]]
name = "region"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877e54ea2adcd70d80e9179344c97f93ef0dffd6b03e1f4529e6e83ab2fa9ae0"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"libc",
"mach",
"winapi",
@@ -1911,6 +1801,24 @@ dependencies = [
"syn-ext",
]
[[package]]
name = "ruff_source_location"
version = "0.0.0"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df"
dependencies = [
"memchr",
"once_cell",
"ruff_text_size",
]
[[package]]
name = "ruff_text_size"
version = "0.0.0"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df"
dependencies = [
"serde",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
@@ -1932,7 +1840,7 @@ version = "0.37.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"errno",
"io-lifetimes",
"libc",
@@ -1967,11 +1875,11 @@ dependencies = [
[[package]]
name = "rustpython-ast"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=7b8844bd3e470d6b835147ecb445202130ec18d8#7b8844bd3e470d6b835147ecb445202130ec18d8"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df"
dependencies = [
"num-bigint",
"rustpython-compiler-core",
"rustpython-literal",
"rustpython-parser-core",
]
[[package]]
@@ -1979,7 +1887,7 @@ name = "rustpython-codegen"
version = "0.2.0"
dependencies = [
"ahash",
"bitflags",
"bitflags 2.2.1",
"indexmap",
"insta",
"itertools",
@@ -1989,6 +1897,7 @@ dependencies = [
"rustpython-ast",
"rustpython-compiler-core",
"rustpython-parser",
"rustpython-parser-core",
]
[[package]]
@@ -1996,7 +1905,7 @@ name = "rustpython-common"
version = "0.2.0"
dependencies = [
"ascii",
"bitflags",
"bitflags 2.2.1",
"bstr",
"cfg-if",
"itertools",
@@ -2027,14 +1936,14 @@ dependencies = [
[[package]]
name = "rustpython-compiler-core"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=7b8844bd3e470d6b835147ecb445202130ec18d8#7b8844bd3e470d6b835147ecb445202130ec18d8"
dependencies = [
"bitflags",
"bstr",
"bitflags 2.2.1",
"itertools",
"lz4_flex",
"num-bigint",
"num-complex",
"rustpython-parser-core",
"serde",
]
[[package]]
@@ -2058,6 +1967,7 @@ dependencies = [
"quote",
"rustpython-compiler-core",
"rustpython-doc",
"rustpython-parser-core",
"syn",
"syn-ext",
"textwrap 0.15.2",
@@ -2089,7 +1999,7 @@ dependencies = [
[[package]]
name = "rustpython-literal"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=7b8844bd3e470d6b835147ecb445202130ec18d8#7b8844bd3e470d6b835147ecb445202130ec18d8"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df"
dependencies = [
"hexf-parse",
"lexical-parse-float",
@@ -2100,12 +2010,10 @@ dependencies = [
[[package]]
name = "rustpython-parser"
version = "0.2.0"
source = "git+https://github.com/RustPython/Parser.git?rev=7b8844bd3e470d6b835147ecb445202130ec18d8#7b8844bd3e470d6b835147ecb445202130ec18d8"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df"
dependencies = [
"ahash",
"anyhow",
"itertools",
"lalrpop",
"lalrpop-util",
"log",
"num-bigint",
@@ -2114,13 +2022,26 @@ dependencies = [
"phf_codegen",
"rustc-hash",
"rustpython-ast",
"rustpython-compiler-core",
"rustpython-parser-core",
"tiny-keccak",
"unic-emoji-char",
"unic-ucd-ident",
"unicode_names2",
]
[[package]]
name = "rustpython-parser-core"
version = "0.2.0"
source = "git+https://github.com/youknowone/RustPython-parser.git?rev=aa101e4f2693624eddabe35eaf3067bba65331df#aa101e4f2693624eddabe35eaf3067bba65331df"
dependencies = [
"itertools",
"lz4_flex",
"num-bigint",
"num-complex",
"ruff_source_location",
"ruff_text_size",
]
[[package]]
name = "rustpython-pylib"
version = "0.2.0"
@@ -2210,7 +2131,7 @@ dependencies = [
"ahash",
"ascii",
"atty",
"bitflags",
"bitflags 2.2.1",
"bstr",
"caseless",
"cfg-if",
@@ -2254,6 +2175,7 @@ dependencies = [
"rustpython-jit",
"rustpython-literal",
"rustpython-parser",
"rustpython-parser-core",
"rustyline",
"schannel",
"serde",
@@ -2308,7 +2230,7 @@ version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dfc8644681285d1fb67a467fb3021bfea306b99b4146b166a1fe3ada965eece"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"cfg-if",
"clipboard-win",
"dirs-next",
@@ -2492,7 +2414,7 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a490c5c46c35dba9a6f5e7ee8e4d67e775eb2d2da0f115750b8d10e1c1ac2d28"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"num_enum",
"optional",
]
@@ -2509,19 +2431,6 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e08d8363704e6c71fc928674353e6b7c23dcea9d82d7012c8faf2a3a025f8d0"
[[package]]
name = "string_cache"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08"
dependencies = [
"new_debug_unreachable",
"once_cell",
"parking_lot",
"phf_shared 0.10.0",
"precomputed-hash",
]
[[package]]
name = "strsim"
version = "0.8.0"
@@ -2579,7 +2488,7 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d75182f12f490e953596550b65ee31bda7c8e043d9386174b353bda50838c3fd"
dependencies = [
"bitflags",
"bitflags 1.3.2",
"core-foundation",
"system-configuration-sys",
]
@@ -2600,17 +2509,6 @@ version = "0.12.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5"
[[package]]
name = "term"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
dependencies = [
"dirs-next",
"rustversion",
"winapi",
]
[[package]]
name = "termcolor"
version = "1.2.0"
@@ -2933,12 +2831,6 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
[[package]]
name = "unicode-xid"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
[[package]]
name = "unicode_names2"
version = "0.6.0"

28
Cargo.toml
View File

@@ -12,17 +12,29 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
[workspace]
resolver = "2"
members = [
"compiler", "compiler/codegen",
"compiler", "compiler/core", "compiler/codegen",
".", "common", "derive", "jit", "vm", "pylib", "stdlib", "wasm/lib", "derive-impl",
]
[workspace.dependencies]
rustpython-literal = { git = "https://github.com/RustPython/Parser.git", rev = "7b8844bd3e470d6b835147ecb445202130ec18d8" }
rustpython-compiler-core = { git = "https://github.com/RustPython/Parser.git", rev = "7b8844bd3e470d6b835147ecb445202130ec18d8" }
rustpython-parser = { git = "https://github.com/RustPython/Parser.git", rev = "7b8844bd3e470d6b835147ecb445202130ec18d8" }
rustpython-ast = { git = "https://github.com/RustPython/Parser.git", rev = "7b8844bd3e470d6b835147ecb445202130ec18d8" }
rustpython-compiler-core = { path = "compiler/core" }
rustpython-compiler = { path = "compiler" }
rustpython-codegen = { path = "compiler/codegen" }
rustpython-common = { path = "common" }
rustpython-derive = { path = "derive" }
rustpython-derive-impl = { path = "derive-impl" }
rustpython-jit = { path = "jit" }
rustpython-vm = { path = "vm" }
rustpython-pylib = { path = "pylib" }
rustpython-stdlib = { path = "stdlib" }
rustpython-doc = { git = "https://github.com/RustPython/__doc__", branch = "main" }
rustpython-literal = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "aa101e4f2693624eddabe35eaf3067bba65331df" }
rustpython-parser-core = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "aa101e4f2693624eddabe35eaf3067bba65331df" }
rustpython-parser = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "aa101e4f2693624eddabe35eaf3067bba65331df" }
rustpython-ast = { git = "https://github.com/youknowone/RustPython-parser.git", rev = "aa101e4f2693624eddabe35eaf3067bba65331df" }
# rustpython-literal = { path = "../RustPython-parser/literal" }
# rustpython-compiler-core = { path = "../RustPython-parser/core" }
# rustpython-parser-core = { path = "../RustPython-parser/core" }
# rustpython-parser = { path = "../RustPython-parser/parser" }
# rustpython-ast = { path = "../RustPython-parser/ast" }
@@ -30,7 +42,7 @@ ahash = "0.7.6"
anyhow = "1.0.45"
ascii = "1.0"
atty = "0.2.14"
bitflags = "1.3.2"
bitflags = "2.2.1"
bstr = "0.2.17"
cfg-if = "1.0"
chrono = "0.4.19"
@@ -65,7 +77,7 @@ unicode_names2 = { version = "0.6.0", git = "https://github.com/youknowone/unico
widestring = "0.5.1"
[features]
default = ["threading", "stdlib", "zlib", "importlib", "encodings", "rustpython-parser/lalrpop"]
default = ["threading", "stdlib", "zlib", "importlib", "encodings"]
importlib = ["rustpython-vm/importlib"]
encodings = ["rustpython-vm/encodings"]
stdlib = ["rustpython-stdlib", "rustpython-pylib"]

2
Lib/test/test_opcodes.py vendored
View File

@@ -21,8 +21,6 @@ class OpcodeTest(unittest.TestCase):
if n != 90:
self.fail('try inside for')
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_setup_annotations_line(self):
# check that SETUP_ANNOTATIONS does not create spurious line numbers
try:

2
Lib/test/test_syntax.py vendored
View File

@@ -2092,8 +2092,6 @@ def func2():
"""
self._check_error(code, "expected ':'")
# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_invalid_line_continuation_error_position(self):
self._check_error(r"a = 3 \ 4",
"unexpected character after line continuation character",

1
common/src/cformat.rs
View File

@@ -83,6 +83,7 @@ impl From<CFormatQuantity> for CFormatPrecision {
}
bitflags! {
#[derive(Copy, Clone, Debug, PartialEq)]
pub struct CConversionFlags: u32 {
const ALTERNATE_FORM = 0b0000_0001;
const ZERO_PAD = 0b0000_0010;

2
compiler/Cargo.toml
View File

@@ -6,6 +6,6 @@ authors = ["RustPython Team"]
edition = "2021"
[dependencies]
rustpython-codegen = { path = "codegen" }
rustpython-codegen = { workspace = true }
rustpython-compiler-core = { workspace = true }
rustpython-parser = { workspace = true }

3
compiler/codegen/Cargo.toml
View File

@@ -8,7 +8,8 @@ license = "MIT"
edition = "2021"
[dependencies]
rustpython-ast = { workspace = true, features=["unparse"] }
rustpython-ast = { workspace = true, features=["unparse", "constant-optimization"] }
rustpython-parser-core = { workspace = true }
rustpython-compiler-core = { workspace = true }
ahash = { workspace = true }

298
compiler/codegen/src/compile.rs
View File

@@ -18,13 +18,15 @@ use num_complex::Complex64;
use num_traits::ToPrimitive;
use rustpython_ast as ast;
use rustpython_compiler_core::{
self as bytecode, Arg as OpArgMarker, CodeObject, ConstantData, Instruction, Location, OpArg,
OpArgType,
bytecode::{self, Arg as OpArgMarker, CodeObject, ConstantData, Instruction, OpArg, OpArgType},
Mode,
};
use rustpython_parser_core::{
source_code::{LineNumber, SourceLocation},
ConversionFlag,
};
use std::borrow::Cow;
pub use rustpython_compiler_core::Mode;
type CompileResult<T> = Result<T, CodegenError>;
#[derive(PartialEq, Eq, Clone, Copy)]
@@ -52,7 +54,7 @@ struct Compiler {
code_stack: Vec<ir::CodeInfo>,
symbol_table_stack: Vec<SymbolTable>,
source_path: String,
current_source_location: Location,
current_source_location: SourceLocation,
qualified_path: Vec<String>,
done_with_future_stmts: bool,
future_annotations: bool,
@@ -90,7 +92,7 @@ impl CompileContext {
/// Compile an ast::Mod produced from rustpython_parser::parse()
pub fn compile_top(
ast: &ast::Mod,
ast: &ast::located::Mod,
source_path: String,
mode: Mode,
opts: CompileOpts,
@@ -131,7 +133,7 @@ fn compile_impl<Ast: ?Sized>(
/// Compile a standard Python program to bytecode
pub fn compile_program(
ast: &[ast::Stmt],
ast: &[ast::located::Stmt],
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
@@ -146,7 +148,7 @@ pub fn compile_program(
/// Compile a Python program to bytecode for the context of a REPL
pub fn compile_program_single(
ast: &[ast::Stmt],
ast: &[ast::located::Stmt],
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
@@ -160,7 +162,7 @@ pub fn compile_program_single(
}
pub fn compile_block_expression(
ast: &[ast::Stmt],
ast: &[ast::located::Stmt],
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
@@ -174,7 +176,7 @@ pub fn compile_block_expression(
}
pub fn compile_expression(
ast: &ast::Expr,
ast: &ast::located::Expr,
source_path: String,
opts: CompileOpts,
) -> CompileResult<CodeObject> {
@@ -210,7 +212,7 @@ impl Compiler {
arg_count: 0,
kwonlyarg_count: 0,
source_path: source_path.clone(),
first_line_number: 0,
first_line_number: LineNumber::MIN,
obj_name: code_name,
blocks: vec![ir::Block::default()],
@@ -225,7 +227,7 @@ impl Compiler {
code_stack: vec![module_code],
symbol_table_stack: Vec::new(),
source_path,
current_source_location: Location::default(),
current_source_location: SourceLocation::default(),
qualified_path: Vec::new(),
done_with_future_stmts: false,
future_annotations: false,
@@ -239,13 +241,13 @@ impl Compiler {
}
}
fn error(&self, error: CodegenErrorType) -> CodegenError {
fn error(&mut self, error: CodegenErrorType) -> CodegenError {
self.error_loc(error, self.current_source_location)
}
fn error_loc(&self, error: CodegenErrorType, location: Location) -> CodegenError {
fn error_loc(&mut self, error: CodegenErrorType, location: SourceLocation) -> CodegenError {
CodegenError {
error,
location,
location: Some(location),
source_path: self.source_path.clone(),
}
}
@@ -342,7 +344,7 @@ impl Compiler {
fn compile_program(
&mut self,
body: &[ast::Stmt],
body: &[ast::located::Stmt],
symbol_table: SymbolTable,
) -> CompileResult<()> {
let size_before = self.code_stack.len();
@@ -371,7 +373,7 @@ impl Compiler {
fn compile_program_single(
&mut self,
body: &[ast::Stmt],
body: &[ast::located::Stmt],
symbol_table: SymbolTable,
) -> CompileResult<()> {
self.symbol_table_stack.push(symbol_table);
@@ -404,7 +406,7 @@ impl Compiler {
fn compile_block_expr(
&mut self,
body: &[ast::Stmt],
body: &[ast::located::Stmt],
symbol_table: SymbolTable,
) -> CompileResult<()> {
self.symbol_table_stack.push(symbol_table);
@@ -434,7 +436,7 @@ impl Compiler {
// Compile statement in eval mode:
fn compile_eval(
&mut self,
expression: &ast::Expr,
expression: &ast::located::Expr,
symbol_table: SymbolTable,
) -> CompileResult<()> {
self.symbol_table_stack.push(symbol_table);
@@ -443,7 +445,7 @@ impl Compiler {
Ok(())
}
fn compile_statements(&mut self, statements: &[ast::Stmt]) -> CompileResult<()> {
fn compile_statements(&mut self, statements: &[ast::located::Stmt]) -> CompileResult<()> {
for statement in statements {
self.compile_statement(statement)?
}
@@ -462,7 +464,7 @@ impl Compiler {
symboltable::mangle_name(self.class_name.as_deref(), name)
}
fn check_forbidden_name(&self, name: &str, usage: NameUsage) -> CompileResult<()> {
fn check_forbidden_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> {
let msg = match usage {
NameUsage::Store if is_forbidden_name(name) => "cannot assign to",
NameUsage::Delete if is_forbidden_name(name) => "cannot delete",
@@ -554,16 +556,16 @@ impl Compiler {
Ok(())
}
fn compile_statement(&mut self, statement: &ast::Stmt) -> CompileResult<()> {
fn compile_statement(&mut self, statement: &ast::located::Stmt) -> CompileResult<()> {
trace!("Compiling {:?}", statement);
self.set_source_location(statement.start());
self.set_source_location(statement.location());
use ast::{StmtKind::*, *};
match &statement.node {
// we do this here because `from __future__` still executes that `from` statement at runtime,
// we still need to compile the ImportFrom down below
ImportFrom(ast::StmtImportFrom { module, names, .. })
if module.as_deref() == Some("__future__") =>
if module.as_ref().map(|id| id.as_str()) == Some("__future__") =>
{
self.compile_future_features(names)?
}
@@ -580,16 +582,16 @@ impl Compiler {
value: num_traits::Zero::zero(),
});
self.emit_constant(ConstantData::None);
let idx = self.name(&name.name);
let idx = self.name(name.name.as_str());
emit!(self, Instruction::ImportName { idx });
if let Some(alias) = &name.asname {
for part in name.name.split('.').skip(1) {
for part in name.name.as_str().split('.').skip(1) {
let idx = self.name(part);
emit!(self, Instruction::LoadAttr { idx });
}
self.store_name(alias)?
self.store_name(alias.as_str())?
} else {
self.store_name(name.name.split('.').next().unwrap())?
self.store_name(name.name.as_str().split('.').next().unwrap())?
}
}
}
@@ -598,13 +600,14 @@ impl Compiler {
module,
names,
}) => {
let import_star = names.iter().any(|n| n.node.name == "*");
let import_star = names.iter().any(|n| n.node.name.as_str() == "*");
let from_list = if import_star {
if self.ctx.in_func() {
return Err(
self.error_loc(CodegenErrorType::FunctionImportStar, statement.start())
);
return Err(self.error_loc(
CodegenErrorType::FunctionImportStar,
statement.location(),
));
}
vec![ConstantData::Str {
value: "*".to_owned(),
@@ -613,16 +616,16 @@ impl Compiler {
names
.iter()
.map(|n| ConstantData::Str {
value: n.node.name.to_owned(),
value: n.node.name.to_string(),
})
.collect()
};
let module_idx = module.as_ref().map(|s| self.name(s));
let module_idx = module.as_ref().map(|s| self.name(s.as_str()));
// from .... import (*fromlist)
self.emit_constant(ConstantData::Integer {
value: (*level).unwrap_or(0).into(),
value: level.as_ref().map_or(0, |level| level.to_u32()).into(),
});
self.emit_constant(ConstantData::Tuple {
elements: from_list,
@@ -641,15 +644,15 @@ impl Compiler {
for name in names {
let name = &name.node;
let idx = self.name(&name.name);
let idx = self.name(name.name.as_str());
// import symbol from module:
emit!(self, Instruction::ImportFrom { idx });
// Store module under proper name:
if let Some(alias) = &name.asname {
self.store_name(alias)?
self.store_name(alias.as_str())?
} else {
self.store_name(&name.name)?
self.store_name(name.name.as_str())?
}
}
@@ -744,7 +747,7 @@ impl Compiler {
returns,
..
}) => self.compile_function_def(
name,
name.as_str(),
args,
body,
decorator_list,
@@ -759,7 +762,7 @@ impl Compiler {
returns,
..
}) => self.compile_function_def(
name,
name.as_str(),
args,
body,
decorator_list,
@@ -772,7 +775,7 @@ impl Compiler {
bases,
keywords,
decorator_list,
}) => self.compile_class_def(name, body, bases, keywords, decorator_list)?,
}) => self.compile_class_def(name.as_str(), body, bases, keywords, decorator_list)?,
Assert(StmtAssert { test, msg }) => {
// if some flag, ignore all assert statements!
if self.opts.optimize == 0 {
@@ -805,7 +808,9 @@ impl Compiler {
emit!(self, Instruction::Break { target: end });
}
None => {
return Err(self.error_loc(CodegenErrorType::InvalidBreak, statement.start()));
return Err(
self.error_loc(CodegenErrorType::InvalidBreak, statement.location())
);
}
},
Continue => match self.ctx.loop_data {
@@ -814,13 +819,15 @@ impl Compiler {
}
None => {
return Err(
self.error_loc(CodegenErrorType::InvalidContinue, statement.start())
self.error_loc(CodegenErrorType::InvalidContinue, statement.location())
);
}
},
Return(StmtReturn { value }) => {
if !self.ctx.in_func() {
return Err(self.error_loc(CodegenErrorType::InvalidReturn, statement.start()));
return Err(
self.error_loc(CodegenErrorType::InvalidReturn, statement.location())
);
}
match value {
Some(v) => {
@@ -830,8 +837,10 @@ impl Compiler {
.flags
.contains(bytecode::CodeFlags::IS_GENERATOR)
{
return Err(self
.error_loc(CodegenErrorType::AsyncReturnValue, statement.start()));
return Err(self.error_loc(
CodegenErrorType::AsyncReturnValue,
statement.location(),
));
}
self.compile_expression(v)?;
}
@@ -873,15 +882,15 @@ impl Compiler {
Ok(())
}
fn compile_delete(&mut self, expression: &ast::Expr) -> CompileResult<()> {
fn compile_delete(&mut self, expression: &ast::located::Expr) -> CompileResult<()> {
match &expression.node {
ast::ExprKind::Name(ast::ExprName { id, .. }) => {
self.compile_name(id, NameUsage::Delete)?
self.compile_name(id.as_str(), NameUsage::Delete)?
}
ast::ExprKind::Attribute(ast::ExprAttribute { value, attr, .. }) => {
self.check_forbidden_name(attr, NameUsage::Delete)?;
self.check_forbidden_name(attr.as_str(), NameUsage::Delete)?;
self.compile_expression(value)?;
let idx = self.name(attr);
let idx = self.name(attr.as_str());
emit!(self, Instruction::DeleteAttr { idx });
}
ast::ExprKind::Subscript(ast::ExprSubscript { value, slice, .. }) => {
@@ -907,7 +916,7 @@ impl Compiler {
fn enter_function(
&mut self,
name: &str,
args: &ast::Arguments,
args: &ast::located::Arguments,
) -> CompileResult<bytecode::MakeFunctionFlags> {
let have_defaults = !args.defaults.is_empty();
if have_defaults {
@@ -926,7 +935,7 @@ impl Compiler {
.zip(&args.kw_defaults)
{
self.emit_constant(ConstantData::Str {
value: kw.node.arg.clone(),
value: kw.node.arg.to_string(),
});
self.compile_expression(default)?;
}
@@ -959,29 +968,29 @@ impl Compiler {
.chain(&args.args)
.chain(&args.kwonlyargs);
for name in args_iter {
self.varname(&name.node.arg)?;
self.varname(name.node.arg.as_str())?;
}
if let Some(name) = args.vararg.as_deref() {
self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARARGS;
self.varname(&name.node.arg)?;
self.varname(name.node.arg.as_str())?;
}
if let Some(name) = args.kwarg.as_deref() {
self.current_code_info().flags |= bytecode::CodeFlags::HAS_VARKEYWORDS;
self.varname(&name.node.arg)?;
self.varname(name.node.arg.as_str())?;
}
Ok(func_flags)
}
fn prepare_decorators(&mut self, decorator_list: &[ast::Expr]) -> CompileResult<()> {
fn prepare_decorators(&mut self, decorator_list: &[ast::located::Expr]) -> CompileResult<()> {
for decorator in decorator_list {
self.compile_expression(decorator)?;
}
Ok(())
}
fn apply_decorators(&mut self, decorator_list: &[ast::Expr]) {
fn apply_decorators(&mut self, decorator_list: &[ast::located::Expr]) {
// Apply decorators:
for _ in decorator_list {
emit!(self, Instruction::CallFunctionPositional { nargs: 1 });
@@ -990,10 +999,10 @@ impl Compiler {
fn compile_try_statement(
&mut self,
body: &[ast::Stmt],
handlers: &[ast::Excepthandler],
orelse: &[ast::Stmt],
finalbody: &[ast::Stmt],
body: &[ast::located::Stmt],
handlers: &[ast::located::Excepthandler],
orelse: &[ast::located::Stmt],
finalbody: &[ast::located::Stmt],
) -> CompileResult<()> {
let handler_block = self.new_block();
let finally_block = self.new_block();
@@ -1057,7 +1066,7 @@ impl Compiler {
// We have a match, store in name (except x as y)
if let Some(alias) = name {
self.store_name(alias)?
self.store_name(alias.as_str())?
} else {
// Drop exception from top of stack:
emit!(self, Instruction::Pop);
@@ -1122,10 +1131,10 @@ impl Compiler {
fn compile_try_star_statement(
&mut self,
_body: &[ast::Stmt],
_handlers: &[ast::Excepthandler],
_orelse: &[ast::Stmt],
_finalbody: &[ast::Stmt],
_body: &[ast::located::Stmt],
_handlers: &[ast::located::Excepthandler],
_orelse: &[ast::located::Stmt],
_finalbody: &[ast::located::Stmt],
) -> CompileResult<()> {
Err(self.error(CodegenErrorType::NotImplementedYet))
}
@@ -1137,10 +1146,10 @@ impl Compiler {
fn compile_function_def(
&mut self,
name: &str,
args: &ast::Arguments,
body: &[ast::Stmt],
decorator_list: &[ast::Expr],
returns: Option<&ast::Expr>, // TODO: use type hint somehow..
args: &ast::located::Arguments,
body: &[ast::located::Stmt],
decorator_list: &[ast::located::Expr],
returns: Option<&ast::located::Expr>, // TODO: use type hint somehow..
is_async: bool,
) -> CompileResult<()> {
// Create bytecode for this function:
@@ -1215,7 +1224,7 @@ impl Compiler {
for arg in args_iter {
if let Some(annotation) = &arg.node.annotation {
self.emit_constant(ConstantData::Str {
value: self.mangle(&arg.node.arg).into_owned(),
value: self.mangle(arg.node.arg.as_str()).into_owned(),
});
self.compile_annotation(annotation)?;
num_annotations += 1;
@@ -1295,7 +1304,7 @@ impl Compiler {
}
// Python/compile.c find_ann
fn find_ann(body: &[ast::Stmt]) -> bool {
fn find_ann(body: &[ast::located::Stmt]) -> bool {
use ast::StmtKind::*;
for statement in body {
@@ -1329,10 +1338,10 @@ impl Compiler {
fn compile_class_def(
&mut self,
name: &str,
body: &[ast::Stmt],
bases: &[ast::Expr],
keywords: &[ast::Keyword],
decorator_list: &[ast::Expr],
body: &[ast::located::Stmt],
bases: &[ast::located::Expr],
keywords: &[ast::located::Keyword],
decorator_list: &[ast::located::Expr],
) -> CompileResult<()> {
self.prepare_decorators(decorator_list)?;
@@ -1448,9 +1457,9 @@ impl Compiler {
fn compile_while(
&mut self,
test: &ast::Expr,
body: &[ast::Stmt],
orelse: &[ast::Stmt],
test: &ast::located::Expr,
body: &[ast::located::Stmt],
orelse: &[ast::located::Stmt],
) -> CompileResult<()> {
let while_block = self.new_block();
let else_block = self.new_block();
@@ -1479,8 +1488,8 @@ impl Compiler {
fn compile_with(
&mut self,
items: &[ast::Withitem],
body: &[ast::Stmt],
items: &[ast::located::Withitem],
body: &[ast::located::Stmt],
is_async: bool,
) -> CompileResult<()> {
let with_location = self.current_source_location;
@@ -1506,7 +1515,7 @@ impl Compiler {
match &item.optional_vars {
Some(var) => {
self.set_source_location(var.start());
self.set_source_location(var.location());
self.compile_store(var)?;
}
None => {
@@ -1549,10 +1558,10 @@ impl Compiler {
fn compile_for(
&mut self,
target: &ast::Expr,
iter: &ast::Expr,
body: &[ast::Stmt],
orelse: &[ast::Stmt],
target: &ast::located::Expr,
iter: &ast::located::Expr,
body: &[ast::located::Stmt],
orelse: &[ast::located::Stmt],
is_async: bool,
) -> CompileResult<()> {
// Start loop
@@ -1610,8 +1619,8 @@ impl Compiler {
fn compile_match(
&mut self,
subject: &ast::Expr,
cases: &[ast::MatchCase],
subject: &ast::located::Expr,
cases: &[ast::located::MatchCase],
) -> CompileResult<()> {
eprintln!("match subject: {subject:?}");
eprintln!("match cases: {cases:?}");
@@ -1620,9 +1629,9 @@ impl Compiler {
fn compile_chained_comparison(
&mut self,
left: &ast::Expr,
left: &ast::located::Expr,
ops: &[ast::Cmpop],
exprs: &[ast::Expr],
exprs: &[ast::located::Expr],
) -> CompileResult<()> {
assert!(!ops.is_empty());
assert_eq!(exprs.len(), ops.len());
@@ -1706,7 +1715,7 @@ impl Compiler {
Ok(())
}
fn compile_annotation(&mut self, annotation: &ast::Expr) -> CompileResult<()> {
fn compile_annotation(&mut self, annotation: &ast::located::Expr) -> CompileResult<()> {
if self.future_annotations {
self.emit_constant(ConstantData::Str {
value: annotation.to_string(),
@@ -1719,9 +1728,9 @@ impl Compiler {
fn compile_annotated_assign(
&mut self,
target: &ast::Expr,
annotation: &ast::Expr,
value: Option<&ast::Expr>,
target: &ast::located::Expr,
annotation: &ast::located::Expr,
value: Option<&ast::located::Expr>,
) -> CompileResult<()> {
if let Some(value) = value {
self.compile_expression(value)?;
@@ -1741,7 +1750,7 @@ impl Compiler {
let annotations = self.name("__annotations__");
emit!(self, Instruction::LoadNameAny(annotations));
self.emit_constant(ConstantData::Str {
value: self.mangle(id).into_owned(),
value: self.mangle(id.as_str()).into_owned(),
});
emit!(self, Instruction::StoreSubscript);
} else {
@@ -1752,18 +1761,18 @@ impl Compiler {
Ok(())
}
fn compile_store(&mut self, target: &ast::Expr) -> CompileResult<()> {
fn compile_store(&mut self, target: &ast::located::Expr) -> CompileResult<()> {
match &target.node {
ast::ExprKind::Name(ast::ExprName { id, .. }) => self.store_name(id)?,
ast::ExprKind::Name(ast::ExprName { id, .. }) => self.store_name(id.as_str())?,
ast::ExprKind::Subscript(ast::ExprSubscript { value, slice, .. }) => {
self.compile_expression(value)?;
self.compile_expression(slice)?;
emit!(self, Instruction::StoreSubscript);
}
ast::ExprKind::Attribute(ast::ExprAttribute { value, attr, .. }) => {
self.check_forbidden_name(attr, NameUsage::Store)?;
self.check_forbidden_name(attr.as_str(), NameUsage::Store)?;
self.compile_expression(value)?;
let idx = self.name(attr);
let idx = self.name(attr.as_str());
emit!(self, Instruction::StoreAttr { idx });
}
ast::ExprKind::List(ast::ExprList { elts, .. })
@@ -1783,7 +1792,7 @@ impl Compiler {
.ok_or_else(|| {
self.error_loc(
CodegenErrorType::TooManyStarUnpack,
target.start(),
target.location(),
)
})?;
let args = bytecode::UnpackExArgs { before, after };
@@ -1824,9 +1833,9 @@ impl Compiler {
fn compile_augassign(
&mut self,
target: &ast::Expr,
target: &ast::located::Expr,
op: &ast::Operator,
value: &ast::Expr,
value: &ast::located::Expr,
) -> CompileResult<()> {
enum AugAssignKind<'a> {
Name { id: &'a str },
@@ -1836,6 +1845,7 @@ impl Compiler {
let kind = match &target.node {
ast::ExprKind::Name(ast::ExprName { id, .. }) => {
let id = id.as_str();
self.compile_name(id, NameUsage::Load)?;
AugAssignKind::Name { id }
}
@@ -1847,6 +1857,7 @@ impl Compiler {
AugAssignKind::Subscript
}
ast::ExprKind::Attribute(ast::ExprAttribute { value, attr, .. }) => {
let attr = attr.as_str();
self.check_forbidden_name(attr, NameUsage::Store)?;
self.compile_expression(value)?;
emit!(self, Instruction::Duplicate);
@@ -1915,7 +1926,7 @@ impl Compiler {
/// (indicated by the condition parameter).
fn compile_jump_if(
&mut self,
expression: &ast::Expr,
expression: &ast::located::Expr,
condition: bool,
target_block: ir::BlockIdx,
) -> CompileResult<()> {
@@ -1998,7 +2009,11 @@ impl Compiler {
/// Compile a boolean operation as an expression.
/// This means, that the last value remains on the stack.
fn compile_bool_op(&mut self, op: &ast::Boolop, values: &[ast::Expr]) -> CompileResult<()> {
fn compile_bool_op(
&mut self,
op: &ast::Boolop,
values: &[ast::located::Expr],
) -> CompileResult<()> {
let after_block = self.new_block();
let (last_value, values) = values.split_last().unwrap();
@@ -2033,8 +2048,8 @@ impl Compiler {
fn compile_dict(
&mut self,
keys: &[Option<ast::Expr>],
values: &[ast::Expr],
keys: &[Option<ast::located::Expr>],
values: &[ast::located::Expr],
) -> CompileResult<()> {
let mut size = 0;
let (packed, unpacked): (Vec<_>, Vec<_>) = keys
@@ -2056,9 +2071,10 @@ impl Compiler {
Ok(())
}
fn compile_expression(&mut self, expression: &ast::Expr) -> CompileResult<()> {
fn compile_expression(&mut self, expression: &ast::located::Expr) -> CompileResult<()> {
trace!("Compiling {:?}", expression);
self.set_source_location(expression.start());
let location = expression.location();
self.set_source_location(location);
use ast::ExprKind::*;
match &expression.node {
@@ -2094,7 +2110,7 @@ impl Compiler {
}
Attribute(ast::ExprAttribute { value, attr, .. }) => {
self.compile_expression(value)?;
let idx = self.name(attr);
let idx = self.name(attr.as_str());
emit!(self, Instruction::LoadAttr { idx });
}
Compare(ast::ExprCompare {
@@ -2135,7 +2151,7 @@ impl Compiler {
self.compile_dict(keys, values)?;
}
Slice(ast::ExprSlice { lower, upper, step }) => {
let mut compile_bound = |bound: Option<&ast::Expr>| match bound {
let mut compile_bound = |bound: Option<&ast::located::Expr>| match bound {
Some(exp) => self.compile_expression(exp),
None => {
self.emit_constant(ConstantData::None);
@@ -2216,12 +2232,12 @@ impl Compiler {
emit!(
self,
Instruction::FormatValue {
conversion: bytecode::ConversionFlag::try_from(*conversion)
conversion: ConversionFlag::from_op_arg(conversion.to_u32())
.expect("invalid conversion flag"),
},
);
}
Name(ast::ExprName { id, .. }) => self.load_name(id)?,
Name(ast::ExprName { id, .. }) => self.load_name(id.as_str())?,
Lambda(ast::ExprLambda { args, body }) => {
let prev_ctx = self.ctx;
@@ -2362,7 +2378,7 @@ impl Compiler {
Ok(())
}
fn compile_keywords(&mut self, keywords: &[ast::Keyword]) -> CompileResult<()> {
fn compile_keywords(&mut self, keywords: &[ast::located::Keyword]) -> CompileResult<()> {
let mut size = 0;
let groupby = keywords.iter().group_by(|e| e.node.arg.is_none());
for (is_unpacking, sub_keywords) in &groupby {
@@ -2376,7 +2392,7 @@ impl Compiler {
for keyword in sub_keywords {
if let Some(name) = &keyword.node.arg {
self.emit_constant(ConstantData::Str {
value: name.to_owned(),
value: name.to_string(),
});
self.compile_expression(&keyword.node.value)?;
sub_size += 1;
@@ -2394,14 +2410,14 @@ impl Compiler {
fn compile_call(
&mut self,
func: &ast::Expr,
args: &[ast::Expr],
keywords: &[ast::Keyword],
func: &ast::located::Expr,
args: &[ast::located::Expr],
keywords: &[ast::located::Keyword],
) -> CompileResult<()> {
let method =
if let ast::ExprKind::Attribute(ast::ExprAttribute { value, attr, .. }) = &func.node {
self.compile_expression(value)?;
let idx = self.name(attr);
let idx = self.name(attr.as_str());
emit!(self, Instruction::LoadMethod { idx });
true
} else {
@@ -2439,8 +2455,8 @@ impl Compiler {
fn compile_call_inner(
&mut self,
additional_positional: u32,
args: &[ast::Expr],
keywords: &[ast::Keyword],
args: &[ast::located::Expr],
keywords: &[ast::located::Keyword],
) -> CompileResult<CallType> {
let count = (args.len() + keywords.len()).to_u32() + additional_positional;
@@ -2450,7 +2466,7 @@ impl Compiler {
for keyword in keywords {
if let Some(name) = &keyword.node.arg {
self.check_forbidden_name(name, NameUsage::Store)?;
self.check_forbidden_name(name.as_str(), NameUsage::Store)?;
}
}
@@ -2473,7 +2489,7 @@ impl Compiler {
for keyword in keywords {
if let Some(name) = &keyword.node.arg {
kwarg_names.push(ConstantData::Str {
value: name.to_owned(),
value: name.to_string(),
});
} else {
// This means **kwargs!
@@ -2498,7 +2514,7 @@ impl Compiler {
fn gather_elements(
&mut self,
before: u32,
elements: &[ast::Expr],
elements: &[ast::located::Expr],
) -> CompileResult<(u32, bool)> {
// First determine if we have starred elements:
let has_stars = elements
@@ -2549,7 +2565,7 @@ impl Compiler {
Ok((size, has_stars))
}
fn compile_comprehension_element(&mut self, element: &ast::Expr) -> CompileResult<()> {
fn compile_comprehension_element(&mut self, element: &ast::located::Expr) -> CompileResult<()> {
self.compile_expression(element).map_err(|e| {
if let CodegenErrorType::InvalidStarExpr = e.error {
self.error(CodegenErrorType::SyntaxError(
@@ -2565,7 +2581,7 @@ impl Compiler {
&mut self,
name: &str,
init_collection: Option<Instruction>,
generators: &[ast::Comprehension],
generators: &[ast::located::Comprehension],
compile_element: &dyn Fn(&mut Self) -> CompileResult<()>,
) -> CompileResult<()> {
let prev_ctx = self.ctx;
@@ -2597,7 +2613,7 @@ impl Compiler {
let mut loop_labels = vec![];
for generator in generators {
if generator.is_async > 0 {
if generator.is_async {
unimplemented!("async for comprehensions");
}
@@ -2684,12 +2700,15 @@ impl Compiler {
Ok(())
}
fn compile_future_features(&mut self, features: &[ast::Alias]) -> Result<(), CodegenError> {
fn compile_future_features(
&mut self,
features: &[ast::located::Alias],
) -> Result<(), CodegenError> {
if self.done_with_future_stmts {
return Err(self.error(CodegenErrorType::InvalidFuturePlacement));
}
for feature in features {
match &*feature.node.name {
match feature.node.name.as_str() {
// Python 3 features; we've already implemented them by default
"nested_scopes" | "generators" | "division" | "absolute_import"
| "with_statement" | "print_function" | "unicode_literals" => {}
@@ -2705,7 +2724,7 @@ impl Compiler {
// Low level helper functions:
fn _emit(&mut self, instr: Instruction, arg: OpArg, target: ir::BlockIdx) {
let location = compile_location(&self.current_source_location);
let location = self.current_source_location;
// TODO: insert source filename
self.current_block().instructions.push(ir::InstructionInfo {
instr,
@@ -2770,12 +2789,13 @@ impl Compiler {
code.current_block = block;
}
fn set_source_location(&mut self, location: Location) {
fn set_source_location(&mut self, location: SourceLocation) {
self.current_source_location = location;
}
fn get_source_line_number(&self) -> u32 {
self.current_source_location.row().to_u32()
fn get_source_line_number(&mut self) -> LineNumber {
let location = self.current_source_location;
location.row
}
fn push_qualified_path(&mut self, name: &str) {
@@ -2811,7 +2831,7 @@ impl EmitArg<bytecode::Label> for ir::BlockIdx {
}
}
fn split_doc(body: &[ast::Stmt]) -> (Option<String>, &[ast::Stmt]) {
fn split_doc(body: &[ast::located::Stmt]) -> (Option<String>, &[ast::located::Stmt]) {
if let Some((val, body_rest)) = body.split_first() {
if let ast::StmtKind::Expr(ast::StmtExpr { value }) = &val.node {
if let Some(doc) = try_get_constant_string(std::slice::from_ref(value)) {
@@ -2822,8 +2842,8 @@ fn split_doc(body: &[ast::Stmt]) -> (Option<String>, &[ast::Stmt]) {
(None, body)
}
fn try_get_constant_string(values: &[ast::Expr]) -> Option<String> {
fn get_constant_string_inner(out_string: &mut String, value: &ast::Expr) -> bool {
fn try_get_constant_string(values: &[ast::located::Expr]) -> Option<String> {
fn get_constant_string_inner(out_string: &mut String, value: &ast::located::Expr) -> bool {
match &value.node {
ast::ExprKind::Constant(ast::ExprConstant {
value: ast::Constant::Str(s),
@@ -2849,10 +2869,6 @@ fn try_get_constant_string(values: &[ast::Expr]) -> Option<String> {
}
}
fn compile_location(location: &Location) -> bytecode::Location {
bytecode::Location::new(location.row(), location.column())
}
fn compile_constant(value: &ast::Constant) -> ConstantData {
match value {
ast::Constant::None => ConstantData::None,
@@ -2886,14 +2902,18 @@ impl ToU32 for usize {
mod tests {
use super::*;
use rustpython_parser as parser;
use rustpython_parser_core::source_code::SourceLocator;
fn compile_exec(source: &str) -> CodeObject {
let mut locator = SourceLocator::new(source);
use rustpython_parser::ast::fold::Fold;
let mut compiler: Compiler = Compiler::new(
CompileOpts::default(),
"source_path".to_owned(),
"<module>".to_owned(),
);
let ast = parser::parse_program(source, "<test>").unwrap();
let ast = locator.fold(ast).unwrap();
let symbol_scope = SymbolTable::scan_program(&ast).unwrap();
compiler.compile_program(&ast, symbol_scope).unwrap();
compiler.pop_code_object()

2
compiler/codegen/src/error.rs
View File

@@ -1,6 +1,6 @@
use std::fmt;
pub type CodegenError = rustpython_compiler_core::BaseError<CodegenErrorType>;
pub type CodegenError = rustpython_parser_core::source_code::LocatedError<CodegenErrorType>;
#[derive(Debug)]
#[non_exhaustive]

12
compiler/codegen/src/ir.rs
View File

@@ -1,10 +1,10 @@
use std::ops;
use crate::IndexSet;
use rustpython_compiler_core::{
CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label,
Location, OpArg,
use rustpython_compiler_core::bytecode::{
CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label, OpArg,
};
use rustpython_parser_core::source_code::{LineNumber, SourceLocation};
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub struct BlockIdx(pub u32);
@@ -42,7 +42,7 @@ pub struct InstructionInfo {
pub instr: Instruction,
pub arg: OpArg,
pub target: BlockIdx,
pub location: Location,
pub location: SourceLocation,
}
// spell-checker:ignore petgraph
@@ -68,7 +68,7 @@ pub struct CodeInfo {
pub arg_count: u32,
pub kwonlyarg_count: u32,
pub source_path: String,
pub first_line_number: u32,
pub first_line_number: LineNumber,
pub obj_name: String, // Name of the object that created this code object
pub blocks: Vec<Block>,
@@ -158,7 +158,7 @@ impl CodeInfo {
arg_count,
kwonlyarg_count,
source_path,
first_line_number,
first_line_number: Some(first_line_number),
obj_name,
max_stackdepth,

130
compiler/codegen/src/symboltable.rs
View File

@@ -13,7 +13,7 @@ use crate::{
};
use bitflags::bitflags;
use rustpython_ast as ast;
use rustpython_compiler_core::Location;
use rustpython_parser_core::source_code::{LineNumber, SourceLocation};
use std::{borrow::Cow, fmt};
/// Captures all symbols in the current scope, and has a list of sub-scopes in this scope.
@@ -26,7 +26,7 @@ pub struct SymbolTable {
pub typ: SymbolTableType,
/// The line number in the source code where this symboltable begins.
pub line_number: usize,
pub line_number: u32,
// Return True if the block is a nested class or function
pub is_nested: bool,
@@ -40,7 +40,7 @@ pub struct SymbolTable {
}
impl SymbolTable {
fn new(name: String, typ: SymbolTableType, line_number: usize, is_nested: bool) -> Self {
fn new(name: String, typ: SymbolTableType, line_number: u32, is_nested: bool) -> Self {
SymbolTable {
name,
typ,
@@ -51,13 +51,13 @@ impl SymbolTable {
}
}
pub fn scan_program(program: &[ast::Stmt]) -> SymbolTableResult<Self> {
pub fn scan_program(program: &[ast::located::Stmt]) -> SymbolTableResult<Self> {
let mut builder = SymbolTableBuilder::new();
builder.scan_statements(program)?;
builder.finish()
}
pub fn scan_expr(expr: &ast::Expr) -> SymbolTableResult<Self> {
pub fn scan_expr(expr: &ast::located::Expr) -> SymbolTableResult<Self> {
let mut builder = SymbolTableBuilder::new();
builder.scan_expression(expr, ExpressionContext::Load)?;
builder.finish()
@@ -96,6 +96,7 @@ pub enum SymbolScope {
}
bitflags! {
#[derive(Copy, Clone, Debug, PartialEq)]
pub struct SymbolFlags: u16 {
const REFERENCED = 0x001;
const ASSIGNED = 0x002;
@@ -118,7 +119,7 @@ bitflags! {
/// return x // is_free_class
/// ```
const FREE_CLASS = 0x100;
const BOUND = Self::ASSIGNED.bits | Self::PARAMETER.bits | Self::IMPORTED.bits | Self::ITER.bits;
const BOUND = Self::ASSIGNED.bits() | Self::PARAMETER.bits() | Self::IMPORTED.bits() | Self::ITER.bits();
}
}
@@ -127,7 +128,6 @@ bitflags! {
#[derive(Debug, Clone)]
pub struct Symbol {
pub name: String,
// pub table: SymbolTableRef,
pub scope: SymbolScope,
pub flags: SymbolFlags,
}
@@ -161,14 +161,17 @@ impl Symbol {
#[derive(Debug)]
pub struct SymbolTableError {
error: String,
location: Location,
location: Option<SourceLocation>,
}
impl SymbolTableError {
pub fn into_codegen_error(self, source_path: String) -> CodegenError {
CodegenError {
error: CodegenErrorType::SyntaxError(self.error),
location: self.location.with_col_offset(1),
location: self.location.map(|l| SourceLocation {
row: l.row,
column: l.column,
}),
source_path,
}
}
@@ -319,7 +322,7 @@ impl SymbolTableAnalyzer {
return Err(SymbolTableError {
error: format!("no binding for nonlocal '{}' found", symbol.name),
// TODO: accurate location info, somehow
location: Location::default(),
location: None,
});
}
} else {
@@ -329,7 +332,7 @@ impl SymbolTableAnalyzer {
symbol.name
),
// TODO: accurate location info, somehow
location: Location::default(),
location: None,
});
}
}
@@ -452,7 +455,7 @@ impl SymbolTableAnalyzer {
symbol.name
),
// TODO: accurate location info, somehow
location: Location::default(),
location: None,
});
}
@@ -465,7 +468,7 @@ impl SymbolTableAnalyzer {
return Err(SymbolTableError {
error: "assignment expression within a comprehension cannot be used in a class body".to_string(),
// TODO: accurate location info, somehow
location: Location::default(),
location: None,
});
}
SymbolTableType::Function => {
@@ -494,8 +497,7 @@ impl SymbolTableAnalyzer {
if parent_symbol.flags.contains(SymbolFlags::ITER) {
return Err(SymbolTableError {
error: format!("assignment expression cannot rebind comprehension iteration variable {}", symbol.name),
// TODO: accurate location info, somehow
location: Location::default(),
location: None,
});
}
@@ -564,7 +566,9 @@ impl SymbolTableBuilder {
this.enter_scope("top", SymbolTableType::Module, 0);
this
}
}
impl SymbolTableBuilder {
fn finish(mut self) -> Result<SymbolTable, SymbolTableError> {
assert_eq!(self.tables.len(), 1);
let mut symbol_table = self.tables.pop().unwrap();
@@ -572,7 +576,7 @@ impl SymbolTableBuilder {
Ok(symbol_table)
}
fn enter_scope(&mut self, name: &str, typ: SymbolTableType, line_number: usize) {
fn enter_scope(&mut self, name: &str, typ: SymbolTableType, line_number: u32) {
let is_nested = self
.tables
.last()
@@ -588,44 +592,47 @@ impl SymbolTableBuilder {
self.tables.last_mut().unwrap().sub_tables.push(table);
}
fn scan_statements(&mut self, statements: &[ast::Stmt]) -> SymbolTableResult {
fn scan_statements(&mut self, statements: &[ast::located::Stmt]) -> SymbolTableResult {
for statement in statements {
self.scan_statement(statement)?;
}
Ok(())
}
fn scan_parameters(&mut self, parameters: &[ast::Arg]) -> SymbolTableResult {
fn scan_parameters(&mut self, parameters: &[ast::located::Arg]) -> SymbolTableResult {
for parameter in parameters {
self.scan_parameter(parameter)?;
}
Ok(())
}
fn scan_parameter(&mut self, parameter: &ast::Arg) -> SymbolTableResult {
fn scan_parameter(&mut self, parameter: &ast::located::Arg) -> SymbolTableResult {
let usage = if parameter.node.annotation.is_some() {
SymbolUsage::AnnotationParameter
} else {
SymbolUsage::Parameter
};
self.register_name(&parameter.node.arg, usage, parameter.start())
self.register_name(parameter.node.arg.as_str(), usage, parameter.location())
}
fn scan_parameters_annotations(&mut self, parameters: &[ast::Arg]) -> SymbolTableResult {
fn scan_parameters_annotations(
&mut self,
parameters: &[ast::located::Arg],
) -> SymbolTableResult {
for parameter in parameters {
self.scan_parameter_annotation(parameter)?;
}
Ok(())
}
fn scan_parameter_annotation(&mut self, parameter: &ast::Arg) -> SymbolTableResult {
fn scan_parameter_annotation(&mut self, parameter: &ast::located::Arg) -> SymbolTableResult {
if let Some(annotation) = &parameter.node.annotation {
self.scan_annotation(annotation)?;
}
Ok(())
}
fn scan_annotation(&mut self, annotation: &ast::Expr) -> SymbolTableResult {
fn scan_annotation(&mut self, annotation: &ast::located::Expr) -> SymbolTableResult {
if self.future_annotations {
Ok(())
} else {
@@ -633,13 +640,13 @@ impl SymbolTableBuilder {
}
}
fn scan_statement(&mut self, statement: &ast::Stmt) -> SymbolTableResult {
fn scan_statement(&mut self, statement: &ast::located::Stmt) -> SymbolTableResult {
use ast::{StmtKind::*, *};
let location = statement.start();
let location = statement.location();
if let ImportFrom(StmtImportFrom { module, names, .. }) = &statement.node {
if module.as_deref() == Some("__future__") {
if module.as_ref().map(|id| id.as_str()) == Some("__future__") {
for feature in names {
if feature.node.name == "annotations" {
if feature.node.name.as_str() == "annotations" {
self.future_annotations = true;
}
}
@@ -648,12 +655,12 @@ impl SymbolTableBuilder {
match &statement.node {
Global(StmtGlobal { names }) => {
for name in names {
self.register_name(name, SymbolUsage::Global, location)?;
self.register_name(name.as_str(), SymbolUsage::Global, location)?;
}
}
Nonlocal(StmtNonlocal { names }) => {
for name in names {
self.register_name(name, SymbolUsage::Nonlocal, location)?;
self.register_name(name.as_str(), SymbolUsage::Nonlocal, location)?;
}
}
FunctionDef(StmtFunctionDef {
@@ -673,11 +680,11 @@ impl SymbolTableBuilder {
..
}) => {
self.scan_expressions(decorator_list, ExpressionContext::Load)?;
self.register_name(name, SymbolUsage::Assigned, location)?;
self.register_name(name.as_str(), SymbolUsage::Assigned, location)?;
if let Some(expression) = returns {
self.scan_annotation(expression)?;
}
self.enter_function(name, args, location.row())?;
self.enter_function(name.as_str(), args, location.row)?;
self.scan_statements(body)?;
self.leave_scope();
}
@@ -688,8 +695,8 @@ impl SymbolTableBuilder {
keywords,
decorator_list,
}) => {
self.enter_scope(name, SymbolTableType::Class, location.row());
let prev_class = std::mem::replace(&mut self.class_name, Some(name.to_owned()));
self.enter_scope(name.as_str(), SymbolTableType::Class, location.row.get());
let prev_class = std::mem::replace(&mut self.class_name, Some(name.to_string()));
self.register_name("__module__", SymbolUsage::Assigned, location)?;
self.register_name("__qualname__", SymbolUsage::Assigned, location)?;
self.register_name("__doc__", SymbolUsage::Assigned, location)?;
@@ -702,7 +709,7 @@ impl SymbolTableBuilder {
self.scan_expression(&keyword.node.value, ExpressionContext::Load)?;
}
self.scan_expressions(decorator_list, ExpressionContext::Load)?;
self.register_name(name, SymbolUsage::Assigned, location)?;
self.register_name(name.as_str(), SymbolUsage::Assigned, location)?;
}
Expr(StmtExpr { value }) => self.scan_expression(value, ExpressionContext::Load)?,
If(StmtIf { test, body, orelse }) => {
@@ -741,11 +748,11 @@ impl SymbolTableBuilder {
for name in names {
if let Some(alias) = &name.node.asname {
// `import my_module as my_alias`
self.register_name(alias, SymbolUsage::Imported, location)?;
self.register_name(alias.as_str(), SymbolUsage::Imported, location)?;
} else {
// `import module`
self.register_name(
name.node.name.split('.').next().unwrap(),
name.node.name.as_str().split('.').next().unwrap(),
SymbolUsage::Imported,
location,
)?;
@@ -782,8 +789,8 @@ impl SymbolTableBuilder {
}) => {
// https://github.com/python/cpython/blob/main/Python/symtable.c#L1233
match &target.node {
ast::ExprKind::Name(ast::ExprName { id, .. }) if *simple > 0 => {
self.register_name(id, SymbolUsage::AnnotationAssigned, location)?;
ast::ExprKind::Name(ast::ExprName { id, .. }) if *simple => {
self.register_name(id.as_str(), SymbolUsage::AnnotationAssigned, location)?;
}
_ => {
self.scan_expression(target, ExpressionContext::Store)?;
@@ -826,20 +833,17 @@ impl SymbolTableBuilder {
self.scan_expression(expression, ExpressionContext::Load)?;
}
if let Some(name) = name {
self.register_name(name, SymbolUsage::Assigned, location)?;
self.register_name(name.as_str(), SymbolUsage::Assigned, location)?;
}
self.scan_statements(body)?;
}
self.scan_statements(orelse)?;
self.scan_statements(finalbody)?;
}
Match(StmtMatch {
subject: _,
cases: _,
}) => {
Match(StmtMatch { subject, cases: _ }) => {
return Err(SymbolTableError {
error: "match expression is not implemented yet".to_owned(),
location: Location::default(),
location: Some(subject.location()),
});
}
Raise(StmtRaise { exc, cause }) => {
@@ -856,7 +860,7 @@ impl SymbolTableBuilder {
fn scan_expressions(
&mut self,
expressions: &[ast::Expr],
expressions: &[ast::located::Expr],
context: ExpressionContext,
) -> SymbolTableResult {
for expression in expressions {
@@ -867,11 +871,11 @@ impl SymbolTableBuilder {
fn scan_expression(
&mut self,
expression: &ast::Expr,
expression: &ast::located::Expr,
context: ExpressionContext,
) -> SymbolTableResult {
use ast::{ExprKind::*, *};
let location = expression.start();
let location = expression.location();
match &expression.node {
BinOp(ExprBinOp { left, right, .. }) => {
self.scan_expression(left, context)?;
@@ -984,6 +988,7 @@ impl SymbolTableBuilder {
}
}
Name(ExprName { id, .. }) => {
let id = id.as_str();
// Determine the contextual usage of this symbol:
match context {
ExpressionContext::Delete => {
@@ -1010,7 +1015,7 @@ impl SymbolTableBuilder {
}
}
Lambda(ExprLambda { args, body }) => {
self.enter_function("lambda", args, expression.start().row())?;
self.enter_function("lambda", args, expression.location().row)?;
match context {
ExpressionContext::IterDefinitionExp => {
self.scan_expression(body, ExpressionContext::IterDefinitionExp)?;
@@ -1033,8 +1038,7 @@ impl SymbolTableBuilder {
if let ExpressionContext::IterDefinitionExp = context {
return Err(SymbolTableError {
error: "assignment expression cannot be used in a comprehension iterable expression".to_string(),
// TODO: accurate location info, somehow
location: Location::default(),
location: Some(target.location()),
});
}
@@ -1045,6 +1049,7 @@ impl SymbolTableBuilder {
// propagate the scope of the named assigned named and not to
// propagate inner names.
if let Name(ExprName { id, .. }) = &target.node {
let id = id.as_str();
let table = self.tables.last().unwrap();
if table.typ == SymbolTableType::Comprehension {
self.register_name(
@@ -1069,14 +1074,17 @@ impl SymbolTableBuilder {
fn scan_comprehension(
&mut self,
scope_name: &str,
elt1: &ast::Expr,
elt2: Option<&ast::Expr>,
generators: &[ast::Comprehension],
location: Location,
elt1: &ast::located::Expr,
elt2: Option<&ast::located::Expr>,
generators: &[ast::located::Comprehension],
location: SourceLocation,
) -> SymbolTableResult {
// Comprehensions are compiled as functions, so create a scope for them:
self.enter_scope(scope_name, SymbolTableType::Comprehension, location.row());
self.enter_scope(
scope_name,
SymbolTableType::Comprehension,
location.row.get(),
);
// Register the passed argument to the generator function as the name ".0"
self.register_name(".0", SymbolUsage::Parameter, location)?;
@@ -1112,8 +1120,8 @@ impl SymbolTableBuilder {
fn enter_function(
&mut self,
name: &str,
args: &ast::Arguments,
line_number: usize,
args: &ast::located::Arguments,
line_number: LineNumber,
) -> SymbolTableResult {
// Evaluate eventual default parameters:
self.scan_expressions(&args.defaults, ExpressionContext::Load)?;
@@ -1132,7 +1140,7 @@ impl SymbolTableBuilder {
self.scan_parameter_annotation(name)?;
}
self.enter_scope(name, SymbolTableType::Function, line_number);
self.enter_scope(name, SymbolTableType::Function, line_number.get());
// Fill scope with parameter names:
self.scan_parameters(&args.posonlyargs)?;
@@ -1151,13 +1159,13 @@ impl SymbolTableBuilder {
&mut self,
name: &str,
role: SymbolUsage,
location: Location,
location: SourceLocation,
) -> SymbolTableResult {
let location = Some(location);
let scope_depth = self.tables.len();
let table = self.tables.last_mut().unwrap();
let name = mangle_name(self.class_name.as_deref(), name);
// Some checks for the symbol that present on this scope level:
let symbol = if let Some(symbol) = table.symbols.get_mut(name.as_ref()) {
let flags = &symbol.flags;

20
compiler/core/Cargo.toml Normal file
View File

@@ -0,0 +1,20 @@
[package]
name = "rustpython-compiler-core"
description = "RustPython specific bytecode."
version = "0.2.0"
authors = ["RustPython Team"]
edition = "2021"
repository = "https://github.com/RustPython/RustPython"
license = "MIT"
[dependencies]
rustpython-parser-core = { workspace = true }
bitflags = { workspace = true }
itertools = { workspace = true }
num-bigint = { workspace = true }
num-complex = { workspace = true }
serde = { version = "1.0.133", optional = true, default-features = false, features = ["derive"] }
lz4_flex = "0.9.2"

1477
compiler/core/src/bytecode.rs Normal file
View File

File diff suppressed because it is too large Load Diff

129
compiler/core/src/frozen.rs Normal file
View File

@@ -0,0 +1,129 @@
use crate::bytecode::*;
use crate::marshal::{self, Read, ReadBorrowed, Write};
/// A frozen module. Holds a frozen code object and whether it is part of a package
#[derive(Copy, Clone)]
pub struct FrozenModule<B = &'static [u8]> {
pub code: FrozenCodeObject<B>,
pub package: bool,
}
#[derive(Copy, Clone)]
pub struct FrozenCodeObject<B> {
pub bytes: B,
}
impl<B: AsRef<[u8]>> FrozenCodeObject<B> {
/// Decode a frozen code object
#[inline]
pub fn decode<Bag: AsBag>(&self, bag: Bag) -> CodeObject<<Bag::Bag as ConstantBag>::Constant> {
Self::_decode(self.bytes.as_ref(), bag.as_bag())
}
fn _decode<Bag: ConstantBag>(data: &[u8], bag: Bag) -> CodeObject<Bag::Constant> {
let decompressed = lz4_flex::decompress_size_prepended(data)
.expect("deserialize frozen CodeObject failed");
marshal::deserialize_code(&mut &decompressed[..], bag)
.expect("deserializing frozen CodeObject failed")
}
}
impl FrozenCodeObject<Vec<u8>> {
pub fn encode<C: Constant>(code: &CodeObject<C>) -> Self {
let mut data = Vec::new();
marshal::serialize_code(&mut data, code);
let bytes = lz4_flex::compress_prepend_size(&data);
FrozenCodeObject { bytes }
}
}
#[repr(transparent)]
pub struct FrozenLib<B: ?Sized = [u8]> {
pub bytes: B,
}
impl<B: AsRef<[u8]> + ?Sized> FrozenLib<B> {
pub const fn from_ref(b: &B) -> &FrozenLib<B> {
unsafe { &*(b as *const B as *const FrozenLib<B>) }
}
/// Decode a library to a iterable of frozen modules
pub fn decode(&self) -> FrozenModulesIter<'_> {
let mut data = self.bytes.as_ref();
let remaining = data.read_u32().unwrap();
FrozenModulesIter { remaining, data }
}
}
impl<'a, B: AsRef<[u8]> + ?Sized> IntoIterator for &'a FrozenLib<B> {
type Item = (&'a str, FrozenModule<&'a [u8]>);
type IntoIter = FrozenModulesIter<'a>;
fn into_iter(self) -> Self::IntoIter {
self.decode()
}
}
pub struct FrozenModulesIter<'a> {
remaining: u32,
data: &'a [u8],
}
impl<'a> Iterator for FrozenModulesIter<'a> {
type Item = (&'a str, FrozenModule<&'a [u8]>);
fn next(&mut self) -> Option<Self::Item> {
if self.remaining > 0 {
let entry = read_entry(&mut self.data).unwrap();
self.remaining -= 1;
Some(entry)
} else {
None
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
(self.remaining as usize, Some(self.remaining as usize))
}
}
impl ExactSizeIterator for FrozenModulesIter<'_> {}
fn read_entry<'a>(
rdr: &mut &'a [u8],
) -> Result<(&'a str, FrozenModule<&'a [u8]>), marshal::MarshalError> {
let len = rdr.read_u32()?;
let name = rdr.read_str_borrow(len)?;
let len = rdr.read_u32()?;
let code_slice = rdr.read_slice_borrow(len)?;
let code = FrozenCodeObject { bytes: code_slice };
let package = rdr.read_u8()? != 0;
Ok((name, FrozenModule { code, package }))
}
impl FrozenLib<Vec<u8>> {
/// Encode the given iterator of frozen modules into a compressed vector of bytes
pub fn encode<'a, I, B: AsRef<[u8]>>(lib: I) -> FrozenLib<Vec<u8>>
where
I: IntoIterator<Item = (&'a str, FrozenModule<B>)>,
I::IntoIter: ExactSizeIterator + Clone,
{
let iter = lib.into_iter();
let mut bytes = Vec::new();
write_lib(&mut bytes, iter);
Self { bytes }
}
}
fn write_lib<'a, B: AsRef<[u8]>>(
buf: &mut Vec<u8>,
lib: impl ExactSizeIterator<Item = (&'a str, FrozenModule<B>)>,
) {
marshal::write_len(buf, lib.len());
for (name, module) in lib {
write_entry(buf, name, module);
}
}
fn write_entry(buf: &mut Vec<u8>, name: &str, module: FrozenModule<impl AsRef<[u8]>>) {
marshal::write_vec(buf, name.as_bytes());
marshal::write_vec(buf, module.code.bytes.as_ref());
buf.write_u8(module.package as u8);
}

9
compiler/core/src/lib.rs Normal file
View File

@@ -0,0 +1,9 @@
#![doc(html_logo_url = "https://raw.githubusercontent.com/RustPython/RustPython/main/logo.png")]
#![doc(html_root_url = "https://docs.rs/rustpython-compiler-core/")]
pub mod bytecode;
pub mod frozen;
pub mod marshal;
mod mode;
pub use mode::Mode;

629
compiler/core/src/marshal.rs Normal file
View File

@@ -0,0 +1,629 @@
use crate::bytecode::*;
use num_bigint::{BigInt, Sign};
use num_complex::Complex64;
use rustpython_parser_core::source_code::{OneIndexed, SourceLocation};
use std::convert::Infallible;
pub const FORMAT_VERSION: u32 = 4;
#[derive(Debug)]
pub enum MarshalError {
/// Unexpected End Of File
Eof,
/// Invalid Bytecode
InvalidBytecode,
/// Invalid utf8 in string
InvalidUtf8,
/// Invalid source location
InvalidLocation,
/// Bad type marker
BadType,
}
impl std::fmt::Display for MarshalError {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
Self::Eof => f.write_str("unexpected end of data"),
Self::InvalidBytecode => f.write_str("invalid bytecode"),
Self::InvalidUtf8 => f.write_str("invalid utf8"),
Self::InvalidLocation => f.write_str("invalid source location"),
Self::BadType => f.write_str("bad type marker"),
}
}
}
impl From<std::str::Utf8Error> for MarshalError {
fn from(_: std::str::Utf8Error) -> Self {
Self::InvalidUtf8
}
}
impl std::error::Error for MarshalError {}
type Result<T, E = MarshalError> = std::result::Result<T, E>;
#[repr(u8)]
enum Type {
// Null = b'0',
None = b'N',
False = b'F',
True = b'T',
StopIter = b'S',
Ellipsis = b'.',
Int = b'i',
Float = b'g',
Complex = b'y',
// Long = b'l', // i32
Bytes = b's', // = TYPE_STRING
// Interned = b't',
// Ref = b'r',
Tuple = b'(',
List = b'[',
Dict = b'{',
Code = b'c',
Unicode = b'u',
// Unknown = b'?',
Set = b'<',
FrozenSet = b'>',
Ascii = b'a',
// AsciiInterned = b'A',
// SmallTuple = b')',
// ShortAscii = b'z',
// ShortAsciiInterned = b'Z',
}
// const FLAG_REF: u8 = b'\x80';
impl TryFrom<u8> for Type {
type Error = MarshalError;
fn try_from(value: u8) -> Result<Self> {
use Type::*;
Ok(match value {
// b'0' => Null,
b'N' => None,
b'F' => False,
b'T' => True,
b'S' => StopIter,
b'.' => Ellipsis,
b'i' => Int,
b'g' => Float,
b'y' => Complex,
// b'l' => Long,
b's' => Bytes,
// b't' => Interned,
// b'r' => Ref,
b'(' => Tuple,
b'[' => List,
b'{' => Dict,
b'c' => Code,
b'u' => Unicode,
// b'?' => Unknown,
b'<' => Set,
b'>' => FrozenSet,
b'a' => Ascii,
// b'A' => AsciiInterned,
// b')' => SmallTuple,
// b'z' => ShortAscii,
// b'Z' => ShortAsciiInterned,
_ => return Err(MarshalError::BadType),
})
}
}
pub trait Read {
fn read_slice(&mut self, n: u32) -> Result<&[u8]>;
fn read_array<const N: usize>(&mut self) -> Result<&[u8; N]> {
self.read_slice(N as u32).map(|s| s.try_into().unwrap())
}
fn read_str(&mut self, len: u32) -> Result<&str> {
Ok(std::str::from_utf8(self.read_slice(len)?)?)
}
fn read_u8(&mut self) -> Result<u8> {
Ok(u8::from_le_bytes(*self.read_array()?))
}
fn read_u16(&mut self) -> Result<u16> {
Ok(u16::from_le_bytes(*self.read_array()?))
}
fn read_u32(&mut self) -> Result<u32> {
Ok(u32::from_le_bytes(*self.read_array()?))
}
fn read_u64(&mut self) -> Result<u64> {
Ok(u64::from_le_bytes(*self.read_array()?))
}
}
pub(crate) trait ReadBorrowed<'a>: Read {
fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]>;
fn read_str_borrow(&mut self, len: u32) -> Result<&'a str> {
Ok(std::str::from_utf8(self.read_slice_borrow(len)?)?)
}
}
impl Read for &[u8] {
fn read_slice(&mut self, n: u32) -> Result<&[u8]> {
self.read_slice_borrow(n)
}
}
impl<'a> ReadBorrowed<'a> for &'a [u8] {
fn read_slice_borrow(&mut self, n: u32) -> Result<&'a [u8]> {
let data = self.get(..n as usize).ok_or(MarshalError::Eof)?;
*self = &self[n as usize..];
Ok(data)
}
}
pub struct Cursor<B> {
pub data: B,
pub position: usize,
}
impl<B: AsRef<[u8]>> Read for Cursor<B> {
fn read_slice(&mut self, n: u32) -> Result<&[u8]> {
let data = &self.data.as_ref()[self.position..];
let slice = data.get(..n as usize).ok_or(MarshalError::Eof)?;
self.position += n as usize;
Ok(slice)
}
}
pub fn deserialize_code<R: Read, Bag: ConstantBag>(
rdr: &mut R,
bag: Bag,
) -> Result<CodeObject<Bag::Constant>> {
let len = rdr.read_u32()?;
let instructions = rdr.read_slice(len * 2)?;
let instructions = instructions
.chunks_exact(2)
.map(|cu| {
let op = Instruction::try_from(cu[0])?;
let arg = OpArgByte(cu[1]);
Ok(CodeUnit { op, arg })
})
.collect::<Result<Box<[CodeUnit]>>>()?;
let len = rdr.read_u32()?;
let locations = (0..len)
.map(|_| {
Ok(SourceLocation {
row: OneIndexed::new(rdr.read_u32()?).ok_or(MarshalError::InvalidLocation)?,
column: OneIndexed::from_zero_indexed(rdr.read_u32()?),
})
})
.collect::<Result<Box<[SourceLocation]>>>()?;
let flags = CodeFlags::from_bits_truncate(rdr.read_u16()?);
let posonlyarg_count = rdr.read_u32()?;
let arg_count = rdr.read_u32()?;
let kwonlyarg_count = rdr.read_u32()?;
let len = rdr.read_u32()?;
let source_path = bag.make_name(rdr.read_str(len)?);
let first_line_number = OneIndexed::new(rdr.read_u32()?);
let max_stackdepth = rdr.read_u32()?;
let len = rdr.read_u32()?;
let obj_name = bag.make_name(rdr.read_str(len)?);
let len = rdr.read_u32()?;
let cell2arg = (len != 0)
.then(|| {
(0..len)
.map(|_| Ok(rdr.read_u32()? as i32))
.collect::<Result<Box<[i32]>>>()
})
.transpose()?;
let len = rdr.read_u32()?;
let constants = (0..len)
.map(|_| deserialize_value(rdr, bag))
.collect::<Result<Box<[_]>>>()?;
let mut read_names = || {
let len = rdr.read_u32()?;
(0..len)
.map(|_| {
let len = rdr.read_u32()?;
Ok(bag.make_name(rdr.read_str(len)?))
})
.collect::<Result<Box<[_]>>>()
};
let names = read_names()?;
let varnames = read_names()?;
let cellvars = read_names()?;
let freevars = read_names()?;
Ok(CodeObject {
instructions,
locations,
flags,
posonlyarg_count,
arg_count,
kwonlyarg_count,
source_path,
first_line_number,
max_stackdepth,
obj_name,
cell2arg,
constants,
names,
varnames,
cellvars,
freevars,
})
}
pub trait MarshalBag: Copy {
type Value;
fn make_bool(&self, value: bool) -> Self::Value;
fn make_none(&self) -> Self::Value;
fn make_ellipsis(&self) -> Self::Value;
fn make_float(&self, value: f64) -> Self::Value;
fn make_complex(&self, value: Complex64) -> Self::Value;
fn make_str(&self, value: &str) -> Self::Value;
fn make_bytes(&self, value: &[u8]) -> Self::Value;
fn make_int(&self, value: BigInt) -> Self::Value;
fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value;
fn make_code(
&self,
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
) -> Self::Value;
fn make_stop_iter(&self) -> Result<Self::Value>;
fn make_list(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>;
fn make_set(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>;
fn make_frozenset(&self, it: impl Iterator<Item = Self::Value>) -> Result<Self::Value>;
fn make_dict(
&self,
it: impl Iterator<Item = (Self::Value, Self::Value)>,
) -> Result<Self::Value>;
type ConstantBag: ConstantBag;
fn constant_bag(self) -> Self::ConstantBag;
}
impl<Bag: ConstantBag> MarshalBag for Bag {
type Value = Bag::Constant;
fn make_bool(&self, value: bool) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Boolean { value })
}
fn make_none(&self) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::None)
}
fn make_ellipsis(&self) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Ellipsis)
}
fn make_float(&self, value: f64) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Float { value })
}
fn make_complex(&self, value: Complex64) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Complex { value })
}
fn make_str(&self, value: &str) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Str { value })
}
fn make_bytes(&self, value: &[u8]) -> Self::Value {
self.make_constant::<Bag::Constant>(BorrowedConstant::Bytes { value })
}
fn make_int(&self, value: BigInt) -> Self::Value {
self.make_int(value)
}
fn make_tuple(&self, elements: impl Iterator<Item = Self::Value>) -> Self::Value {
self.make_tuple(elements)
}
fn make_code(
&self,
code: CodeObject<<Self::ConstantBag as ConstantBag>::Constant>,
) -> Self::Value {
self.make_code(code)
}
fn make_stop_iter(&self) -> Result<Self::Value> {
Err(MarshalError::BadType)
}
fn make_list(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
Err(MarshalError::BadType)
}
fn make_set(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
Err(MarshalError::BadType)
}
fn make_frozenset(&self, _: impl Iterator<Item = Self::Value>) -> Result<Self::Value> {
Err(MarshalError::BadType)
}
fn make_dict(
&self,
_: impl Iterator<Item = (Self::Value, Self::Value)>,
) -> Result<Self::Value> {
Err(MarshalError::BadType)
}
type ConstantBag = Self;
fn constant_bag(self) -> Self::ConstantBag {
self
}
}
pub fn deserialize_value<R: Read, Bag: MarshalBag>(rdr: &mut R, bag: Bag) -> Result<Bag::Value> {
let typ = Type::try_from(rdr.read_u8()?)?;
let value = match typ {
Type::True => bag.make_bool(true),
Type::False => bag.make_bool(false),
Type::None => bag.make_none(),
Type::StopIter => bag.make_stop_iter()?,
Type::Ellipsis => bag.make_ellipsis(),
Type::Int => {
let len = rdr.read_u32()? as i32;
let sign = if len < 0 { Sign::Minus } else { Sign::Plus };
let bytes = rdr.read_slice(len.unsigned_abs())?;
let int = BigInt::from_bytes_le(sign, bytes);
bag.make_int(int)
}
Type::Float => {
let value = f64::from_bits(rdr.read_u64()?);
bag.make_float(value)
}
Type::Complex => {
let re = f64::from_bits(rdr.read_u64()?);
let im = f64::from_bits(rdr.read_u64()?);
let value = Complex64 { re, im };
bag.make_complex(value)
}
Type::Ascii | Type::Unicode => {
let len = rdr.read_u32()?;
let value = rdr.read_str(len)?;
bag.make_str(value)
}
Type::Tuple => {
let len = rdr.read_u32()?;
let it = (0..len).map(|_| deserialize_value(rdr, bag));
itertools::process_results(it, |it| bag.make_tuple(it))?
}
Type::List => {
let len = rdr.read_u32()?;
let it = (0..len).map(|_| deserialize_value(rdr, bag));
itertools::process_results(it, |it| bag.make_list(it))??
}
Type::Set => {
let len = rdr.read_u32()?;
let it = (0..len).map(|_| deserialize_value(rdr, bag));
itertools::process_results(it, |it| bag.make_set(it))??
}
Type::FrozenSet => {
let len = rdr.read_u32()?;
let it = (0..len).map(|_| deserialize_value(rdr, bag));
itertools::process_results(it, |it| bag.make_frozenset(it))??
}
Type::Dict => {
let len = rdr.read_u32()?;
let it = (0..len).map(|_| {
let k = deserialize_value(rdr, bag)?;
let v = deserialize_value(rdr, bag)?;
Ok::<_, MarshalError>((k, v))
});
itertools::process_results(it, |it| bag.make_dict(it))??
}
Type::Bytes => {
// Following CPython, after marshaling, byte arrays are converted into bytes.
let len = rdr.read_u32()?;
let value = rdr.read_slice(len)?;
bag.make_bytes(value)
}
Type::Code => bag.make_code(deserialize_code(rdr, bag.constant_bag())?),
};
Ok(value)
}
pub trait Dumpable: Sized {
type Error;
type Constant: Constant;
fn with_dump<R>(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result<R, Self::Error>;
}
pub enum DumpableValue<'a, D: Dumpable> {
Integer(&'a BigInt),
Float(f64),
Complex(Complex64),
Boolean(bool),
Str(&'a str),
Bytes(&'a [u8]),
Code(&'a CodeObject<D::Constant>),
Tuple(&'a [D]),
None,
Ellipsis,
StopIter,
List(&'a [D]),
Set(&'a [D]),
Frozenset(&'a [D]),
Dict(&'a [(D, D)]),
}
impl<'a, C: Constant> From<BorrowedConstant<'a, C>> for DumpableValue<'a, C> {
fn from(c: BorrowedConstant<'a, C>) -> Self {
match c {
BorrowedConstant::Integer { value } => Self::Integer(value),
BorrowedConstant::Float { value } => Self::Float(value),
BorrowedConstant::Complex { value } => Self::Complex(value),
BorrowedConstant::Boolean { value } => Self::Boolean(value),
BorrowedConstant::Str { value } => Self::Str(value),
BorrowedConstant::Bytes { value } => Self::Bytes(value),
BorrowedConstant::Code { code } => Self::Code(code),
BorrowedConstant::Tuple { elements } => Self::Tuple(elements),
BorrowedConstant::None => Self::None,
BorrowedConstant::Ellipsis => Self::Ellipsis,
}
}
}
impl<C: Constant> Dumpable for C {
type Error = Infallible;
type Constant = Self;
#[inline(always)]
fn with_dump<R>(&self, f: impl FnOnce(DumpableValue<'_, Self>) -> R) -> Result<R, Self::Error> {
Ok(f(self.borrow_constant().into()))
}
}
pub trait Write {
fn write_slice(&mut self, slice: &[u8]);
fn write_u8(&mut self, v: u8) {
self.write_slice(&v.to_le_bytes())
}
fn write_u16(&mut self, v: u16) {
self.write_slice(&v.to_le_bytes())
}
fn write_u32(&mut self, v: u32) {
self.write_slice(&v.to_le_bytes())
}
fn write_u64(&mut self, v: u64) {
self.write_slice(&v.to_le_bytes())
}
}
impl Write for Vec<u8> {
fn write_slice(&mut self, slice: &[u8]) {
self.extend_from_slice(slice)
}
}
pub(crate) fn write_len<W: Write>(buf: &mut W, len: usize) {
let Ok(len) = len.try_into() else { panic!("too long to serialize") };
buf.write_u32(len);
}
pub(crate) fn write_vec<W: Write>(buf: &mut W, slice: &[u8]) {
write_len(buf, slice.len());
buf.write_slice(slice);
}
pub fn serialize_value<W: Write, D: Dumpable>(
buf: &mut W,
constant: DumpableValue<'_, D>,
) -> Result<(), D::Error> {
match constant {
DumpableValue::Integer(int) => {
buf.write_u8(Type::Int as u8);
let (sign, bytes) = int.to_bytes_le();
let len: i32 = bytes.len().try_into().expect("too long to serialize");
let len = if sign == Sign::Minus { -len } else { len };
buf.write_u32(len as u32);
buf.write_slice(&bytes);
}
DumpableValue::Float(f) => {
buf.write_u8(Type::Float as u8);
buf.write_u64(f.to_bits());
}
DumpableValue::Complex(c) => {
buf.write_u8(Type::Complex as u8);
buf.write_u64(c.re.to_bits());
buf.write_u64(c.im.to_bits());
}
DumpableValue::Boolean(b) => {
buf.write_u8(if b { Type::True } else { Type::False } as u8);
}
DumpableValue::Str(s) => {
buf.write_u8(Type::Unicode as u8);
write_vec(buf, s.as_bytes());
}
DumpableValue::Bytes(b) => {
buf.write_u8(Type::Bytes as u8);
write_vec(buf, b);
}
DumpableValue::Code(c) => {
buf.write_u8(Type::Code as u8);
serialize_code(buf, c);
}
DumpableValue::Tuple(tup) => {
buf.write_u8(Type::Tuple as u8);
write_len(buf, tup.len());
for val in tup {
val.with_dump(|val| serialize_value(buf, val))??
}
}
DumpableValue::None => {
buf.write_u8(Type::None as u8);
}
DumpableValue::Ellipsis => {
buf.write_u8(Type::Ellipsis as u8);
}
DumpableValue::StopIter => {
buf.write_u8(Type::StopIter as u8);
}
DumpableValue::List(l) => {
buf.write_u8(Type::List as u8);
write_len(buf, l.len());
for val in l {
val.with_dump(|val| serialize_value(buf, val))??
}
}
DumpableValue::Set(set) => {
buf.write_u8(Type::Set as u8);
write_len(buf, set.len());
for val in set {
val.with_dump(|val| serialize_value(buf, val))??
}
}
DumpableValue::Frozenset(set) => {
buf.write_u8(Type::FrozenSet as u8);
write_len(buf, set.len());
for val in set {
val.with_dump(|val| serialize_value(buf, val))??
}
}
DumpableValue::Dict(d) => {
buf.write_u8(Type::Dict as u8);
write_len(buf, d.len());
for (k, v) in d {
k.with_dump(|val| serialize_value(buf, val))??;
v.with_dump(|val| serialize_value(buf, val))??;
}
}
}
Ok(())
}
pub fn serialize_code<W: Write, C: Constant>(buf: &mut W, code: &CodeObject<C>) {
write_len(buf, code.instructions.len());
// SAFETY: it's ok to transmute CodeUnit to [u8; 2]
let (_, instructions_bytes, _) = unsafe { code.instructions.align_to() };
buf.write_slice(instructions_bytes);
write_len(buf, code.locations.len());
for loc in &*code.locations {
buf.write_u32(loc.row.get());
buf.write_u32(loc.column.to_zero_indexed());
}
buf.write_u16(code.flags.bits());
buf.write_u32(code.posonlyarg_count);
buf.write_u32(code.arg_count);
buf.write_u32(code.kwonlyarg_count);
write_vec(buf, code.source_path.as_ref().as_bytes());
buf.write_u32(code.first_line_number.map_or(0, |x| x.get()));
buf.write_u32(code.max_stackdepth);
write_vec(buf, code.obj_name.as_ref().as_bytes());
let cell2arg = code.cell2arg.as_deref().unwrap_or(&[]);
write_len(buf, cell2arg.len());
for &i in cell2arg {
buf.write_u32(i as u32)
}
write_len(buf, code.constants.len());
for constant in &*code.constants {
serialize_value(buf, constant.borrow_constant().into()).unwrap_or_else(|x| match x {})
}
let mut write_names = |names: &[C::Name]| {
write_len(buf, names.len());
for name in names {
write_vec(buf, name.as_ref().as_bytes());
}
};
write_names(&code.names);
write_names(&code.varnames);
write_names(&code.cellvars);
write_names(&code.freevars);
}

33
compiler/core/src/mode.rs Normal file
View File

@@ -0,0 +1,33 @@
pub use rustpython_parser_core::mode::ModeParseError;
#[derive(Clone, Copy)]
pub enum Mode {
Exec,
Eval,
Single,
BlockExpr,
}
impl std::str::FromStr for Mode {
type Err = ModeParseError;
// To support `builtins.compile()` `mode` argument
fn from_str(s: &str) -> Result<Self, Self::Err> {
match s {
"exec" => Ok(Mode::Exec),
"eval" => Ok(Mode::Eval),
"single" => Ok(Mode::Single),
_ => Err(ModeParseError),
}
}
}
impl From<Mode> for rustpython_parser_core::Mode {
fn from(mode: Mode) -> Self {
match mode {
Mode::Exec => Self::Module,
Mode::Eval => Self::Expression,
Mode::Single | Mode::BlockExpr => Self::Interactive,
}
}
}

37
compiler/src/lib.rs
View File

@@ -2,32 +2,30 @@ use rustpython_codegen::{compile, symboltable};
use rustpython_parser::ast::{fold::Fold, ConstantOptimizer};
pub use rustpython_codegen::compile::CompileOpts;
pub use rustpython_compiler_core::{CodeObject, Mode};
pub use rustpython_compiler_core::{bytecode::CodeObject, Mode};
pub use rustpython_parser::source_code::SourceLocator;
// these modules are out of repository. re-exporting them here for convenience.
pub use rustpython_codegen as codegen;
pub use rustpython_compiler_core as core;
pub use rustpython_parser as parser;
use std::error::Error as StdError;
use std::fmt;
#[derive(Debug)]
pub enum CompileErrorType {
Codegen(rustpython_codegen::error::CodegenErrorType),
Parse(parser::ParseErrorType),
}
impl StdError for CompileErrorType {
fn source(&self) -> Option<&(dyn StdError + 'static)> {
impl std::error::Error for CompileErrorType {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
CompileErrorType::Codegen(e) => e.source(),
CompileErrorType::Parse(e) => e.source(),
}
}
}
impl fmt::Display for CompileErrorType {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
impl std::fmt::Display for CompileErrorType {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
CompileErrorType::Codegen(e) => e.fmt(f),
CompileErrorType::Parse(e) => e.fmt(f),
@@ -45,39 +43,46 @@ impl From<parser::ParseErrorType> for CompileErrorType {
}
}
pub type CompileError = rustpython_compiler_core::BaseError<CompileErrorType>;
pub type CompileError = rustpython_parser::source_code::LocatedError<CompileErrorType>;
/// Compile a given source code into a bytecode object.
pub fn compile(
source: &str,
mode: compile::Mode,
mode: Mode,
source_path: String,
opts: CompileOpts,
) -> Result<CodeObject, CompileError> {
let mut locator = SourceLocator::new(source);
let mut ast = match parser::parse(source, mode.into(), &source_path) {
Ok(x) => x,
Err(e) => return Err(e.into()),
Err(e) => return Err(locator.locate_error(e)),
};
if opts.optimize > 0 {
ast = ConstantOptimizer::new()
.fold_mod(ast)
.unwrap_or_else(|e| match e {});
}
let ast = locator.fold_mod(ast).unwrap_or_else(|e| match e {});
compile::compile_top(&ast, source_path, mode, opts).map_err(|e| e.into())
}
pub fn compile_symtable(
source: &str,
mode: compile::Mode,
mode: Mode,
source_path: &str,
) -> Result<symboltable::SymbolTable, CompileError> {
let mut locator = SourceLocator::new(source);
let res = match mode {
compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
let ast = parser::parse_program(source, source_path).map_err(|e| e.into())?;
Mode::Exec | Mode::Single | Mode::BlockExpr => {
let ast =
parser::parse_program(source, source_path).map_err(|e| locator.locate_error(e))?;
let ast = locator.fold(ast).unwrap();
symboltable::SymbolTable::scan_program(&ast)
}
compile::Mode::Eval => {
let expr = parser::parse_expression(source, source_path).map_err(|e| e.into())?;
Mode::Eval => {
let expr = parser::parse_expression(source, source_path)
.map_err(|e| locator.locate_error(e))?;
let expr = locator.fold(expr).unwrap();
symboltable::SymbolTable::scan_expr(&expr)
}
};

3
derive-impl/Cargo.toml
View File

@@ -5,7 +5,8 @@ edition = "2021"
[dependencies]
rustpython-compiler-core = { workspace = true }
rustpython-doc = { git = "https://github.com/RustPython/__doc__", branch = "main" }
rustpython-parser-core = { workspace = true }
rustpython-doc = { workspace = true }
indexmap = { workspace = true }
itertools = { workspace = true }

16
derive-impl/src/compile_bytecode.rs
View File

@@ -17,7 +17,7 @@ use crate::{extract_spans, Diagnostic};
use once_cell::sync::Lazy;
use proc_macro2::{Span, TokenStream};
use quote::quote;
use rustpython_compiler_core::{frozen_lib, CodeObject, Mode};
use rustpython_compiler_core::{bytecode::CodeObject, frozen, Mode};
use std::{
collections::HashMap,
env, fs,
@@ -318,7 +318,7 @@ impl PyCompileInput {
source,
mode: mode.unwrap_or(Mode::Exec),
module_name: module_name.unwrap_or_else(|| "frozen".to_owned()),
crate_name: crate_name.unwrap_or_else(|| syn::parse_quote!(::rustpython_vm::bytecode)),
crate_name: crate_name.unwrap_or_else(|| syn::parse_quote!(::rustpython_vm)),
})
}
}
@@ -374,11 +374,11 @@ pub fn impl_py_compile(
.source
.compile_single(args.mode, args.module_name, compiler)?;
let frozen = frozen_lib::FrozenCodeObject::encode(&code);
let frozen = frozen::FrozenCodeObject::encode(&code);
let bytes = LitByteStr::new(&frozen.bytes, Span::call_site());
let output = quote! {
#crate_name::frozen_lib::FrozenCodeObject { bytes: &#bytes[..] }
#crate_name::frozen::FrozenCodeObject { bytes: &#bytes[..] }
};
Ok(output)
@@ -394,9 +394,9 @@ pub fn impl_py_freeze(
let crate_name = args.crate_name;
let code_map = args.source.compile(args.mode, args.module_name, compiler)?;
let data = frozen_lib::FrozenLib::encode(code_map.iter().map(|(k, v)| {
let v = frozen_lib::FrozenModule {
code: frozen_lib::FrozenCodeObject::encode(&v.code),
let data = frozen::FrozenLib::encode(code_map.iter().map(|(k, v)| {
let v = frozen::FrozenModule {
code: frozen::FrozenCodeObject::encode(&v.code),
package: v.package,
};
(&**k, v)
@@ -404,7 +404,7 @@ pub fn impl_py_freeze(
let bytes = LitByteStr::new(&data.bytes, Span::call_site());
let output = quote! {
#crate_name::frozen_lib::FrozenLib::from_ref(#bytes)
#crate_name::frozen::FrozenLib::from_ref(#bytes)
};
Ok(output)

6
jit/src/instructions.rs
View File

@@ -1,8 +1,8 @@
use cranelift::prelude::*;
use num_traits::cast::ToPrimitive;
use rustpython_compiler_core::{
self as bytecode, BinaryOperator, BorrowedConstant, CodeObject, ComparisonOperator,
Instruction, Label, OpArg, OpArgState, UnaryOperator,
use rustpython_compiler_core::bytecode::{
self, BinaryOperator, BorrowedConstant, CodeObject, ComparisonOperator, Instruction, Label,
OpArg, OpArgState, UnaryOperator,
};
use std::collections::HashMap;

2
jit/src/lib.rs
View File

@@ -4,7 +4,7 @@ use cranelift::prelude::*;
use cranelift_jit::{JITBuilder, JITModule};
use cranelift_module::{FuncId, Linkage, Module, ModuleError};
use instructions::FunctionCompiler;
use rustpython_compiler_core as bytecode;
use rustpython_compiler_core::bytecode;
use std::{fmt, mem::ManuallyDrop};
#[derive(Debug, thiserror::Error)]

6
jit/tests/common.rs
View File

@@ -1,4 +1,6 @@
use rustpython_compiler_core::{CodeObject, ConstantData, Instruction, OpArg, OpArgState};
use rustpython_compiler_core::bytecode::{
CodeObject, ConstantData, Instruction, OpArg, OpArgState,
};
use rustpython_jit::{CompiledCode, JitType};
use std::collections::HashMap;
use std::ops::ControlFlow;
@@ -165,7 +167,7 @@ macro_rules! jit_function {
crate_name = "rustpython_compiler_core",
source = $($t)*
);
let code = code.decode(rustpython_compiler_core::BasicBag);
let code = code.decode(rustpython_compiler_core::bytecode::BasicBag);
let mut machine = $crate::common::StackMachine::new();
machine.run(code);
machine.get_function(stringify!($func_name)).compile()

2
pylib/src/lib.rs
View File

@@ -10,5 +10,5 @@ pub const LIB_PATH: &str = match option_env!("win_lib_path") {
};
#[cfg(feature = "freeze-stdlib")]
pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen_lib::FrozenLib =
pub const FROZEN_STDLIB: &rustpython_compiler_core::frozen::FrozenLib =
rustpython_derive::py_freeze!(dir = "./Lib", crate_name = "rustpython_compiler_core");

2
src/shell.rs
View File

@@ -55,7 +55,7 @@ fn shell_exec(
p,
ParseErrorType::Lexical(LexicalErrorType::IndentationError)
) {
continuing && err.location.column() != 0
continuing && err.location.is_some()
} else {
!matches!(p, ParseErrorType::UnrecognizedToken(Tok::Dedent, _))
}

2
stdlib/src/faulthandler.rs
View File

@@ -10,7 +10,7 @@ mod decl {
stderr,
" File \"{}\", line {} in {}",
frame.code.source_path,
frame.current_location().row(),
frame.current_location().row.to_usize(),
frame.code.obj_name
)
}

3
vm/Cargo.toml
View File

@@ -33,6 +33,7 @@ rustpython-jit = { path = "../jit", optional = true, version = "0.2.0" }
rustpython-ast = { workspace = true, optional = true }
rustpython-parser = { workspace = true, optional = true }
rustpython-compiler-core = { workspace = true }
rustpython-parser-core = { workspace = true }
rustpython-literal = { workspace = true }
ascii = { workspace = true }
@@ -69,7 +70,7 @@ caseless = "0.2.1"
getrandom = { version = "0.2.6", features = ["js"] }
flamer = { version = "0.4", optional = true }
half = "1.8.2"
is-macro = "0.2.0"
is-macro = "0.2.2"
memchr = "2.4.1"
memoffset = "0.6.5"
optional = "0.5.0"

12
vm/src/builtins/code.rs
View File

@@ -8,7 +8,9 @@ use crate::{
bytecode::{self, AsBag, BorrowedConstant, CodeFlags, Constant, ConstantBag},
class::{PyClassImpl, StaticType},
convert::ToPyObject,
frozen,
function::{FuncArgs, OptionalArg},
source_code::OneIndexed,
types::Representable,
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyResult, VirtualMachine,
};
@@ -180,7 +182,7 @@ impl IntoCodeObject for bytecode::CodeObject {
}
}
impl<B: AsRef<[u8]>> IntoCodeObject for bytecode::frozen_lib::FrozenCodeObject<B> {
impl<B: AsRef<[u8]>> IntoCodeObject for frozen::FrozenCodeObject<B> {
fn into_code_object(self, ctx: &Context) -> CodeObject {
self.decode(ctx)
}
@@ -225,7 +227,7 @@ impl Representable for PyCode {
code.obj_name,
zelf.get_id(),
code.source_path.as_str(),
code.first_line_number
code.first_line_number.map_or(-1, |n| n.get() as i32)
))
}
}
@@ -275,8 +277,8 @@ impl PyCode {
}
#[pygetset]
fn co_firstlineno(&self) -> usize {
self.code.first_line_number as usize
fn co_firstlineno(&self) -> u32 {
self.code.first_line_number.map_or(0, |n| n.get())
}
#[pygetset]
@@ -348,7 +350,7 @@ impl PyCode {
};
let first_line_number = match args.co_firstlineno {
OptionalArg::Present(first_line_number) => first_line_number,
OptionalArg::Present(first_line_number) => OneIndexed::new(first_line_number),
OptionalArg::Missing => self.code.first_line_number,
};

2
vm/src/builtins/frame.rs
View File

@@ -60,7 +60,7 @@ impl Frame {
#[pygetset]
pub fn f_lineno(&self) -> usize {
self.current_location().row()
self.current_location().row.to_usize()
}
#[pygetset]

12
vm/src/builtins/traceback.rs
View File

@@ -1,7 +1,9 @@
use rustpython_common::lock::PyMutex;
use super::PyType;
use crate::{class::PyClassImpl, frame::FrameRef, Context, Py, PyPayload, PyRef};
use crate::{
class::PyClassImpl, frame::FrameRef, source_code::LineNumber, Context, Py, PyPayload, PyRef,
};
#[pyclass(module = false, name = "traceback", traverse)]
#[derive(Debug)]
@@ -11,7 +13,7 @@ pub struct PyTraceback {
#[pytraverse(skip)]
pub lasti: u32,
#[pytraverse(skip)]
pub lineno: usize,
pub lineno: LineNumber,
}
pub type PyTracebackRef = PyRef<PyTraceback>;
@@ -24,7 +26,7 @@ impl PyPayload for PyTraceback {
#[pyclass]
impl PyTraceback {
pub fn new(next: Option<PyRef<Self>>, frame: FrameRef, lasti: u32, lineno: usize) -> Self {
pub fn new(next: Option<PyRef<Self>>, frame: FrameRef, lasti: u32, lineno: LineNumber) -> Self {
PyTraceback {
next: PyMutex::new(next),
frame,
@@ -45,7 +47,7 @@ impl PyTraceback {
#[pygetset]
fn tb_lineno(&self) -> usize {
self.lineno
self.lineno.to_usize()
}
#[pygetset]
@@ -76,7 +78,7 @@ impl serde::Serialize for PyTraceback {
let mut struc = s.serialize_struct("PyTraceback", 3)?;
struc.serialize_field("name", self.frame.code.obj_name.as_str())?;
struc.serialize_field("lineno", &self.lineno)?;
struc.serialize_field("lineno", &self.lineno.get())?;
struc.serialize_field("filename", self.frame.code.source_path.as_str())?;
struc.end()
}

7
vm/src/compiler.rs
View File

@@ -4,7 +4,12 @@ use crate::{builtins::PyBaseExceptionRef, convert::ToPyException, VirtualMachine
pub use rustpython_codegen::CompileOpts;
#[cfg(feature = "rustpython-compiler")]
pub use rustpython_compiler::*;
pub use rustpython_compiler_core::Mode;
#[cfg(not(feature = "rustpython-compiler"))]
pub use rustpython_compiler_core as core;
#[cfg(all(not(feature = "rustpython-compiler"), feature = "rustpython-parser"))]
pub use rustpython_parser_core as parser;
#[cfg(not(feature = "rustpython-compiler"))]
mod error {

2
vm/src/exceptions.rs
View File

@@ -259,7 +259,7 @@ fn write_traceback_entry<W: Write>(
r##" File "{}", line {}, in {}"##,
filename, tb_entry.lineno, tb_entry.frame.code.obj_name
)?;
print_source_line(output, filename, tb_entry.lineno)?;
print_source_line(output, filename, tb_entry.lineno.to_usize())?;
Ok(())
}

14
vm/src/frame.rs
View File

@@ -14,16 +14,16 @@ use crate::{
function::{ArgMapping, Either, FuncArgs},
protocol::{PyIter, PyIterReturn},
scope::Scope,
source_code::SourceLocation,
stdlib::builtins,
vm::{Context, PyMethod},
AsObject, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject, VirtualMachine,
};
use indexmap::IndexMap;
use itertools::Itertools;
use std::fmt;
use std::iter::zip;
#[cfg(feature = "threading")]
use std::sync::atomic;
use std::{fmt, iter::zip};
#[derive(Clone, Debug)]
struct Block {
@@ -165,7 +165,7 @@ impl Frame {
}
}
pub fn current_location(&self) -> bytecode::Location {
pub fn current_location(&self) -> SourceLocation {
self.code.locations[self.lasti() as usize - 1]
}
@@ -376,12 +376,8 @@ impl ExecutingFrame<'_> {
let loc = frame.code.locations[idx];
let next = exception.traceback();
let new_traceback = PyTraceback::new(
next,
frame.object.to_owned(),
frame.lasti(),
loc.row(),
);
let new_traceback =
PyTraceback::new(next, frame.object.to_owned(), frame.lasti(), loc.row);
vm_trace!("Adding to traceback: {:?} {:?}", new_traceback, loc.row());
exception.set_traceback(Some(new_traceback.into_ref(&vm.ctx)));

37
vm/src/frozen.rs
View File

@@ -1,37 +0,0 @@
use crate::bytecode::frozen_lib::FrozenModule;
pub fn core_frozen_inits() -> impl Iterator<Item = (&'static str, FrozenModule)> {
let iter = std::iter::empty();
macro_rules! ext_modules {
($iter:ident, $($t:tt)*) => {
let $iter = $iter.chain(py_freeze!($($t)*));
};
}
// keep as example but use file one now
// ext_modules!(
// iter,
// source = "initialized = True; print(\"Hello world!\")\n",
// module_name = "__hello__",
// );
// Python modules that the vm calls into, but are not actually part of the stdlib. They could
// in theory be implemented in Rust, but are easiest to do in Python for one reason or another.
// Includes _importlib_bootstrap and _importlib_bootstrap_external
ext_modules!(
iter,
dir = "./Lib/python_builtins",
crate_name = "rustpython_compiler_core"
);
// core stdlib Python modules that the vm calls into, but are still used in Python
// application code, e.g. copyreg
#[cfg(not(feature = "freeze-stdlib"))]
ext_modules!(
iter,
dir = "./Lib/core_modules",
crate_name = "rustpython_compiler_core"
);
iter
}

1
vm/src/function/method.rs
View File

@@ -10,6 +10,7 @@ use crate::{
bitflags::bitflags! {
// METH_XXX flags in CPython
#[derive(Copy, Clone, Debug, PartialEq)]
pub struct PyMethodFlags: u32 {
// const VARARGS = 0x0001;
// const KEYWORDS = 0x0002;

5
vm/src/lib.rs
View File

@@ -55,7 +55,6 @@ pub mod eval;
pub mod exceptions;
pub mod format;
pub mod frame;
mod frozen;
pub mod function;
pub mod import;
mod intern;
@@ -72,6 +71,7 @@ pub mod scope;
pub mod sequence;
pub mod signal;
pub mod sliceable;
mod source;
pub mod stdlib;
pub mod suggestion;
pub mod types;
@@ -80,6 +80,7 @@ pub mod version;
pub mod vm;
pub mod warn;
pub use self::compiler::parser::source_code;
pub use self::convert::{TryFromBorrowedObject, TryFromObject};
pub use self::object::{
AsObject, Py, PyAtomicRef, PyExact, PyObject, PyObjectRef, PyPayload, PyRef, PyRefExact,
@@ -88,7 +89,7 @@ pub use self::object::{
pub use self::vm::{Context, Interpreter, Settings, VirtualMachine};
pub use rustpython_common as common;
pub use rustpython_compiler_core as bytecode;
pub use rustpython_compiler_core::{bytecode, frozen};
pub use rustpython_literal as literal;
#[doc(hidden)]

20
vm/src/source.rs Normal file
View File

@@ -0,0 +1,20 @@
use crate::source_code::SourceLocation;
// pub(crate) fn new_location_error(
// index: usize,
// field: &str,
// vm: &VirtualMachine,
// ) -> PyRef<PyBaseException> {
// vm.new_value_error(format!("value {index} is too large for location {field}"))
// }
pub(crate) struct AtLocation<'a>(pub Option<&'a SourceLocation>);
impl std::fmt::Display for AtLocation<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let (row, column) = self
.0
.map_or((0, 0), |l| (l.row.to_usize(), l.column.to_usize()));
write!(f, " at line {row} column {column}",)
}
}

112
vm/src/stdlib/ast.rs
View File

@@ -9,13 +9,18 @@ use crate::{
builtins::{self, PyModule, PyStrRef, PyType},
class::{PyClassImpl, StaticType},
compiler::CompileError,
compiler::{
core::bytecode::OpArgType,
parser::text_size::{TextRange, TextSize},
},
convert::ToPyException,
source_code::{OneIndexed, SourceLocation, SourceLocator, SourceRange},
AsObject, Context, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, TryFromObject,
VirtualMachine,
};
use num_complex::Complex64;
use num_traits::{ToPrimitive, Zero};
use rustpython_ast as ast;
use rustpython_ast::{self as ast, fold::Fold};
#[cfg(feature = "rustpython-codegen")]
use rustpython_codegen as codegen;
#[cfg(feature = "rustpython-parser")]
@@ -144,35 +149,44 @@ impl<T: Node> Node for Option<T> {
}
}
impl<T: NamedNode> Node for ast::Located<T> {
impl<T: NamedNode> Node for ast::located::Located<T> {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
let location = self.location();
let end_location = self.end_location();
let obj = self.node.ast_to_object(vm);
node_add_location(&obj, self.location, self.end_location, vm);
node_add_location(&obj, location, end_location, vm);
obj
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
let location = ast::Location::new(
Node::ast_from_object(vm, get_node_field(vm, &object, "lineno", T::NAME)?)?,
Node::ast_from_object(vm, get_node_field(vm, &object, "col_offset", T::NAME)?)?,
);
let end_location = if let (Some(end_lineno), Some(end_col_offset)) = (
get_node_field_opt(vm, &object, "end_lineno")?
.map(|obj| Node::ast_from_object(vm, obj))
.transpose()?,
get_node_field_opt(vm, &object, "end_col_offset")?
.map(|obj| Node::ast_from_object(vm, obj))
.transpose()?,
) {
Some(ast::Location::new(end_lineno, end_col_offset))
fn make_location(row: u32, column: u32) -> Option<SourceLocation> {
Some(SourceLocation {
row: OneIndexed::new(row)?,
column: OneIndexed::from_zero_indexed(column),
})
}
let row = ast::Int::ast_from_object(vm, get_node_field(vm, &object, "lineno", T::NAME)?)?;
let column =
ast::Int::ast_from_object(vm, get_node_field(vm, &object, "col_offset", T::NAME)?)?;
let location = make_location(row.to_u32(), column.to_u32());
let end_row = get_node_field_opt(vm, &object, "end_lineno")?
.map(|obj| ast::Int::ast_from_object(vm, obj))
.transpose()?;
let end_column = get_node_field_opt(vm, &object, "end_col_offset")?
.map(|obj| ast::Int::ast_from_object(vm, obj))
.transpose()?;
let end_location = if let (Some(row), Some(column)) = (end_row, end_column) {
make_location(row.to_u32(), column.to_u32())
} else {
None
};
let node = T::ast_from_object(vm, object)?;
Ok(ast::Located {
location,
end_location,
custom: (),
Ok(ast::located::Located {
range: TextRange::empty(TextSize::new(0)),
custom: SourceRange {
start: location.unwrap_or(SourceLocation::default()),
end: end_location,
},
node,
})
}
@@ -180,44 +194,66 @@ impl<T: NamedNode> Node for ast::Located<T> {
fn node_add_location(
node: &PyObject,
location: ast::Location,
end_location: Option<ast::Location>,
location: SourceLocation,
end_location: Option<SourceLocation>,
vm: &VirtualMachine,
) {
let dict = node.dict().unwrap();
dict.set_item("lineno", vm.ctx.new_int(location.row()).into(), vm)
.unwrap();
dict.set_item("col_offset", vm.ctx.new_int(location.column()).into(), vm)
dict.set_item("lineno", vm.ctx.new_int(location.row.get()).into(), vm)
.unwrap();
dict.set_item(
"col_offset",
vm.ctx.new_int(location.column.to_zero_indexed()).into(),
vm,
)
.unwrap();
if let Some(end_location) = end_location {
dict.set_item("end_lineno", vm.ctx.new_int(end_location.row()).into(), vm)
.unwrap();
dict.set_item(
"end_lineno",
vm.ctx.new_int(end_location.row.get()).into(),
vm,
)
.unwrap();
dict.set_item(
"end_col_offset",
vm.ctx.new_int(end_location.column()).into(),
vm.ctx.new_int(end_location.column.to_zero_indexed()).into(),
vm,
)
.unwrap();
};
}
impl Node for String {
impl Node for ast::String {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
vm.ctx.new_str(self).into()
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
PyStrRef::try_from_object(vm, object).map(|s| s.as_str().to_owned())
let py_str = PyStrRef::try_from_object(vm, object)?;
Ok(py_str.as_str().to_owned())
}
}
impl Node for usize {
impl Node for ast::Identifier {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
vm.ctx.new_int(self).into()
let id: String = self.into();
vm.ctx.new_str(id).into()
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
object.try_into_value(vm)
let py_str = PyStrRef::try_from_object(vm, object)?;
Ok(ast::Identifier::new(py_str.as_str()))
}
}
impl Node for ast::Int {
fn ast_to_object(self, vm: &VirtualMachine) -> PyObjectRef {
vm.ctx.new_int(self.to_u32()).into()
}
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
let value = object.try_into_value(vm)?;
Ok(ast::Int::new(value))
}
}
@@ -295,8 +331,8 @@ impl Node for ast::ConversionFlag {
fn ast_from_object(vm: &VirtualMachine, object: PyObjectRef) -> PyResult<Self> {
i32::try_from_object(vm, object)?
.to_usize()
.and_then(|f| f.try_into().ok())
.to_u32()
.and_then(ast::ConversionFlag::from_op_arg)
.ok_or_else(|| vm.new_value_error("invalid conversion flag".to_owned()))
}
}
@@ -307,7 +343,9 @@ pub(crate) fn parse(
source: &str,
mode: parser::Mode,
) -> Result<PyObjectRef, CompileError> {
let top = parser::parse(source, mode, "<unknown>").map_err(CompileError::from)?;
let mut locator = SourceLocator::new(source);
let top = parser::parse(source, mode, "<unknown>").map_err(|e| locator.locate_error(e))?;
let top = locator.fold_mod(top).unwrap();
Ok(top.ast_to_object(vm))
}
@@ -316,7 +354,7 @@ pub(crate) fn compile(
vm: &VirtualMachine,
object: PyObjectRef,
filename: &str,
mode: codegen::compile::Mode,
mode: crate::compiler::Mode,
) -> PyResult {
let opts = vm.compile_opts();
let ast = Node::ast_from_object(vm, object)?;

586
vm/src/stdlib/ast/gen.rs generated
View File

File diff suppressed because it is too large Load Diff

2
vm/src/stdlib/imp.rs
View File

@@ -1,4 +1,4 @@
use crate::bytecode::frozen_lib::FrozenModule;
use crate::frozen::FrozenModule;
use crate::{builtins::PyBaseExceptionRef, VirtualMachine};
pub(crate) use _imp::make_module;

3
vm/src/stdlib/io.rs
View File

@@ -722,7 +722,7 @@ mod _io {
}
bitflags::bitflags! {
#[derive(Default)]
#[derive(Copy, Clone, Debug, PartialEq, Default)]
struct BufferedFlags: u8 {
const DETACHED = 1 << 0;
const WRITABLE = 1 << 1;
@@ -3723,6 +3723,7 @@ mod fileio {
use std::io::{Read, Write};
bitflags::bitflags! {
#[derive(Copy, Clone, Debug, PartialEq)]
struct Mode: u8 {
const CREATED = 0b0001;
const READABLE = 0b0010;

3
vm/src/stdlib/marshal.rs
View File

@@ -215,6 +215,9 @@ mod decl {
marshal::MarshalError::InvalidUtf8 => {
vm.new_value_error("invalid utf8 in marshalled string".to_owned())
}
marshal::MarshalError::InvalidLocation => {
vm.new_value_error("invalid location in marshalled object".to_owned())
}
marshal::MarshalError::BadType => {
vm.new_value_error("bad marshal data (unknown type code)".to_owned())
}

1
vm/src/stdlib/posix.rs
View File

@@ -164,6 +164,7 @@ pub mod module {
// Flags for os_access
bitflags! {
#[derive(Copy, Clone, Debug, PartialEq)]
pub struct AccessFlags: u8 {
const F_OK = _os::F_OK;
const R_OK = _os::R_OK;

9
vm/src/stdlib/symtable.rs
View File

@@ -16,7 +16,7 @@ mod symtable {
filename: PyStrRef,
mode: PyStrRef,
vm: &VirtualMachine,
) -> PyResult<PySymbolTableRef> {
) -> PyResult<PyRef<PySymbolTable>> {
let mode = mode
.as_str()
.parse::<compiler::Mode>()
@@ -33,9 +33,6 @@ mod symtable {
PySymbolTable { symtable }
}
type PySymbolTableRef = PyRef<PySymbolTable>;
type PySymbolRef = PyRef<PySymbol>;
#[pyattr]
#[pyclass(name = "SymbolTable")]
#[derive(PyPayload)]
@@ -62,7 +59,7 @@ mod symtable {
}
#[pymethod]
fn get_lineno(&self) -> usize {
fn get_lineno(&self) -> u32 {
self.symtable.line_number
}
@@ -77,7 +74,7 @@ mod symtable {
}
#[pymethod]
fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<PySymbolRef> {
fn lookup(&self, name: PyStrRef, vm: &VirtualMachine) -> PyResult<PyRef<PySymbol>> {
let name = name.as_str();
if let Some(symbol) = self.symtable.symbols.get(name) {
Ok(PySymbol {

9
vm/src/types/slot.rs
View File

@@ -118,6 +118,7 @@ impl std::fmt::Debug for PyTypeSlots {
}
bitflags! {
#[derive(Copy, Clone, Debug, PartialEq)]
#[non_exhaustive]
pub struct PyTypeFlags: u64 {
const IMMUTABLETYPE = 1 << 8;
@@ -140,10 +141,10 @@ impl PyTypeFlags {
/// Used for types created in Python. Subclassable and are a
/// heaptype.
pub const fn heap_type_flags() -> Self {
unsafe {
Self::from_bits_unchecked(
Self::DEFAULT.bits | Self::HEAPTYPE.bits | Self::BASETYPE.bits,
)
match Self::from_bits(Self::DEFAULT.bits() | Self::HEAPTYPE.bits() | Self::BASETYPE.bits())
{
Some(flags) => flags,
None => unreachable!(),
}
}

41
vm/src/vm/mod.rs
View File

@@ -22,12 +22,11 @@ use crate::{
PyBaseExceptionRef, PyDictRef, PyInt, PyList, PyModule, PyStr, PyStrInterned, PyStrRef,
PyTypeRef,
},
bytecode::frozen_lib::FrozenModule,
codecs::CodecsRegistry,
common::{hash::HashSecret, lock::PyMutex, rc::PyRc},
convert::ToPyObject,
frame::{ExecutionResult, Frame, FrameRef},
frozen,
frozen::FrozenModule,
function::{ArgMapping, FuncArgs, PySetterValue},
import,
protocol::PyIterIter,
@@ -196,7 +195,7 @@ impl VirtualMachine {
panic!("Interpreters in same process must share the hash seed");
}
let frozen = frozen::core_frozen_inits().collect();
let frozen = core_frozen_inits().collect();
PyRc::get_mut(&mut vm.state).unwrap().frozen = frozen;
vm.builtins.init_dict(
@@ -837,6 +836,42 @@ impl AsRef<Context> for VirtualMachine {
}
}
fn core_frozen_inits() -> impl Iterator<Item = (&'static str, FrozenModule)> {
let iter = std::iter::empty();
macro_rules! ext_modules {
($iter:ident, $($t:tt)*) => {
let $iter = $iter.chain(py_freeze!($($t)*));
};
}
// keep as example but use file one now
// ext_modules!(
// iter,
// source = "initialized = True; print(\"Hello world!\")\n",
// module_name = "__hello__",
// );
// Python modules that the vm calls into, but are not actually part of the stdlib. They could
// in theory be implemented in Rust, but are easiest to do in Python for one reason or another.
// Includes _importlib_bootstrap and _importlib_bootstrap_external
ext_modules!(
iter,
dir = "./Lib/python_builtins",
crate_name = "rustpython_compiler_core"
);
// core stdlib Python modules that the vm calls into, but are still used in Python
// application code, e.g. copyreg
#[cfg(not(feature = "freeze-stdlib"))]
ext_modules!(
iter,
dir = "./Lib/core_modules",
crate_name = "rustpython_compiler_core"
);
iter
}
#[test]
fn test_nested_frozen() {
use rustpython_vm as vm;

35
vm/src/vm/vm_new.rs
View File

@@ -8,6 +8,8 @@ use crate::{
convert::ToPyObject,
function::{IntoPyNativeFn, PyMethodFlags},
scope::Scope,
source::AtLocation,
source_code::SourceLocation,
vm::VirtualMachine,
AsObject, Py, PyObject, PyObjectRef, PyRef,
};
@@ -266,11 +268,12 @@ impl VirtualMachine {
}
.to_owned();
fn get_statement(source: &str, loc: rustpython_compiler_core::Location) -> Option<String> {
if loc.column() == 0 || loc.row() == 0 {
return None;
}
let line = source.split('\n').nth(loc.row() - 1)?.to_owned();
// TODO: replace to SourceCode
fn get_statement(source: &str, loc: Option<SourceLocation>) -> Option<String> {
let line = source
.split('\n')
.nth(loc?.row.to_zero_indexed_usize())?
.to_owned();
Some(line + "\n")
}
@@ -288,10 +291,21 @@ impl VirtualMachine {
let loc = error.location;
if let Some(ref stmt) = statement {
// visualize the error when location and statement are provided
loc.fmt_with(f, &error.error)?;
write!(f, "\n{stmt}{arrow:>pad$}", pad = loc.column(), arrow = "^")
write!(
f,
"{error}{at_location}\n{stmt}{arrow:>pad$}",
error = error.error,
at_location = AtLocation(loc.as_ref()),
pad = loc.map_or(0, |loc| loc.column.to_usize()),
arrow = "^"
)
} else {
loc.fmt_with(f, &error.error)
write!(
f,
"{error}{at_location}",
error = error.error,
at_location = AtLocation(loc.as_ref()),
)
}
}
@@ -299,8 +313,9 @@ impl VirtualMachine {
fmt(error, statement.as_deref(), &mut msg).unwrap();
let syntax_error = self.new_exception_msg(syntax_error_type, msg);
let lineno = self.ctx.new_int(error.location.row());
let offset = self.ctx.new_int(error.location.column());
let (lineno, offset) = error.python_location();
let lineno = self.ctx.new_int(lineno);
let offset = self.ctx.new_int(offset);
syntax_error
.as_object()
.set_attr("lineno", lineno, self)

8
wasm/lib/src/convert.rs
View File

@@ -246,11 +246,15 @@ pub fn js_to_py(vm: &VirtualMachine, js_val: JsValue) -> PyObjectRef {
pub fn syntax_err(err: CompileError) -> SyntaxError {
let js_err = SyntaxError::new(&format!("Error parsing Python code: {err}"));
let _ = Reflect::set(&js_err, &"row".into(), &(err.location.row() as u32).into());
let _ = Reflect::set(
&js_err,
&"row".into(),
&(err.location.unwrap().row.get()).into(),
);
let _ = Reflect::set(
&js_err,
&"col".into(),
&(err.location.column() as u32).into(),
&(err.location.unwrap().column.get()).into(),
);
let can_continue = matches!(&err.error, CompileErrorType::Parse(ParseErrorType::Eof));
let _ = Reflect::set(&js_err, &"canContinue".into(), &can_continue.into());